#!/usr/bin/env bash
# chacra-vs-download.sh -- verify every package chacra built for a release
# actually exists on download.ceph.com (production), by filename.
#
# Pure external HTTPS checks against chacra.ceph.com's public REST/file API
# and download.ceph.com's public directory listings. No SSH, no dependency
# on any other script in this repo or on being run from any particular
# host -- just curl + python3 (for parsing chacra's one JSON response).
#
# Usage: chacra-vs-download.sh [--all] <release> <sha1> <version> [package]
#   --all    print every filename compared (matched rows too), not just
#            mismatches -- for positively confirming one exact filename.
#   release  e.g. squid, tentacle
#   sha1     the chacra build sha1
#   version  package version used in download.ceph.com dir names, e.g. 19.2.5
#   package  defaults to "ceph"
#
# Example:
#   chacra-vs-download.sh --all tentacle 06c2f9c35b67055a8a6fb99d1be236b3c4832ace 20.2.3
#
# ponytail: only the "default" flavor and filename-existence (not checksum)
# are checked -- sufficient to catch a silent omission or a partial build
# that got promoted anyway. Add checksum comparison later if that's not
# enough.

set -euo pipefail

usage() {
    cat <<'EOF'
Usage: chacra-vs-download.sh [--all] <release> <sha1> <version> [package]

  --all    print every filename compared (matched rows too), not just
           mismatches -- for positively confirming one exact filename.
  release  e.g. squid, tentacle
  sha1     the chacra build sha1
  version  package version used in download.ceph.com dir names, e.g. 19.2.5
  package  defaults to "ceph"

Example:
  chacra-vs-download.sh --all tentacle 06c2f9c35b67055a8a6fb99d1be236b3c4832ace 20.2.3
EOF
}

if [[ $# -eq 0 ]]; then
    usage
    exit 1
fi
if [[ "$1" == "-h" || "$1" == "--help" ]]; then
    usage
    exit 0
fi

SHOW_ALL=0
if [[ "$1" == "--all" ]]; then
    SHOW_ALL=1
    shift
fi

if [[ $# -lt 3 ]]; then
    usage
    exit 1
fi

RELEASE="$1"
SHA1="$2"
VERSION="$3"
PACKAGE="${4:-ceph}"

CHACRA="https://chacra.ceph.com"
DOWNLOAD="https://download.ceph.com"
FLAVOR="default"

# Expected distros per release -- "Distro Codemap" table, section 2a:
# https://docs.ceph.com/en/latest/dev/release-process/
# (verified 2026-08-05, matches verbatim). windows excluded here: not
# published to download.ceph.com in the deb/rpm sense this script checks.
# ponytail: plain data table, update by hand as releases/distros change --
# unknown $RELEASE just skips this pre-flight check, doesn't error.
declare -A EXPECTED_DISTROS=(
    [squid]="jammy centos9 bookworm"
    [tentacle]="jammy noble centos9 rocky10 bookworm"   # noble added 2026-10-28: ceph/ceph#73212, confirmed by Patrick
    [umbrella]="jammy noble centos9 rocky10 bookworm trixie"
)

RED=$'\033[0;31m'; GREEN=$'\033[0;32m'; YELLOW=$'\033[1;33m'; BLUE=$'\033[0;34m'; BOLD=$'\033[1m'; RESET=$'\033[0m'
MISMATCHES=0

hrefs() { grep -oE 'href="[^"?/][^"]*"' | sed -E 's/href="([^"]*)"/\1/'; }
subdirs() { hrefs | grep '/$' | sed 's#/$##' | grep -v '^\.\.$'; }
fetch() { curl -sf --max-time 20 "$1" 2>/dev/null || true; }

compare_filelists() {
    local label="$1" chacra_list="$2" dl_list="$3"
    local c_count d_count
    c_count=$(wc -l < "$chacra_list")
    d_count=$(wc -l < "$dl_list")
    if [[ "$c_count" -eq 0 && "$d_count" -eq 0 ]]; then
        echo -e "  ${YELLOW}⚠${RESET} ${label} -- no files found on either side (check path assumptions)"
        return
    fi
    if diff -q "$chacra_list" "$dl_list" >/dev/null; then
        printf "  ${GREEN}✓ %-40s chacra:%-4s download:%-4s match${RESET}\n" "$label" "$c_count" "$d_count"
        if [[ "$SHOW_ALL" -eq 1 ]]; then
            sed "s#^#      chacra+download: ${label}: #" "$chacra_list"
        fi
        return
    fi
    MISMATCHES=1
    printf "  ${RED}✗ %-40s chacra:%-4s download:%-4s MISMATCH${RESET}\n" "$label" "$c_count" "$d_count"
    comm -23 "$chacra_list" "$dl_list" | sed 's/^/      missing on download: /' | sed "s/^/${RED}/;s/\$/${RESET}/"
    comm -13 "$chacra_list" "$dl_list" | sed 's/^/      extra on download:   /' | sed "s/^/${RED}/;s/\$/${RESET}/"
    if [[ "$SHOW_ALL" -eq 1 ]]; then
        comm -12 "$chacra_list" "$dl_list" | sed "s#^#      chacra+download: ${label}: #"
    fi
}

TMPDIR=$(mktemp -d); trap 'rm -rf "$TMPDIR"' EXIT

echo -e "${BOLD}chacra vs download.ceph.com -- $PACKAGE $RELEASE $SHA1 (version $VERSION)${RESET}"
echo "----------------------------------------------------------------------"

DISTROS_JSON=$(fetch "$CHACRA/repos/$PACKAGE/$RELEASE/$SHA1/")
if [[ -z "$DISTROS_JSON" ]]; then
    echo -e "${RED}Could not reach chacra or build not found: $CHACRA/repos/$PACKAGE/$RELEASE/$SHA1/${RESET}"
    exit 2
fi

# Pre-flight: catch a distro chacra never built at all for this release --
# the per-distro loop below can only check distros chacra *reports having*,
# so a whole missing distro would otherwise pass silently.
if [[ -n "${EXPECTED_DISTROS[$RELEASE]:-}" ]]; then
    echo -e "${BLUE}Expected $RELEASE distros (excl. windows): ${BOLD}${EXPECTED_DISTROS[$RELEASE]}${RESET}"
    MISSING=$(python3 -c '
import json, sys
tokens = sys.argv[1:]
data = json.load(sys.stdin)
def split_token(t):
    if t in ("jammy", "noble"):
        return "ubuntu", t
    if t in ("bookworm", "trixie"):
        return "debian", t
    if t.startswith("centos"):
        return "centos", t[len("centos"):]
    if t.startswith("rocky"):
        return "rocky", t[len("rocky"):]
    return None, None
missing = []
for t in tokens:
    distro, ver = split_token(t)
    if distro is None or ver not in data.get(distro, []):
        missing.append(t)
print(" ".join(missing))
' ${EXPECTED_DISTROS[$RELEASE]} <<< "$DISTROS_JSON")
    if [[ -n "$MISSING" ]]; then
        echo -e "${RED}✗ chacra has no build at all for: $MISSING${RESET} (expected for $RELEASE, excl. windows)"
        MISMATCHES=1
    else
        echo -e "${GREEN}✓ chacra has a build for every expected $RELEASE distro${RESET} (excl. windows)"
    fi

    # Reverse of the check above: a distro chacra DID build that ISN'T in
    # the matrix -- e.g. noble getting built for a squid release.
    # The per-distro loop below only checks chacra-vs-download consistency
    # for whatever chacra reports, regardless of whether it's expected; a
    # matrix-violating distro that got promoted cleanly would otherwise
    # print a silent green match and never get flagged.
    EXTRA=$(python3 -c '
import json, sys
expected = set(sys.argv[1:])
data = json.load(sys.stdin)
def token_for(distro, ver):
    if distro in ("ubuntu", "debian"):
        return ver
    if distro in ("centos", "rocky"):
        return f"{distro}{ver}"
    return None
extras = []
for distro, versions in data.items():
    if distro == "windows":
        continue
    for v in versions:
        t = token_for(distro, v) or f"{distro}{v}"
        if t not in expected:
            extras.append(f"{distro}/{v}")
print(" ".join(extras))
' ${EXPECTED_DISTROS[$RELEASE]} <<< "$DISTROS_JSON")
    if [[ -n "$EXTRA" ]]; then
        echo -e "${RED}✗ chacra built but NOT in the $RELEASE matrix (possible mistake): $EXTRA${RESET}"
        MISMATCHES=1
    else
        echo -e "${GREEN}✓ chacra built nothing outside the expected $RELEASE matrix${RESET}"
    fi
    echo "----------------------------------------------------------------------"
fi

# Step 1-3: Find authoritative pool for debian/ubuntu (highest .deb count)
# and parse all filenames to extract (codename, arch) groups.
# This fixes the bug where chacra's pool directories are not cleanly isolated
# per codename -- e.g. ubuntu/jammy/pool/ might be empty while debian/trixie/pool/
# contains ALL packages for all codenames mixed together.
declare -A DEB_POOL_PACKAGES  # key: "codename arch", value: newline-separated filenames
AUTH_POOL_DISTRO=""
AUTH_POOL_CODENAME=""
AUTH_POOL_COUNT=0

while IFS=$'\t' read -r DISTRO DVERSION; do
    [[ -z "$DVERSION" ]] && continue
    [[ "$DISTRO" != "debian" && "$DISTRO" != "ubuntu" ]] && continue
    
    POOL_URL="$CHACRA/r/$PACKAGE/$RELEASE/$SHA1/$DISTRO/$DVERSION/flavors/$FLAVOR/pool/main/c/ceph/"
    POOL_LISTING=$(fetch "$POOL_URL")
    if [[ -z "$POOL_LISTING" ]]; then
        continue
    fi
    
    DEB_COUNT=$(echo "$POOL_LISTING" | hrefs | grep -c '\.deb$' || true)
    if [[ "$DEB_COUNT" -gt "$AUTH_POOL_COUNT" ]]; then
        AUTH_POOL_COUNT="$DEB_COUNT"
        AUTH_POOL_DISTRO="$DISTRO"
        AUTH_POOL_CODENAME="$DVERSION"
    fi
done < <(python3 -c "
import json, sys
d = json.load(sys.stdin)
for distro, versions in d.items():
    for v in versions:
        print(f'{distro}\t{v}')
" <<< "$DISTROS_JSON")

if [[ "$AUTH_POOL_COUNT" -gt 0 ]]; then
    echo -e "${BLUE}Authoritative chacra pool: $AUTH_POOL_DISTRO/$AUTH_POOL_CODENAME ($AUTH_POOL_COUNT .deb files)${RESET}"
    
    # Fetch and parse the authoritative pool's complete file listing
    AUTH_POOL_URL="$CHACRA/r/$PACKAGE/$RELEASE/$SHA1/$AUTH_POOL_DISTRO/$AUTH_POOL_CODENAME/flavors/$FLAVOR/pool/main/c/ceph/"
    AUTH_POOL_FILES=$(fetch "$AUTH_POOL_URL" | hrefs | grep '\.deb$' || true)
    
    # Parse each filename: extract codename and arch from pattern like:
    # ceph-common_19.2.5-1jammy_amd64.deb -> codename=jammy, arch=amd64
    # ceph-base_19.2.5-1jammy_all.deb -> codename=jammy, arch=all (goes to all arches)
    while IFS= read -r FILENAME; do
        [[ -z "$FILENAME" ]] && continue
        # Match: _<version>-<build><codename>_<arch>.deb
        if [[ "$FILENAME" =~ _[0-9.]+-[0-9]+([a-z]+)_(amd64|arm64|armhf|i386|all)\.deb$ ]]; then
            CODENAME="${BASH_REMATCH[1]}"
            ARCH="${BASH_REMATCH[2]}"
            
            if [[ "$ARCH" == "all" ]]; then
                # Architecture:all packages belong to every arch for this codename
                for A in amd64 arm64 armhf i386; do
                    KEY="${CODENAME} ${A}"
                    DEB_POOL_PACKAGES["$KEY"]+="${FILENAME}"$'\n'
                done
            else
                KEY="${CODENAME} ${ARCH}"
                DEB_POOL_PACKAGES["$KEY"]+="${FILENAME}"$'\n'
            fi
        fi
    done <<< "$AUTH_POOL_FILES"
    echo "----------------------------------------------------------------------"
fi

while IFS=$'\t' read -r DISTRO DVERSION; do
    [[ -z "$DVERSION" ]] && continue
    case "$DISTRO" in
        debian|ubuntu)
            # Use pool-based package lists instead of per-codename Packages fetch
            DL_MAIN="$DOWNLOAD/debian-$VERSION/dists/$DVERSION/main"
            
            # Determine which arches to check based on what's in the pool
            ARCH_LIST=""
            for A in amd64 arm64 armhf i386; do
                KEY="${DVERSION} ${A}"
                if [[ -n "${DEB_POOL_PACKAGES[$KEY]:-}" ]]; then
                    ARCH_LIST+="binary-${A} "
                fi
            done
            
            [[ -z "$ARCH_LIST" ]] && echo -e "  ${YELLOW}⚠ no packages found for $DISTRO/$DVERSION in authoritative pool${RESET}"
            
            for BINARCH in $ARCH_LIST; do
                ARCH="${BINARCH#binary-}"  # strip "binary-" prefix
                KEY="${DVERSION} ${ARCH}"
                
                CFILE="$TMPDIR/c_${DISTRO}_${DVERSION}_${BINARCH}"
                DFILE="$TMPDIR/d_${DISTRO}_${DVERSION}_${BINARCH}"
                
                # Write chacra packages from pool (already parsed)
                echo -n "${DEB_POOL_PACKAGES[$KEY]}" | grep -v '^$' | sort -u > "$CFILE" || true

                # Fetch download.ceph.com packages (unchanged). "|| true" --
                # a 404 (e.g. wrong VERSION, or a real "never promoted" gap)
                # makes grep return 1 on the empty input, which under
                # pipefail+set -e would otherwise kill the whole script
                # silently mid-run instead of reporting a clean 0-count
                # MISMATCH row (real bug, hit live 2026-08-05).
                fetch "$DL_MAIN/$BINARCH/Packages" | grep '^Filename: ' | sed 's#^Filename: .*/##' | sort -u > "$DFILE" || true
                
                compare_filelists "$DISTRO/$DVERSION/$BINARCH" "$CFILE" "$DFILE"
            done
            ;;
        centos|rocky|rhel|fedora)
            ELVER="el$DVERSION"
            CHACRA_FLAVOR="$CHACRA/r/$PACKAGE/$RELEASE/$SHA1/$DISTRO/$DVERSION/flavors/$FLAVOR"
            DL_ELVER="$DOWNLOAD/rpm-$VERSION/$ELVER"
            ARCH_LIST=$(fetch "$CHACRA_FLAVOR/" | subdirs || true)
            [[ -z "$ARCH_LIST" ]] && echo -e "  ${YELLOW}⚠ no arch dirs found for $DISTRO/$DVERSION on chacra${RESET}"
            for ARCH in $ARCH_LIST; do
                CFILE="$TMPDIR/c_${DISTRO}_${DVERSION}_${ARCH}"
                DFILE="$TMPDIR/d_${DISTRO}_${DVERSION}_${ARCH}"
                fetch "$CHACRA_FLAVOR/$ARCH/" | hrefs | grep '\.rpm$' | sort -u > "$CFILE" || true
                fetch "$DL_ELVER/$ARCH/" | hrefs | grep '\.rpm$' | sort -u > "$DFILE" || true
                compare_filelists "$DISTRO/$ELVER/$ARCH" "$CFILE" "$DFILE"
            done
            ;;
        *)
            ;; # windows, foo, etc -- not published to download.ceph.com this way
    esac
done < <(python3 -c "
import json, sys
d = json.load(sys.stdin)
for distro, versions in d.items():
    for v in versions:
        print(f'{distro}\t{v}')
" <<< "$DISTROS_JSON")

echo "----------------------------------------------------------------------"
if [[ "$MISMATCHES" -eq 0 ]]; then
    echo -e "${GREEN}All chacra packages found on download.ceph.com.${RESET}"
else
    echo -e "${RED}Mismatches found -- see above.${RESET}"
fi
exit "$MISMATCHES"