packaging: harden packages index generator and tests

Fix macOS version parsing in build-catalog.awk, validate LATEST_VERSION
overrides, and strip tree HTML footers without relying on a fixed line
count. Extend regression tests for automatic latest selection, S3 parity,
and invalid inputs. Disable persisted checkout credentials in the
release index job.

Signed-off-by: lecaros <lecaros@chronosphere.io>

Author: lecaros

.github/workflows/staging-release.yaml

@@ -373,6 +373,8 @@ jobs:
     steps:
     - name: Checkout code
       uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      with:
+        persist-credentials: false
 
     - name: Setup runner
       run: |

packaging/build-catalog.awk

@@ -15,7 +15,7 @@ function schema_version(name) {
     return name
 }
 
-function extract_version(name,    rest, i, c) {
+function extract_version(name,    rest, i, c, following) {
     if (substr(name, 1, 11) == "fluent-bit-") {
         rest = substr(name, 12)
     } else if (substr(name, 1, 11) == "fluent-bit_") {
@@ -26,7 +26,19 @@ function extract_version(name,    rest, i, c) {
 
     for (i = 1; i <= length(rest); i++) {
         c = substr(rest, i, 1)
-        if (c != "." && (c < "0" || c > "9")) {
+        if (c == ".") {
+            if (i < length(rest)) {
+                following = substr(rest, i + 1, 1)
+                if (following !~ /[0-9]/) {
+                    if (i == 1) {
+                        return ""
+                    }
+                    return substr(rest, 1, i - 1)
+                }
+            }
+            continue
+        }
+        if (c < "0" || c > "9") {
             if (i == 1) {
                 return ""
             }
@@ -104,7 +116,7 @@ function json_url(path) {
     return json_str(base_url "/" path)
 }
 
-function note_linux(v, path, name, repo,    key, label) {
+function note_linux(v, path, name, repo,    key, label, idx) {
     if (!version_includes_linux(v)) {
         return
     }

packaging/generate-packages-index.sh

@@ -209,7 +209,7 @@ if [[ ! -s "$TREE_LIST" ]]; then
 fi
 
 tree --noreport --charset utf-8 --fromfile "$TREE_LIST" -H "$BASE_URL" | \
-    sed -e '/<hr>/,+7d' > "$WORK_DIR/tree.html"
+    awk '/<hr>/ { exit } { print } END { print "</body></html>" }' > "$WORK_DIR/tree.html"
 
 REPO_PATHS="$(printf '%s ' "${LINUX_REPO_PATHS[@]}")"
 awk -v emit_mode=versions \
@@ -231,6 +231,17 @@ while IFS= read -r version; do
 done < <(awk -F '\t' '{ print $1 }' "$VERSION_ROWS")
 
 if [[ -n "$LATEST_VERSION" ]]; then
+    found=0
+    for version in "${SORTED_VERSIONS[@]}"; do
+        if [[ "$version" == "$LATEST_VERSION" ]]; then
+            found=1
+            break
+        fi
+    done
+    if [[ "$found" -eq 0 ]]; then
+        echo "ERROR: LATEST_VERSION '$LATEST_VERSION' was not found among discovered versions: ${SORTED_VERSIONS[*]}" >&2
+        exit 1
+    fi
     LATEST="$LATEST_VERSION"
 else
     LATEST="${SORTED_VERSIONS[${#SORTED_VERSIONS[@]}-1]}"

packaging/testing/test-packages-index.sh

@@ -9,7 +9,8 @@ S3_LISTING_FILE=""
 
 cleanup()
 {
-    rm -rf "$OUTPUT_DIR" "${OUTPUT_DIR_S3:-}" "$S3_LISTING_FILE"
+    rm -rf "$OUTPUT_DIR" "${OUTPUT_DIR_S3:-}" "${OUTPUT_DIR_OVERRIDE:-}" \
+        "${EMPTY_BASE_PATH:-}" "$S3_LISTING_FILE"
 }
 
 trap cleanup EXIT
@@ -87,11 +88,41 @@ assert_equals()
     fi
 }
 
+assert_fails_with()
+{
+    local expected_msg="$1"
+    shift
+    local output=""
+
+    if output=$("$@" 2>&1); then
+        echo "ERROR: expected command to fail: $*" >&2
+        exit 1
+    fi
+
+    if [[ -n "$expected_msg" ]] && ! grep -Fq "$expected_msg" <<< "$output"; then
+        echo "ERROR: expected error message '$expected_msg', got: $output" >&2
+        exit 1
+    fi
+}
+
+assert_versions_json_match()
+{
+    local left="$1"
+    local right="$2"
+
+    if ! diff <(jq -S 'del(.generated_at)' "$left") \
+              <(jq -S 'del(.generated_at)' "$right") >/dev/null; then
+        echo "ERROR: versions.json mismatch between $left and $right" >&2
+        diff -u <(jq -S 'del(.generated_at)' "$left") \
+                <(jq -S 'del(.generated_at)' "$right") >&2 || true
+        exit 1
+    fi
+}
+
 setup_local_fixture "$OUTPUT_DIR"
 
 BASE_PATH="$OUTPUT_DIR" \
 AWS_S3_REMOTE_DISCOVERY=false \
-LATEST_VERSION=4.2.7 \
 BASE_URL=https://packages.example.test \
 "$GENERATOR"
 
@@ -125,13 +156,18 @@ if ! jq -e '.versions[] | select(.version == "4.2.7") | .artifacts.windows.win64
     exit 1
 fi
 
+if ! jq -e '.versions[] | select(.version == "4.2.7") | .artifacts.macos.pkg' \
+    "$OUTPUT_DIR/versions.json" >/dev/null; then
+    echo "ERROR: versions.json missing macOS pkg artifact for 4.2.7" >&2
+    exit 1
+fi
+
 echo "packages-index generator test passed"
 
 OUTPUT_DIR_S3="$(mktemp -d)"
 setup_s3_listing_fixture
 
 BASE_PATH="$OUTPUT_DIR_S3" \
-LATEST_VERSION=4.2.7 \
 BASE_URL=https://packages.example.test \
 AWS_S3_BUCKET=packages.example.test \
 AWS_S3_REMOTE_DISCOVERY=true \
@@ -140,11 +176,45 @@ AWS_S3_LISTING_FILE="$S3_LISTING_FILE" \
 
 assert_equals "$OUTPUT_DIR_S3/latest-version.txt" "4.2.7"
 assert_contains "$OUTPUT_DIR_S3/index.html" "fluent-bit-4.2.7-win64.exe"
+assert_versions_json_match "$OUTPUT_DIR/versions.json" "$OUTPUT_DIR_S3/versions.json"
 
-if ! jq -e '.latest == "4.2.7" and (.versions | length) == 2' "$OUTPUT_DIR_S3/versions.json" >/dev/null; then
-    echo "ERROR: S3 listing parse did not produce expected versions" >&2
+if ! jq -e '.latest == "4.2.7" and .base_url == "https://packages.example.test" and (.versions | length) == 2' \
+    "$OUTPUT_DIR_S3/versions.json" >/dev/null; then
+    echo "ERROR: S3 listing parse did not produce expected catalog metadata" >&2
     jq . "$OUTPUT_DIR_S3/versions.json" >&2 || true
     exit 1
 fi
 
 echo "packages-index S3 listing parse test passed"
+
+OUTPUT_DIR_OVERRIDE="$(mktemp -d)"
+setup_local_fixture "$OUTPUT_DIR_OVERRIDE"
+
+BASE_PATH="$OUTPUT_DIR_OVERRIDE" \
+AWS_S3_REMOTE_DISCOVERY=false \
+LATEST_VERSION=4.2.6 \
+BASE_URL=https://packages.example.test \
+"$GENERATOR"
+
+assert_equals "$OUTPUT_DIR_OVERRIDE/latest-version.txt" "4.2.6"
+if ! jq -e '.latest == "4.2.6"' "$OUTPUT_DIR_OVERRIDE/versions.json" >/dev/null; then
+    echo "ERROR: LATEST_VERSION override did not update latest field" >&2
+    exit 1
+fi
+
+assert_fails_with "LATEST_VERSION '9.9.9' was not found among discovered versions" \
+    env BASE_PATH="$OUTPUT_DIR" AWS_S3_REMOTE_DISCOVERY=false LATEST_VERSION=9.9.9 \
+    BASE_URL=https://packages.example.test "$GENERATOR"
+
+EMPTY_BASE_PATH="$(mktemp -d)"
+assert_fails_with "ERROR: no objects found under" \
+    env BASE_PATH="$EMPTY_BASE_PATH" AWS_S3_REMOTE_DISCOVERY=false \
+    BASE_URL=https://packages.example.test "$GENERATOR"
+
+assert_fails_with "AWS_S3_LISTING_FILE does not exist" \
+    env BASE_PATH="$OUTPUT_DIR_S3" AWS_S3_REMOTE_DISCOVERY=true \
+    AWS_S3_BUCKET=packages.example.test \
+    AWS_S3_LISTING_FILE=/tmp/does-not-exist-packages-index-test \
+    BASE_URL=https://packages.example.test "$GENERATOR"
+
+echo "packages-index validation error handling test passed"