From f62acfc42fc83192501093556375f52a67fc63a1 Mon Sep 17 00:00:00 2001 From: Siddharth2207 Date: Mon, 24 Aug 2026 15:10:21 +0530 Subject: [PATCH 1/2] Enforce Goldsky subgraph version cap after deploy. After each network deploy, keep at most two always-on versions and fail when a two-version migration exceeds 24h. Also add a check-only task for scheduled audits (RAI-1962). Co-authored-by: Cursor --- README.md | 12 ++ flake.nix | 35 ++- lib/subgraph.sh | 201 ++++++++++++++++++ .../subgraph-goldsky-version-cap.test.bats | 83 ++++++++ 4 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 test/bats/task/subgraph-goldsky-version-cap.test.bats diff --git a/README.md b/README.md index 843d241..986f4d9 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,18 @@ All tasks are Nix packages run via `nix run`. From a consuming repo: - `nix run ..#rainix-rs-test` — cargo test - `nix run ..#rainix-rs-static` — cargo fmt + clippy +#### Subgraph + +Available on `subgraph-shell` / default shell (requires `GOLDSKY_TOKEN` and +`GOLDSKY_SUBGRAPH_NAME`, e.g. `raindex`): + +- `subgraph-deploy` — build + deploy each `networks.json` entry, then enforce a + hard cap of **2** always-on Goldsky versions per chain (deletes older + versions; fails if a 2-version migration is older than + `GOLDSKY_MIGRATION_HOURS`, default 24) +- `subgraph-goldsky-version-cap` — check-only version-cap / 24h migration audit + across `networks.json` (for cron or manual runs; does not delete) + ### Reusable Outputs Downstream flakes can compose their own tasks and shells using: diff --git a/flake.nix b/flake.nix index 0d809e6..46400ab 100644 --- a/flake.nix +++ b/flake.nix @@ -424,7 +424,8 @@ for network in $(subgraph_networks ./subgraph/networks.json); do address=$(subgraph_network_address ./subgraph/networks.json "$network") version=$(subgraph_deploy_version "$address" "$commit") - name_and_version="''${GOLDSKY_SUBGRAPH_NAME}-$network/$version" + subgraph_name="''${GOLDSKY_SUBGRAPH_NAME}-$network" + name_and_version="$subgraph_name/$version" if ${goldsky}/bin/goldsky --token ''${GOLDSKY_TOKEN} subgraph list "$name_and_version" 2>/dev/null | grep -q "$name_and_version"; then echo "Subgraph $name_and_version already deployed, skipping." @@ -434,7 +435,37 @@ echo "Deploying subgraph $name_and_version..." (cd ./subgraph && ${goldsky}/bin/goldsky --token ''${GOLDSKY_TOKEN} subgraph deploy "$name_and_version") fi + + # RAI-1962: keep at most 2 always-on versions per chain subgraph. + # Deletes older versions, then fails if the cap is still exceeded or + # a 2-version migration has been live for >24h. + GOLDSKY_BIN=${goldsky}/bin/goldsky \ + subgraph_goldsky_enforce_version_cap "$subgraph_name" --keep "$version" + done + ''; + additionalBuildInputs = node-build-inputs; + }; + + # Check-only Goldsky version budget across networks.json (for cron / manual). + # Does not deploy or delete — fails if any network has >2 versions or a + # 2-version migration older than GOLDSKY_MIGRATION_HOURS (default 24). + subgraph-goldsky-version-cap = mkTask { + name = "subgraph-goldsky-version-cap"; + body = '' + set -euo pipefail + source ${./lib/subgraph.sh} + + failed=0 + for network in $(subgraph_networks ./subgraph/networks.json); do + subgraph_name="''${GOLDSKY_SUBGRAPH_NAME}-$network" + echo "::group::''${subgraph_name}" + if ! GOLDSKY_BIN=${goldsky}/bin/goldsky \ + subgraph_goldsky_enforce_version_cap "$subgraph_name" --check-only; then + failed=1 + fi + echo "::endgroup::" done + exit "$failed" ''; additionalBuildInputs = node-build-inputs; }; @@ -443,6 +474,7 @@ subgraph-build subgraph-test subgraph-deploy + subgraph-goldsky-version-cap ]; source-dotenv = '' @@ -466,6 +498,7 @@ bats test/bats/task/skip-simulation.test.bats bats test/bats/task/subgraph-build.test.bats bats test/bats/task/subgraph-deploy-version.test.bats + bats test/bats/task/subgraph-goldsky-version-cap.test.bats bats test/bats/task/sol-single-contract.test.bats bats test/bats/task/no-custom-natspec.test.bats ''; diff --git a/lib/subgraph.sh b/lib/subgraph.sh index 0221c2e..565f407 100644 --- a/lib/subgraph.sh +++ b/lib/subgraph.sh @@ -22,3 +22,204 @@ subgraph_networks() { local networks_json="$1" jq -r 'keys[]' "$networks_json" } + +# Parse version names for a subgraph from goldsky list text on stdin. +# Usage: goldsky_list_text | subgraph_goldsky_parse_versions +subgraph_goldsky_parse_versions() { + local subgraph_name="$1" + sed 's/\x1b\[[0-9;]*m//g' | + grep -oE "${subgraph_name}/[^[:space:]│|]+" | + sed "s|^${subgraph_name}/||" | + sed 's/[^A-Za-z0-9._-]//g' | + awk 'NF' | + sort -u +} + +# Decide which versions to delete given keep/max. +# Input lines: version|epoch (epoch may be 0). +# Output: versions to delete, one per line (excess beyond max after keep preference). +# Usage: printf 'v|e\n...' | subgraph_goldsky_versions_to_delete [keep_version] +subgraph_goldsky_versions_to_delete() { + local max_versions="$1" + local keep_version="${2:-}" + awk -F'|' -v keep="$keep_version" ' + { + version=$1 + epoch=$2+0 + priority=(keep != "" && version == keep) ? 2 : 0 + printf "%d %020d %s\n", priority, epoch, version + } + ' | sort -k1,1nr -k2,2nr -k3,3r | + awk -v max="$max_versions" 'NR > max { print $3 }' +} + +# Return 0 if a 2-version migration overlap should fail. +# Args: newer_epoch older_epoch migration_hours [has_keep] +# has_keep=1 means a fresh deploy keep was provided (unknown timestamps OK). +subgraph_goldsky_migration_overlap_fail() { + local newer_epoch="$1" + local older_epoch="$2" + local migration_hours="$3" + local has_keep="${4:-0}" + local now_epoch limit_seconds + now_epoch="$(date -u +%s)" + limit_seconds=$((migration_hours * 3600)) + + if [[ "$newer_epoch" -gt 0 ]]; then + if [[ $((now_epoch - newer_epoch)) -gt $limit_seconds ]]; then + return 0 + fi + return 1 + fi + + if [[ "$older_epoch" -gt 0 ]]; then + if [[ $((now_epoch - older_epoch)) -gt $limit_seconds ]]; then + return 0 + fi + return 1 + fi + + # No timestamps: fail closed on scheduled checks; allow during fresh deploy keep. + if [[ "$has_keep" -eq 1 ]]; then + return 1 + fi + return 0 +} + +# Enforce always-on Goldsky version budget for one subgraph name (RAI-1962). +# +# Env: +# GOLDSKY_TOKEN required for goldsky CLI +# GOLDSKY_MAX_VERSIONS default 2 +# GOLDSKY_MIGRATION_HOURS default 24 +# +# Usage: +# subgraph_goldsky_enforce_version_cap [--keep ] [--check-only] +subgraph_goldsky_enforce_version_cap() { + local subgraph_name="${1:?subgraph name required}" + shift || true + + local keep_version="" + local check_only=0 + local max_versions="${GOLDSKY_MAX_VERSIONS:-2}" + local migration_hours="${GOLDSKY_MIGRATION_HOURS:-24}" + + while [[ $# -gt 0 ]]; do + case "$1" in + --keep) + keep_version="${2:?}" + shift 2 + ;; + --check-only) + check_only=1 + shift + ;; + --max) + max_versions="${2:?}" + shift 2 + ;; + --migration-hours) + migration_hours="${2:?}" + shift 2 + ;; + *) + echo "Unknown option for subgraph_goldsky_enforce_version_cap: $1" >&2 + return 2 + ;; + esac + done + + if [[ -z "${GOLDSKY_TOKEN:-}" ]]; then + echo "GOLDSKY_TOKEN is required for Goldsky version-cap enforcement." >&2 + return 1 + fi + + local goldsky_bin="${GOLDSKY_BIN:-goldsky}" + + _subgraph_goldsky_cmd() { + "$goldsky_bin" --token "$GOLDSKY_TOKEN" --color=false "$@" + } + + _subgraph_goldsky_list_version_rows() { + local raw versions version detail created epoch + raw="$(_subgraph_goldsky_cmd subgraph list "$subgraph_name" --filter deployments 2>&1 || true)" + printf '%s\n' "$raw" >&2 + versions="$(printf '%s\n' "$raw" | subgraph_goldsky_parse_versions "$subgraph_name")" + while IFS= read -r version; do + [[ -z "$version" ]] && continue + epoch=0 + detail="$(_subgraph_goldsky_cmd subgraph list "${subgraph_name}/${version}" --filter deployments 2>/dev/null || true)" + created="$( + printf '%s\n' "$detail" | + sed 's/\x1b\[[0-9;]*m//g' | + grep -oiE '(created([ _]at)?|created):[[:space:]]*[0-9T:Z.+-]+' | + head -n1 | + grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2}[^[:space:]]*' || true + )" + if [[ -n "$created" ]]; then + epoch="$(date -u -d "$created" +%s 2>/dev/null || echo 0)" + fi + printf '%s|%s\n' "$version" "$epoch" + done <<<"$versions" + } + + echo "==> Enforcing Goldsky version cap for ${subgraph_name} (max=${max_versions}, keep=${keep_version:-none}, check_only=${check_only})" + + local -a rows=() + local row + while IFS= read -r row; do + [[ -n "$row" ]] && rows+=("$row") + done < <(_subgraph_goldsky_list_version_rows) + + if [[ ${#rows[@]} -eq 0 ]]; then + echo "No deployments found for ${subgraph_name}." + return 0 + fi + + echo "Live versions (${#rows[@]}):" + for row in "${rows[@]}"; do + echo " - ${subgraph_name}/${row%%|*}" + done + + if [[ "$check_only" -eq 0 ]]; then + local to_delete + to_delete="$(printf '%s\n' "${rows[@]}" | subgraph_goldsky_versions_to_delete "$max_versions" "$keep_version")" + if [[ -n "$to_delete" ]]; then + while IFS= read -r version; do + [[ -z "$version" ]] && continue + echo "Deleting ${subgraph_name}/${version}" + _subgraph_goldsky_cmd subgraph delete "${subgraph_name}/${version}" --force + done <<<"$to_delete" + + rows=() + while IFS= read -r row; do + [[ -n "$row" ]] && rows+=("$row") + done < <(_subgraph_goldsky_list_version_rows) + + echo "After cleanup (${#rows[@]}):" + for row in "${rows[@]}"; do + echo " - ${subgraph_name}/${row%%|*}" + done + fi + fi + + if [[ ${#rows[@]} -gt $max_versions ]]; then + echo "::error title=Goldsky version cap exceeded::${subgraph_name} has ${#rows[@]} live versions; max allowed is ${max_versions}." + return 1 + fi + + if [[ ${#rows[@]} -eq 2 ]]; then + local sorted newer_epoch older_epoch has_keep=0 + sorted="$(printf '%s\n' "${rows[@]}" | awk -F'|' '{ printf "%020d %s|%s\n", $2+0, $1, $2 }' | sort -k1,1nr | awk '{ print $2 }')" + newer_epoch="$(printf '%s\n' "$sorted" | sed -n '1p' | cut -d'|' -f2)" + older_epoch="$(printf '%s\n' "$sorted" | sed -n '2p' | cut -d'|' -f2)" + [[ -n "$keep_version" ]] && has_keep=1 + if subgraph_goldsky_migration_overlap_fail "$newer_epoch" "$older_epoch" "$migration_hours" "$has_keep"; then + echo "::error title=Goldsky migration overlap >${migration_hours}h::${subgraph_name} still has 2 live versions past the ${migration_hours}h migration window." + return 1 + fi + echo "Two versions present for ${subgraph_name}; migration window OK." + fi + + echo "Version cap OK for ${subgraph_name}." +} diff --git a/test/bats/task/subgraph-goldsky-version-cap.test.bats b/test/bats/task/subgraph-goldsky-version-cap.test.bats new file mode 100644 index 0000000..a921703 --- /dev/null +++ b/test/bats/task/subgraph-goldsky-version-cap.test.bats @@ -0,0 +1,83 @@ +setup() { + # shellcheck disable=SC1091 + source lib/subgraph.sh +} + +@test "subgraph_goldsky_parse_versions extracts versions for a subgraph name" { + list_text="$( + cat <<'EOF' +┌──────────────┬──────────────────────────┐ +│ Name │ Version │ +├──────────────┼──────────────────────────┤ +│ raindex-base │ 0xabc-aaa1111 │ +│ raindex-base │ 0xabc-bbb2222 │ +│ raindex-eth │ 0xdef-ccc3333 │ +└──────────────┴──────────────────────────┘ +raindex-base/0xabc-aaa1111 +raindex-base/0xabc-bbb2222 +EOF + )" + run bash -c "source lib/subgraph.sh; printf '%s\n' \"$list_text\" | subgraph_goldsky_parse_versions raindex-base" + [ "$status" -eq 0 ] + [[ "$output" == *"0xabc-aaa1111"* ]] + [[ "$output" == *"0xabc-bbb2222"* ]] + [[ "$output" != *"0xdef-ccc3333"* ]] +} + +@test "subgraph_goldsky_versions_to_delete keeps preferred version and newest extras" { + fixture_rows="$( + cat <<'EOF' +old|100 +keepme|50 +mid|200 +newest|300 +EOF + )" + run bash -c "source lib/subgraph.sh; printf '%s\n' \"$fixture_rows\" | subgraph_goldsky_versions_to_delete 2 keepme" + [ "$status" -eq 0 ] + # keepme must be retained; only excess beyond max=2 are deleted. + [[ "$output" != *"keepme"* ]] + # Two deletes expected from the four inputs. + count="$(printf '%s\n' "$output" | awk 'NF' | wc -l | tr -d ' ')" + [ "$count" -eq 2 ] +} + +@test "subgraph_goldsky_versions_to_delete with max 2 and no keep deletes oldest two of four" { + fixture_rows="$( + cat <<'EOF' +a|10 +b|20 +c|30 +d|40 +EOF + )" + run bash -c "source lib/subgraph.sh; printf '%s\n' \"$fixture_rows\" | subgraph_goldsky_versions_to_delete 2" + [ "$status" -eq 0 ] + [[ "$output" == *"a"* ]] + [[ "$output" == *"b"* ]] + [[ "$output" != *"c"* ]] + [[ "$output" != *"d"* ]] +} + +@test "subgraph_goldsky_migration_overlap_fail when newer is older than window" { + now="$(date -u +%s)" + newer=$((now - 90000)) # >24h + older=$((now - 200000)) + run subgraph_goldsky_migration_overlap_fail "$newer" "$older" 24 0 + [ "$status" -eq 0 ] +} + +@test "subgraph_goldsky_migration_overlap_fail is false for fresh newer version" { + now="$(date -u +%s)" + newer=$((now - 60)) + older=$((now - 200000)) + run subgraph_goldsky_migration_overlap_fail "$newer" "$older" 24 0 + [ "$status" -eq 1 ] +} + +@test "subgraph_goldsky_migration_overlap_fail without timestamps fails closed unless keep" { + run subgraph_goldsky_migration_overlap_fail 0 0 24 0 + [ "$status" -eq 0 ] + run subgraph_goldsky_migration_overlap_fail 0 0 24 1 + [ "$status" -eq 1 ] +} From 2f1e575bda8d1c705fe9db7b1416e7f5d78676fd Mon Sep 17 00:00:00 2001 From: Siddharth2207 Date: Mon, 31 Aug 2026 13:22:56 +0530 Subject: [PATCH 2/2] Fix Goldsky version-cap parsing and reclaim per live CLI review. Parse Created timestamps from name-only listings (US locale + ISO), skip URL phantom versions, fail closed on list/parse errors, delete by real age, reclaim expired 2-version migrations, enforce on scheduled audit, and scan account summary for orphan subgraph names. Co-authored-by: Cursor --- README.md | 9 +- flake.nix | 21 +- lib/subgraph.sh | 293 ++++++++++++++++-- .../subgraph-goldsky-version-cap.test.bats | 108 ++++++- 4 files changed, 379 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 986f4d9..5eb9d95 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,11 @@ Available on `subgraph-shell` / default shell (requires `GOLDSKY_TOKEN` and - `subgraph-deploy` — build + deploy each `networks.json` entry, then enforce a hard cap of **2** always-on Goldsky versions per chain (deletes older - versions; fails if a 2-version migration is older than - `GOLDSKY_MIGRATION_HOURS`, default 24) -- `subgraph-goldsky-version-cap` — check-only version-cap / 24h migration audit - across `networks.json` (for cron or manual runs; does not delete) + versions; reclaims a 2-version migration older than `GOLDSKY_MIGRATION_HOURS`, + default 24) +- `subgraph-goldsky-version-cap` — same reclaim enforcement across + `networks.json` (for cron/manual), plus an account-level orphan audit against + the networks allowlist ### Reusable Outputs diff --git a/flake.nix b/flake.nix index 46400ab..337500c 100644 --- a/flake.nix +++ b/flake.nix @@ -446,9 +446,10 @@ additionalBuildInputs = node-build-inputs; }; - # Check-only Goldsky version budget across networks.json (for cron / manual). - # Does not deploy or delete — fails if any network has >2 versions or a - # 2-version migration older than GOLDSKY_MIGRATION_HOURS (default 24). + # Enforce Goldsky version budget across networks.json (for cron / manual). + # Deletes excess / expired migration versions (same reclaim path as deploy), + # then fails on remaining cap violations and on account-level orphan names + # outside the networks.json allowlist. subgraph-goldsky-version-cap = mkTask { name = "subgraph-goldsky-version-cap"; body = '' @@ -456,15 +457,27 @@ source ${./lib/subgraph.sh} failed=0 + allowlist="$(mktemp)" + trap 'rm -f "$allowlist"' EXIT + for network in $(subgraph_networks ./subgraph/networks.json); do subgraph_name="''${GOLDSKY_SUBGRAPH_NAME}-$network" + printf '%s\n' "$subgraph_name" >>"$allowlist" echo "::group::''${subgraph_name}" if ! GOLDSKY_BIN=${goldsky}/bin/goldsky \ - subgraph_goldsky_enforce_version_cap "$subgraph_name" --check-only; then + subgraph_goldsky_enforce_version_cap "$subgraph_name"; then failed=1 fi echo "::endgroup::" done + + echo "::group::orphan-audit" + if ! GOLDSKY_BIN=${goldsky}/bin/goldsky \ + subgraph_goldsky_audit_orphans "$allowlist"; then + failed=1 + fi + echo "::endgroup::" + exit "$failed" ''; additionalBuildInputs = node-build-inputs; diff --git a/lib/subgraph.sh b/lib/subgraph.sh index 565f407..fc1166b 100644 --- a/lib/subgraph.sh +++ b/lib/subgraph.sh @@ -23,26 +23,153 @@ subgraph_networks() { jq -r 'keys[]' "$networks_json" } +# Convert a Goldsky "Created:" timestamp to unix epoch seconds. +# Accepts US locale (7/16/2026, 9:51:30 PM) and ISO-ish strings. +# Prints 0 and returns 1 on failure. +# Usage: subgraph_goldsky_created_to_epoch +subgraph_goldsky_created_to_epoch() { + local raw="$1" + local s mon day year hour min sec ampm iso + + s="$( + printf '%s' "$raw" | + sed -E 's/^[Cc]reated([ _][Aa]t)?:[[:space:]]*//; s/,//g; s/^[[:space:]]+//; s/[[:space:]]+$//' + )" + [[ -z "$s" ]] && { + echo 0 + return 1 + } + + # US locale: M/D/YYYY H:MM:SS AM/PM (comma already stripped). + if [[ "$s" =~ ^([0-9]{1,2})/([0-9]{1,2})/([0-9]{4})[[:space:]]+([0-9]{1,2}):([0-9]{2}):([0-9]{2})[[:space:]]+([AaPp][Mm])$ ]]; then + mon=$(printf '%02d' "$((10#${BASH_REMATCH[1]}))") + day=$(printf '%02d' "$((10#${BASH_REMATCH[2]}))") + year="${BASH_REMATCH[3]}" + hour=$((10#${BASH_REMATCH[4]})) + min="${BASH_REMATCH[5]}" + sec="${BASH_REMATCH[6]}" + ampm="$(printf '%s' "${BASH_REMATCH[7]}" | tr 'apm' 'APM')" + if [[ "$ampm" == "PM" && "$hour" -ne 12 ]]; then + hour=$((hour + 12)) + elif [[ "$ampm" == "AM" && "$hour" -eq 12 ]]; then + hour=0 + fi + iso="$(printf '%s-%s-%s %02d:%s:%s' "$year" "$mon" "$day" "$hour" "$min" "$sec")" + if date -u -d "$iso" +%s 2>/dev/null; then + return 0 + fi + if date -u -j -f "%Y-%m-%d %H:%M:%S" "$iso" +%s 2>/dev/null; then + return 0 + fi + echo 0 + return 1 + fi + + # ISO / RFC3339-ish: 2026-07-16T21:51:30Z or with space. + if [[ "$s" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2} ]]; then + if date -u -d "$s" +%s 2>/dev/null; then + return 0 + fi + # BSD: strip trailing Z / fractional seconds. + s="${s%%.*}" + s="${s%Z}" + s="${s/T/ }" + if date -u -j -f "%Y-%m-%d %H:%M:%S" "$s" +%s 2>/dev/null; then + return 0 + fi + fi + + echo 0 + return 1 +} + # Parse version names for a subgraph from goldsky list text on stdin. +# Skips GraphQL URL lines so trailing /gn is not absorbed into phantom versions. # Usage: goldsky_list_text | subgraph_goldsky_parse_versions subgraph_goldsky_parse_versions() { local subgraph_name="$1" sed 's/\x1b\[[0-9;]*m//g' | - grep -oE "${subgraph_name}/[^[:space:]│|]+" | + grep -viE 'https?://' | + grep -oE "${subgraph_name}/[A-Za-z0-9._-]+" | sed "s|^${subgraph_name}/||" | - sed 's/[^A-Za-z0-9._-]//g' | + awk 'NF' | + sort -u +} + +# Parse version|epoch rows from a name-only goldsky subgraph list on stdin. +# Pairs each non-URL "name/version" sighting with the following Created: line. +# Usage: goldsky_list_text | subgraph_goldsky_parse_version_rows +subgraph_goldsky_parse_version_rows() { + local subgraph_name="$1" + local line cleaned version created epoch current="" + local -A seen=() + + while IFS= read -r line || [[ -n "$line" ]]; do + cleaned="$(printf '%s' "$line" | sed 's/\x1b\[[0-9;]*m//g')" + + if printf '%s' "$cleaned" | grep -qiE 'https?://'; then + continue + fi + + version="$( + printf '%s' "$cleaned" | + grep -oE "${subgraph_name}/[A-Za-z0-9._-]+" | + head -n1 | + sed "s|^${subgraph_name}/||" + )" + if [[ -n "$version" ]]; then + current="$version" + continue + fi + + if [[ -n "$current" ]] && printf '%s' "$cleaned" | grep -qiE 'Created([ _]at)?[[:space:]]*:'; then + created="$( + printf '%s' "$cleaned" | + sed -E 's/^.*[Cc]reated([ _][Aa]t)?[[:space:]]*:[[:space:]]*//' + )" + epoch="$(subgraph_goldsky_created_to_epoch "$created")" + # First Created for a version wins (listing order). + if [[ -z "${seen[$current]:-}" ]]; then + seen[$current]=1 + printf '%s|%s\n' "$current" "$epoch" + fi + current="" + fi + done +} + +# Parse distinct subgraph base names from `goldsky subgraph list --summary` stdin. +# Usage: goldsky_summary_text | subgraph_goldsky_parse_summary_names +subgraph_goldsky_parse_summary_names() { + sed 's/\x1b\[[0-9;]*m//g' | + grep -viE 'https?://' | + grep -oE '[A-Za-z0-9._-]+/[A-Za-z0-9._-]+' | + cut -d/ -f1 | awk 'NF' | sort -u } # Decide which versions to delete given keep/max. -# Input lines: version|epoch (epoch may be 0). +# Input lines: version|epoch (epoch must be >0 for age ordering). # Output: versions to delete, one per line (excess beyond max after keep preference). # Usage: printf 'v|e\n...' | subgraph_goldsky_versions_to_delete [keep_version] subgraph_goldsky_versions_to_delete() { local max_versions="$1" local keep_version="${2:-}" - awk -F'|' -v keep="$keep_version" ' + local input + input="$(cat)" + + if [[ -z "$input" ]]; then + return 0 + fi + + # Fail closed: never fall back to string-sort deletes when ages are missing. + if printf '%s\n' "$input" | awk -F'|' '$2+0 <= 0 { found=1 } END { exit found ? 0 : 1 }'; then + echo "Refusing to choose delete targets: one or more Created timestamps are missing/unparseable." >&2 + return 1 + fi + + printf '%s\n' "$input" | awk -F'|' -v keep="$keep_version" ' { version=$1 epoch=$2+0 @@ -53,7 +180,7 @@ subgraph_goldsky_versions_to_delete() { awk -v max="$max_versions" 'NR > max { print $3 }' } -# Return 0 if a 2-version migration overlap should fail. +# Return 0 if a 2-version migration overlap should fail / be reclaimed. # Args: newer_epoch older_epoch migration_hours [has_keep] # has_keep=1 means a fresh deploy keep was provided (unknown timestamps OK). subgraph_goldsky_migration_overlap_fail() { @@ -86,6 +213,22 @@ subgraph_goldsky_migration_overlap_fail() { return 0 } +# Older version from version|epoch rows (requires epochs > 0). +# Usage: printf 'v|e\n...' | subgraph_goldsky_older_version +subgraph_goldsky_older_version() { + local input + input="$(cat)" + if printf '%s\n' "$input" | awk -F'|' '$2+0 <= 0 { found=1 } END { exit found ? 0 : 1 }'; then + echo "Cannot pick older version without Created timestamps." >&2 + return 1 + fi + printf '%s\n' "$input" | + awk -F'|' '{ printf "%020d %s\n", $2+0, $1 }' | + sort -k1,1n | + head -n1 | + awk '{ print $2 }' +} + # Enforce always-on Goldsky version budget for one subgraph name (RAI-1962). # # Env: @@ -140,27 +283,31 @@ subgraph_goldsky_enforce_version_cap() { "$goldsky_bin" --token "$GOLDSKY_TOKEN" --color=false "$@" } + # Lists version|epoch from name-only listing. Fails closed on CLI errors. _subgraph_goldsky_list_version_rows() { - local raw versions version detail created epoch - raw="$(_subgraph_goldsky_cmd subgraph list "$subgraph_name" --filter deployments 2>&1 || true)" + local raw rc=0 + set +e + raw="$(_subgraph_goldsky_cmd subgraph list "$subgraph_name" --filter deployments 2>&1)" + rc=$? + set -e printf '%s\n' "$raw" >&2 - versions="$(printf '%s\n' "$raw" | subgraph_goldsky_parse_versions "$subgraph_name")" - while IFS= read -r version; do - [[ -z "$version" ]] && continue - epoch=0 - detail="$(_subgraph_goldsky_cmd subgraph list "${subgraph_name}/${version}" --filter deployments 2>/dev/null || true)" - created="$( - printf '%s\n' "$detail" | - sed 's/\x1b\[[0-9;]*m//g' | - grep -oiE '(created([ _]at)?|created):[[:space:]]*[0-9T:Z.+-]+' | - head -n1 | - grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2}[^[:space:]]*' || true - )" - if [[ -n "$created" ]]; then - epoch="$(date -u -d "$created" +%s 2>/dev/null || echo 0)" + + if [[ $rc -ne 0 ]]; then + echo "Goldsky subgraph list failed for ${subgraph_name} (exit ${rc})." >&2 + return 1 + fi + if printf '%s\n' "$raw" | grep -qiE 'listing failed|not found|unauthorized|forbidden|invalid token|rate limit'; then + # "not found" alone can mean zero deployments for a brand-new name; only + # treat as hard failure when the CLI also signals an error-shaped message + # that is not a clean empty listing. Prefer exit-code above; this catches + # soft-failure text with exit 0. + if printf '%s\n' "$raw" | grep -qiE 'listing failed|unauthorized|forbidden|invalid token|rate limit'; then + echo "Goldsky subgraph list returned an error for ${subgraph_name}." >&2 + return 1 fi - printf '%s|%s\n' "$version" "$epoch" - done <<<"$versions" + fi + + printf '%s\n' "$raw" | subgraph_goldsky_parse_version_rows "$subgraph_name" } echo "==> Enforcing Goldsky version cap for ${subgraph_name} (max=${max_versions}, keep=${keep_version:-none}, check_only=${check_only})" @@ -178,12 +325,21 @@ subgraph_goldsky_enforce_version_cap() { echo "Live versions (${#rows[@]}):" for row in "${rows[@]}"; do - echo " - ${subgraph_name}/${row%%|*}" + echo " - ${subgraph_name}/${row%%|*} (created_epoch=${row##*|})" done + if [[ ${#rows[@]} -gt 1 ]]; then + if printf '%s\n' "${rows[@]}" | awk -F'|' '$2+0 <= 0 { found=1 } END { exit found ? 0 : 1 }'; then + echo "::error title=Goldsky Created timestamps missing::Could not parse Created dates for ${subgraph_name}; refusing unsafe age-based deletes." >&2 + return 1 + fi + fi + if [[ "$check_only" -eq 0 ]]; then local to_delete - to_delete="$(printf '%s\n' "${rows[@]}" | subgraph_goldsky_versions_to_delete "$max_versions" "$keep_version")" + if ! to_delete="$(printf '%s\n' "${rows[@]}" | subgraph_goldsky_versions_to_delete "$max_versions" "$keep_version")"; then + return 1 + fi if [[ -n "$to_delete" ]]; then while IFS= read -r version; do [[ -z "$version" ]] && continue @@ -198,7 +354,7 @@ subgraph_goldsky_enforce_version_cap() { echo "After cleanup (${#rows[@]}):" for row in "${rows[@]}"; do - echo " - ${subgraph_name}/${row%%|*}" + echo " - ${subgraph_name}/${row%%|*} (created_epoch=${row##*|})" done fi fi @@ -209,17 +365,94 @@ subgraph_goldsky_enforce_version_cap() { fi if [[ ${#rows[@]} -eq 2 ]]; then - local sorted newer_epoch older_epoch has_keep=0 + local sorted newer_epoch older_epoch older_version has_keep=0 sorted="$(printf '%s\n' "${rows[@]}" | awk -F'|' '{ printf "%020d %s|%s\n", $2+0, $1, $2 }' | sort -k1,1nr | awk '{ print $2 }')" newer_epoch="$(printf '%s\n' "$sorted" | sed -n '1p' | cut -d'|' -f2)" older_epoch="$(printf '%s\n' "$sorted" | sed -n '2p' | cut -d'|' -f2)" [[ -n "$keep_version" ]] && has_keep=1 if subgraph_goldsky_migration_overlap_fail "$newer_epoch" "$older_epoch" "$migration_hours" "$has_keep"; then - echo "::error title=Goldsky migration overlap >${migration_hours}h::${subgraph_name} still has 2 live versions past the ${migration_hours}h migration window." - return 1 + if [[ "$check_only" -eq 1 ]]; then + echo "::error title=Goldsky migration overlap >${migration_hours}h::${subgraph_name} still has 2 live versions past the ${migration_hours}h migration window." + return 1 + fi + + # Reclaim: delete the older version so the 2nd slot stops billing. + if ! older_version="$(printf '%s\n' "${rows[@]}" | subgraph_goldsky_older_version)"; then + return 1 + fi + if [[ -n "$keep_version" && "$older_version" == "$keep_version" ]]; then + echo "::error title=Goldsky migration reclaim blocked::Older version is the keep target (${keep_version}); manual intervention required." >&2 + return 1 + fi + echo "Migration window exceeded; reclaiming older version ${subgraph_name}/${older_version}" + _subgraph_goldsky_cmd subgraph delete "${subgraph_name}/${older_version}" --force + + rows=() + while IFS= read -r row; do + [[ -n "$row" ]] && rows+=("$row") + done < <(_subgraph_goldsky_list_version_rows) + + if [[ ${#rows[@]} -gt 1 ]]; then + echo "::error title=Goldsky migration reclaim incomplete::${subgraph_name} still has ${#rows[@]} live versions after reclaim." + return 1 + fi + else + echo "Two versions present for ${subgraph_name}; migration window OK." fi - echo "Two versions present for ${subgraph_name}; migration window OK." fi echo "Version cap OK for ${subgraph_name}." } + +# Fail if account-level Goldsky subgraph names are outside the allowlist. +# Usage: subgraph_goldsky_audit_orphans +# allowlist_file: one subgraph base name per line (e.g. raindex-base). +subgraph_goldsky_audit_orphans() { + local allowlist_file="${1:?allowlist file required}" + + if [[ -z "${GOLDSKY_TOKEN:-}" ]]; then + echo "GOLDSKY_TOKEN is required for Goldsky orphan audit." >&2 + return 1 + fi + + local goldsky_bin="${GOLDSKY_BIN:-goldsky}" + local raw rc=0 + set +e + raw="$("$goldsky_bin" --token "$GOLDSKY_TOKEN" --color=false subgraph list --summary --filter deployments 2>&1)" + rc=$? + set -e + printf '%s\n' "$raw" >&2 + + if [[ $rc -ne 0 ]]; then + echo "Goldsky account summary list failed (exit ${rc})." >&2 + return 1 + fi + if printf '%s\n' "$raw" | grep -qiE 'listing failed|unauthorized|forbidden|invalid token|rate limit'; then + echo "Goldsky account summary list returned an error." >&2 + return 1 + fi + + local names orphans + names="$(printf '%s\n' "$raw" | subgraph_goldsky_parse_summary_names)" + if [[ -z "$names" ]]; then + echo "No subgraph names found in account summary." + return 0 + fi + + orphans="$( + printf '%s\n' "$names" | while IFS= read -r name; do + [[ -z "$name" ]] && continue + if ! grep -Fxq "$name" "$allowlist_file"; then + printf '%s\n' "$name" + fi + done + )" + + if [[ -n "$orphans" ]]; then + echo "::error title=Goldsky orphan subgraphs::Names outside networks.json allowlist (manual cleanup needed):" + printf '%s\n' "$orphans" | sed 's/^/ - /' + return 1 + fi + + echo "No orphan Goldsky subgraph names outside allowlist." +} diff --git a/test/bats/task/subgraph-goldsky-version-cap.test.bats b/test/bats/task/subgraph-goldsky-version-cap.test.bats index a921703..379bdc4 100644 --- a/test/bats/task/subgraph-goldsky-version-cap.test.bats +++ b/test/bats/task/subgraph-goldsky-version-cap.test.bats @@ -3,25 +3,64 @@ setup() { source lib/subgraph.sh } -@test "subgraph_goldsky_parse_versions extracts versions for a subgraph name" { +@test "subgraph_goldsky_parse_versions skips GraphQL URL /gn phantoms" { list_text="$( cat <<'EOF' -┌──────────────┬──────────────────────────┐ -│ Name │ Version │ -├──────────────┼──────────────────────────┤ -│ raindex-base │ 0xabc-aaa1111 │ -│ raindex-base │ 0xabc-bbb2222 │ -│ raindex-eth │ 0xdef-ccc3333 │ -└──────────────┴──────────────────────────┘ -raindex-base/0xabc-aaa1111 -raindex-base/0xabc-bbb2222 +* raindex-base/0xb05D73E6abc-105c526 + https://api.goldsky.com/api/public/project_cmexample/subgraphs/raindex-base/0xb05D73E6abc-105c526/gn +* raindex-base/0xe522cB4adef-8e9477b + https://api.goldsky.com/api/public/project_cmexample/subgraphs/raindex-base/0xe522cB4adef-8e9477b/gn +* raindex-eth/0xdef-ccc3333 EOF )" run bash -c "source lib/subgraph.sh; printf '%s\n' \"$list_text\" | subgraph_goldsky_parse_versions raindex-base" [ "$status" -eq 0 ] - [[ "$output" == *"0xabc-aaa1111"* ]] - [[ "$output" == *"0xabc-bbb2222"* ]] + [[ "$output" == *"0xb05D73E6abc-105c526"* ]] + [[ "$output" == *"0xe522cB4adef-8e9477b"* ]] + [[ "$output" != *"0xb05D73E6abc-105c526gn"* ]] + [[ "$output" != *"0xe522cB4adef-8e9477bgn"* ]] [[ "$output" != *"0xdef-ccc3333"* ]] + count="$(printf '%s\n' "$output" | awk 'NF' | wc -l | tr -d ' ')" + [ "$count" -eq 2 ] +} + +@test "subgraph_goldsky_parse_version_rows pairs Created US dates without URL phantoms" { + list_text="$( + cat <<'EOF' +* raindex-base/0xb05D73E6abc-105c526 + Status: LIVE + Created: 7/16/2026, 9:51:30 PM + https://api.goldsky.com/api/public/project_cmexample/subgraphs/raindex-base/0xb05D73E6abc-105c526/gn + +* raindex-base/0xe522cB4adef-8e9477b + Status: LIVE + Created: 8/20/2026, 1:02:03 AM + https://api.goldsky.com/api/public/project_cmexample/subgraphs/raindex-base/0xe522cB4adef-8e9477b/gn +EOF + )" + run bash -c "source lib/subgraph.sh; printf '%s\n' \"$list_text\" | subgraph_goldsky_parse_version_rows raindex-base" + [ "$status" -eq 0 ] + [[ "$output" == *"0xb05D73E6abc-105c526|"* ]] + [[ "$output" == *"0xe522cB4adef-8e9477b|"* ]] + [[ "$output" != *"gn|"* ]] + while IFS='|' read -r _ver epoch; do + [[ -z "$_ver" ]] && continue + [ "$epoch" -gt 0 ] + done <<<"$output" + count="$(printf '%s\n' "$output" | awk 'NF' | wc -l | tr -d ' ')" + [ "$count" -eq 2 ] +} + +@test "subgraph_goldsky_created_to_epoch parses US locale Created strings" { + run subgraph_goldsky_created_to_epoch "7/16/2026, 9:51:30 PM" + [ "$status" -eq 0 ] + [ "$output" -gt 0 ] +} + +@test "subgraph_goldsky_created_to_epoch parses ISO Created strings" { + run subgraph_goldsky_created_to_epoch "2026-07-16T21:51:30Z" + [ "$status" -eq 0 ] + [ "$output" -gt 0 ] } @test "subgraph_goldsky_versions_to_delete keeps preferred version and newest extras" { @@ -35,9 +74,7 @@ EOF )" run bash -c "source lib/subgraph.sh; printf '%s\n' \"$fixture_rows\" | subgraph_goldsky_versions_to_delete 2 keepme" [ "$status" -eq 0 ] - # keepme must be retained; only excess beyond max=2 are deleted. [[ "$output" != *"keepme"* ]] - # Two deletes expected from the four inputs. count="$(printf '%s\n' "$output" | awk 'NF' | wc -l | tr -d ' ')" [ "$count" -eq 2 ] } @@ -59,6 +96,49 @@ EOF [[ "$output" != *"d"* ]] } +@test "subgraph_goldsky_versions_to_delete fails closed when epochs are missing" { + fixture_rows="$( + cat <<'EOF' +a|0 +b|20 +c|30 +EOF + )" + run bash -c "source lib/subgraph.sh; printf '%s\n' \"$fixture_rows\" | subgraph_goldsky_versions_to_delete 2" + [ "$status" -ne 0 ] +} + +@test "subgraph_goldsky_older_version picks lowest epoch" { + fixture_rows="$( + cat <<'EOF' +newer|300 +older|100 +mid|200 +EOF + )" + run bash -c "source lib/subgraph.sh; printf '%s\n' \"$fixture_rows\" | subgraph_goldsky_older_version" + [ "$status" -eq 0 ] + [ "$output" = "older" ] +} + +@test "subgraph_goldsky_parse_summary_names skips URL paths" { + summary="$( + cat <<'EOF' +* raindex-base/0xaaa + https://api.goldsky.com/api/public/project_x/subgraphs/raindex-base/0xaaa/gn +* ob4-base/1.0.0 + https://api.goldsky.com/api/public/project_x/subgraphs/ob4-base/1.0.0/gn +* metadata-base/2 +EOF + )" + run bash -c "source lib/subgraph.sh; printf '%s\n' \"$summary\" | subgraph_goldsky_parse_summary_names" + [ "$status" -eq 0 ] + [[ "$output" == *"raindex-base"* ]] + [[ "$output" == *"ob4-base"* ]] + [[ "$output" == *"metadata-base"* ]] + [[ "$output" != *"api.goldsky.com"* ]] +} + @test "subgraph_goldsky_migration_overlap_fail when newer is older than window" { now="$(date -u +%s)" newer=$((now - 90000)) # >24h