From 5a4a8950c8ea8de409ffecd6e24f400c404d4811 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 23:04:01 +0000 Subject: [PATCH] caller-pins: read callers from raw, and never trust the session sentinel The first live census (run 30860781598) failed two ways, both of which the guards caught rather than hid. RATE LIMIT. Unauthenticated api.github.com allows 60 requests/hour and the census makes one per repo (~90). It took 23 consecutive 403s and still printed a confident 'callers: 43' -- truncated to whichever repos sorted early. Reading callers from raw.githubusercontent instead: a CDN, unmetered against that budget, no credential needed. The org listing stays on the API because it has no raw equivalent, but that is one call, not ninety. Authenticating was not the fix: github.token is scoped to THIS repo and cannot read another repo's contents at all. SENTINEL. A cloud session's GH_TOKEN is the literal string 'proxy-injected' -- set, non-empty, and not a credential. Forwarding it as a Bearer made raw 404 every file, i.e. 'no repo has a caller'. Only the empty-census guard caught it. real_token() now treats the sentinel as absent. Private repos are skipped and COUNTED (unexamined_private) rather than attempted: without a credential they 404 on raw, which is indistinguishable from having no caller. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017RMKHpGXzw9Zkv49fzVssg --- test/test_caller_pins.py | 35 ++++++++++++++++++++++ tools/caller-pins.py | 65 ++++++++++++++++++++++++++++++++-------- 2 files changed, 87 insertions(+), 13 deletions(-) diff --git a/test/test_caller_pins.py b/test/test_caller_pins.py index 0c0ebb9..f7a6016 100644 --- a/test/test_caller_pins.py +++ b/test/test_caller_pins.py @@ -86,6 +86,41 @@ def test_non_ancestor_is_not_behind(self): self.assertEqual(caller_pins.classify(STALE, TEMPLATE, False), "ahead-or-diverged") +class TestCallerUrl(unittest.TestCase): + def test_reads_from_raw_not_the_api(self): + # Load-bearing: the API's contents endpoint is capped at 60 req/hour + # unauthenticated, and the census makes one call per repo (~90). Run + # 30860781598 hit that ceiling and took 23 straight 403s. raw is a CDN, + # unmetered against that budget, and needs no credential. + url = caller_pins.caller_url("bounded-systems", "brand") + self.assertTrue(url.startswith("https://raw.githubusercontent.com/")) + self.assertNotIn("api.github.com", url) + + def test_points_at_the_default_branch_and_the_caller_path(self): + url = caller_pins.caller_url("bounded-systems", "brand") + self.assertEqual( + url, "https://raw.githubusercontent.com/bounded-systems/brand/HEAD/.github/workflows/deps.yml" + ) + + +class TestRealToken(unittest.TestCase): + def test_sentinel_is_not_a_credential(self): + # A cloud session's GH_TOKEN is the literal string `proxy-injected`. + # Forwarding it as a Bearer makes raw 404 every file, which reads exactly + # like "no repo has a caller" — a silently empty census. + self.assertIsNone(caller_pins.real_token({"GH_TOKEN": "proxy-injected"})) + + def test_empty_is_none(self): + self.assertIsNone(caller_pins.real_token({"GH_TOKEN": ""})) + self.assertIsNone(caller_pins.real_token({})) + + def test_a_real_token_passes_through(self): + self.assertEqual(caller_pins.real_token({"GH_TOKEN": "ghs_abc123"}), "ghs_abc123") + + def test_github_token_is_a_fallback(self): + self.assertEqual(caller_pins.real_token({"GITHUB_TOKEN": "ghs_xyz"}), "ghs_xyz") + + class TestTemplateIsItsOwnFixture(unittest.TestCase): def test_the_real_template_carries_a_pin(self): # Guards the comparison basis itself: if templates/deps.yml ever stops diff --git a/tools/caller-pins.py b/tools/caller-pins.py index 5a06103..a5be15f 100644 --- a/tools/caller-pins.py +++ b/tools/caller-pins.py @@ -60,6 +60,20 @@ import urllib.request API = "https://api.github.com" +# Caller files are read from raw.githubusercontent, NOT the API's contents +# endpoint, and that is a correctness fix rather than a preference. +# +# Unauthenticated api.github.com allows 60 requests/hour. The census makes one +# call per repo — ~90 for this org — so the first run (30860781598) sailed past +# the limit and took 23 consecutive HTTP 403s, silently truncating the census to +# whichever repos happened to be alphabetically early. Authenticating would fix +# the limit and break the census a different way: `github.token` is scoped to +# THIS repo, so it cannot read another repo's contents at all. +# +# raw.githubusercontent serves public files from a CDN, is not metered against +# that 60/hour budget, and needs no credential. The org listing stays on the API +# because it has no raw equivalent — but that is one or two calls, not ninety. +RAW = "https://raw.githubusercontent.com" CALLER_PATH = ".github/workflows/deps.yml" # `uses: bounded-systems/ci-workflows/.github/workflows/osv-scan.yml@<40hex>`. @@ -147,18 +161,37 @@ def list_org_repos(org, token=None): return sorted(repos, key=lambda r: r["name"]) +# A cloud session's GH_TOKEN is the literal string `proxy-injected` — a sentinel, +# not a credential. The real one is injected at the egress proxy for GitHub hosts, +# so the variable is set and non-empty and cannot be presented to anything. +# Forwarding it as `Authorization: Bearer proxy-injected` makes raw 404 EVERY +# file, which reads exactly like "no repo has a caller". Caught here only because +# the empty-census guard refuses to report success on zero rows. +SENTINEL_TOKEN = "proxy-injected" + + +def real_token(env=None): + """The ambient token, or None if it is absent or the session sentinel.""" + env = os.environ if env is None else env + tok = env.get("GH_TOKEN") or env.get("GITHUB_TOKEN") or "" + return None if (not tok or tok == SENTINEL_TOKEN) else tok + + +def caller_url(org, repo): + """Where a caller file is read from. `HEAD` resolves to the default branch.""" + return f"{RAW}/{org}/{repo}/HEAD/{CALLER_PATH}" + + def fetch_caller(org, repo, token=None): """The caller file's text, or None if the repo does not have one. - A 404 means "not a caller" and is normal. Anything else is an error worth - surfacing — a rate-limited 403 read as "no caller" would silently shrink the - census, which is the failure mode this whole file exists to prevent. + A 404 means "not a caller" and is normal. Anything else RAISES — a + rate-limited 403 quietly read as "no caller" would shrink the census while + still reporting success, which is the exact hollow-green shape this file + exists to close. Run 30860781598 is the evidence: 23 repos 403'd and the + summary line still printed a confident-looking `callers: 43`. """ - status, body, _ = _get( - f"{API}/repos/{org}/{repo}/contents/{CALLER_PATH}", - token, - accept="application/vnd.github.raw", - ) + status, body, _ = _get(caller_url(org, repo), token, accept="*/*") if status == 404: return None if status != 200: @@ -189,7 +222,7 @@ def main(argv=None): import pathlib root = pathlib.Path(args.repo_root) - token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") or None + token = real_token() tpl = template_pin(root) if args.repos: @@ -199,6 +232,12 @@ def main(argv=None): rows, errors, unreadable_private = [], [], 0 for r in listing: + # A private repo is not on raw without a credential. Skipping it and + # COUNTING it is the honest move: attempting it would 404 and be + # indistinguishable from "has no caller", quietly shrinking the census. + if r["private"] and not token: + unreadable_private += 1 + continue try: text = fetch_caller(args.org, r["name"], token) except RuntimeError as e: @@ -217,8 +256,7 @@ def main(argv=None): "state": classify(sha, tpl, is_ancestor(sha, tpl, cwd=root)), **({"pins_found": n} if n != 1 else {}), }) - if not token: - unreadable_private = sum(1 for r in listing if r["private"]) + summary_note = f"{unreadable_private} private repo(s) were not examined" if unreadable_private else None behind = [r for r in rows if r["state"] == "behind"] summary = { @@ -229,6 +267,7 @@ def main(argv=None): "unknown": sum(1 for r in rows if r["state"] == "unknown"), "ahead_or_diverged": sum(1 for r in rows if r["state"] == "ahead-or-diverged"), "authenticated": bool(token), + "unexamined_private": unreadable_private, "errors": len(errors), } @@ -241,8 +280,8 @@ def main(argv=None): print(f"{mark} {r['repo']:<{width}} {r['pin'][:7]} {r['state']}") for e in errors: print(f"::warning::{e}") - if not token: - print(f"::notice::unauthenticated — {unreadable_private} private repo(s) were not examined") + if summary_note: + print(f"::notice::unauthenticated — {summary_note}") print() # One machine-readable line, the same idiom as FDS-CLAIM-RESULT / FDS-PARITY-RESULT: # greppable out of a job log without parsing the table.