fix(sync): retry network errors and report unreachable sources

.github/workflows/scheduled.yml

@@ -47,22 +47,48 @@ jobs:
 
       - name: Run Drift Detection
         id: drift_check
+        # Exit status (app/scripts/sync_sources.py): bit 1 drift, 2 check error,
+        # 4 unreachable source. Drift and unreachable sources are reported together.
         run: |
           cd app
           set +e
           uv run python scripts/sync_sources.py --check --json --alert-body ../issue_body.md > ../drift_report.json
           EXIT_CODE=$?
           set -e
-          echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT"
+          if ! jq -e 'type == "array"' ../drift_report.json > /dev/null 2>&1; then
+            echo "::error::Drift report is missing or not valid JSON (exit code $EXIT_CODE)"
+            exit 1
+          fi
           cat ../drift_report.json
-          if [ "$EXIT_CODE" -gt 1 ]; then
-            echo "::error::Drift detection encountered an operational error (exit code $EXIT_CODE)"
+          if [ "$EXIT_CODE" -gt 7 ]; then
+            echo "::error::Drift detection did not run to completion (exit code $EXIT_CODE)"
             exit "$EXIT_CODE"
           fi
-
+          {
+            echo "exit_code=$EXIT_CODE"
+            echo "drift=$(( EXIT_CODE & 1 ? 1 : 0 ))"
+            echo "check_error=$(( EXIT_CODE & 2 ? 1 : 0 ))"
+            echo "unreachable=$(( EXIT_CODE & 4 ? 1 : 0 ))"
+          } >> "$GITHUB_OUTPUT"
+          {
+            echo "## Document Drift Check"
+            echo ""
+            echo "- Drifted: $(jq '[.[] | select(.status == "DRIFT_DETECTED")] | length' ../drift_report.json)"
+            echo "- Check errors: $(jq '[.[] | select(.status == "ERROR")] | length' ../drift_report.json)"
+            echo "- Unreachable: $(jq '[.[] | select(.status == "UNREACHABLE")] | length' ../drift_report.json)"
+            if [ $(( EXIT_CODE & 6 )) -ne 0 ]; then
+              echo ""
+              echo "| Status | Document Path | Upstream URL | Error |"
+              echo "| :--- | :--- | :--- | :--- |"
+              jq -r '.[] | select(.status == "ERROR" or .status == "UNREACHABLE")
+                | "| \(.status) | `\(.path)` | \(.url) | \(.error // "" | gsub("\\|"; "\\|")) |"' ../drift_report.json
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+          jq -r '.[] | select(.status == "UNREACHABLE")
+            | "::error title=Unreachable source::\(.path) (\(.url)): \(.error)"' ../drift_report.json
 
       - name: File Drift Alert Issue
-        if: steps.drift_check.outputs.exit_code == '1'
+        if: steps.drift_check.outputs.drift == '1'
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -79,4 +105,16 @@ jobs:
               --label "documentation"
           fi
 
-
+      - name: Fail on Unreachable Sources or Check Errors
+        if: steps.drift_check.outputs.unreachable == '1' || steps.drift_check.outputs.check_error == '1'
+        env:
+          UNREACHABLE: ${{ steps.drift_check.outputs.unreachable }}
+          CHECK_ERROR: ${{ steps.drift_check.outputs.check_error }}
+        run: |
+          if [ "$UNREACHABLE" = "1" ]; then
+            echo "::error::One or more upstream sources could not be reached after retries. See the job summary."
+          fi
+          if [ "$CHECK_ERROR" = "1" ]; then
+            echo "::error::One or more sources failed to check. See the job summary."
+          fi
+          exit 1

app/scripts/sync_sources.py

@@ -18,6 +18,7 @@
 import os
 import re
 import sys
+import time
 import urllib.error
 import urllib.request
 from dataclasses import asdict, dataclass
@@ -52,6 +53,18 @@
 # changes the Markdown those functions emit. A new value forces a full fetch.
 EXTRACTOR_VERSION = "1"
 
+# A fetch that fails before any HTTP status arrives is retried with backoff.
+# An HTTP status response (including 4xx and 5xx) is not retried.
+# One delay per retry, so FETCH_ATTEMPTS is the delay count plus the first try.
+FETCH_BACKOFF_SECONDS = (5, 15)
+FETCH_ATTEMPTS = len(FETCH_BACKOFF_SECONDS) + 1
+
+# --check exit status is a bit set, so outcomes that happen together are all
+# visible in one code (drift plus an unreachable source is 1 | 4 == 5).
+EXIT_DRIFT = 1
+EXIT_CHECK_ERROR = 2
+EXIT_UNREACHABLE = 4
+
 
 @dataclass
 class SourceEntry:
@@ -1004,6 +1017,36 @@ def _header_map(headers: object) -> dict[str, str]:
     return {str(key).lower(): str(value) for key, value in items()}
 
 
+def _is_network_error(exc: BaseException) -> bool:
+    """True for a failure with no HTTP status: DNS, connect, reset, closed connection, timeout.
+
+    ``http.client.RemoteDisconnected`` ("Remote end closed connection without
+    response") is a ``ConnectionResetError``, so it is a network error.
+    """
+    if isinstance(exc, urllib.error.HTTPError):
+        return False
+    return isinstance(
+        exc,
+        (urllib.error.URLError, ConnectionError, TimeoutError),
+    )
+
+
+def _urlopen_with_retries(req: urllib.request.Request, timeout: int) -> Any:
+    """Open ``req``; a network error is retried until FETCH_ATTEMPTS attempts are spent."""
+    for attempt, delay in enumerate(FETCH_BACKOFF_SECONDS, start=1):
+        try:
+            return urllib.request.urlopen(req, timeout=timeout)
+        except Exception as exc:
+            if not _is_network_error(exc):
+                raise
+            logger.warning(
+                f"Network error fetching {req.full_url} (attempt {attempt}/{FETCH_ATTEMPTS}): "
+                f"{exc}. Retrying in {delay}s."
+            )
+            time.sleep(delay)
+    return urllib.request.urlopen(req, timeout=timeout)
+
+
 def fetch_upstream(
     url: str,
     timeout: int = 15,
@@ -1028,7 +1071,7 @@ def fetch_upstream(
         req_headers["If-Modified-Since"] = upstream_last_modified
     req = urllib.request.Request(url, headers=req_headers)
     try:
-        resp_ctx = urllib.request.urlopen(req, timeout=timeout)
+        resp_ctx = _urlopen_with_retries(req, timeout)
     except urllib.error.HTTPError as exc:
         if exc.code != 304 or not (etag or upstream_last_modified):
             raise
@@ -1302,16 +1345,16 @@ def check_source_drift(source: SourceEntry, repo_root: Path) -> DriftResult:
             upstream_hash=registry_hash,
             upstream_last_modified=source.upstream_last_modified,
         )
-    except (urllib.error.URLError, TimeoutError) as e:
-        logger.warning(f"Network error checking {source.url}: {e}")
-        return DriftResult(
-            path=source.path,
-            url=source.url,
-            status="ERROR",
-            local_hash=local_hash or registry_hash,
-            error=f"Network error: {e}",
-        )
     except Exception as e:
+        if _is_network_error(e):
+            logger.warning(f"Unreachable {source.url}: {e}")
+            return DriftResult(
+                path=source.path,
+                url=source.url,
+                status="UNREACHABLE",
+                local_hash=local_hash or registry_hash,
+                error=f"Network error: {e}",
+            )
         logger.error(f"Failed to check {source.url}: {e}", exc_info=True)
         return DriftResult(
             path=source.path,
@@ -1430,6 +1473,7 @@ def sync_source(
 def format_drift_alert_markdown(results: list[DriftResult]) -> str:
     """Formats markdown issue body for drifted documents."""
     drifted = [r for r in results if r.status == "DRIFT_DETECTED"]
+    unreachable = [r for r in results if r.status == "UNREACHABLE"]
     errored = [r for r in results if r.status == "ERROR"]
     lines = [
         "## Upstream Document Drift Detected",
@@ -1446,15 +1490,20 @@ def format_drift_alert_markdown(results: list[DriftResult]) -> str:
         last_mod = d.upstream_last_modified or "Unknown"
         lines.append(f"| `{d.path}` | [{doc_name}]({d.url}) | `{local_h}` | `{upstream_h}` | {last_mod} |")
 
-    if errored:
+    for heading, group in (
+        ("Sources That Could Not Be Reached", unreachable),
+        ("Sources That Failed To Check", errored),
+    ):
+        if not group:
+            continue
         lines.extend([
             "",
-            "### Sources That Could Not Be Reached",
+            f"### {heading}",
             "",
             "| Document Path | Upstream URL | Error |",
             "| :--- | :--- | :--- |",
         ])
-        for e in errored:
+        for e in group:
             doc_name = e.path.split("/")[-1]
             error_msg = (e.error or "Unknown error").replace("|", "\\|")
             lines.append(f"| `{e.path}` | [{doc_name}]({e.url}) | {error_msg} |")
@@ -1541,6 +1590,7 @@ def _run(args: argparse.Namespace) -> int:
         results: list[DriftResult] = []
         has_drift = False
         has_error = False
+        has_unreachable = False
 
         for entry in entries:
             logger.info(f"Checking {entry.path}...")
@@ -1550,6 +1600,8 @@ def _run(args: argparse.Namespace) -> int:
                 has_drift = True
             elif res.status == "ERROR":
                 has_error = True
+            elif res.status == "UNREACHABLE":
+                has_unreachable = True
 
         if args.alert_body and has_drift:
             args.alert_body.resolve().write_text(
@@ -1576,16 +1628,16 @@ def _run(args: argparse.Namespace) -> int:
                 if r.status == "DRIFT_DETECTED":
                     print(f"    Local Hash:    {r.local_hash}")
                     print(f"    Upstream Hash: {r.upstream_hash}")
-                elif r.status == "ERROR":
+                elif r.status in ("ERROR", "UNREACHABLE"):
                     print(f"    Error:         {r.error}")
                 elif r.status == "UNTRACKED":
                     print(f"    No local file and no registry hash — run --sync to initialise.")
 
-        if has_drift:
-            return 1
-        if has_error:
-            return 2
-        return 0
+        return (
+            (EXIT_DRIFT if has_drift else 0)
+            | (EXIT_CHECK_ERROR if has_error else 0)
+            | (EXIT_UNREACHABLE if has_unreachable else 0)
+        )
 
     if args.sync or args.dry_run:
         updated = 0