fix: harden static build reliability

.github/workflows/build.yml

@@ -19,6 +19,7 @@ permissions:
 jobs:
   update:
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -35,32 +36,35 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         run: |
           set -euo pipefail
-          run_id="$(gh run list \
+          mapfile -t run_ids < <(gh run list \
             --repo "$GITHUB_REPOSITORY" \
             --workflow build.yml \
             --branch master \
             --status success \
-            --limit 1 \
+            --limit 7 \
             --json databaseId \
-            --jq '.[0].databaseId // empty')"
-          if [ -z "$run_id" ]; then
+            --jq '.[].databaseId')
+          if [ "${#run_ids[@]}" -eq 0 ]; then
             echo "No previous successful build artifact found"
             exit 0
           fi
 
-          mkdir -p /tmp/previous-pages build
-          if ! gh run download "$run_id" \
-            --repo "$GITHUB_REPOSITORY" \
-            --name github-pages \
-            --dir /tmp/previous-pages; then
-            echo "Previous build artifact $run_id is unavailable"
-            exit 0
-          fi
-
-          if [ -f /tmp/previous-pages/artifact.tar ]; then
-            tar -xf /tmp/previous-pages/artifact.tar -C build
-            echo "Restored previous build artifact $run_id"
-          fi
+          mkdir -p build
+          for run_id in "${run_ids[@]}"; do
+            rm -rf /tmp/previous-pages
+            mkdir -p /tmp/previous-pages
+            if gh run download "$run_id" \
+              --repo "$GITHUB_REPOSITORY" \
+              --name github-pages \
+              --dir /tmp/previous-pages &&
+              test -f /tmp/previous-pages/artifact.tar; then
+              tar -xf /tmp/previous-pages/artifact.tar -C build
+              echo "Restored previous build artifact $run_id"
+              exit 0
+            fi
+            echo "::warning title=Previous build cache unavailable::Artifact from run $run_id could not be restored"
+          done
+          echo "::warning title=Incremental cache unavailable::No recent successful Pages artifact could be restored; running a full rebuild"
       - name: Run Program
         env:
           TZ: "Asia/Shanghai"
@@ -71,17 +75,32 @@ jobs:
           uv run --no-dev python main.py
       - name: Verify SQLite snapshots and schemas
         run: |
-          test -f build/life-ustc-static.sqlite
-          test -f build/life-ustc-static-guesses.sqlite
+          test -s build/build-status.json
           test -d build/schemas/upstream
           test "$(find build/schemas/upstream -name '*.schema.json' | wc -l)" -ge 5
+          uv run --no-dev python - <<'PY'
+          import json
+          from pathlib import Path
+
+          build = Path("build")
+          builders = json.loads((build / "build-status.json").read_text())["builders"]
+          if builders["curriculum"]["status"] == "ok":
+              assert (build / "life-ustc-static.sqlite").is_file()
+              assert (build / "life-ustc-static-guesses.sqlite").is_file()
+          if builders["young"]["status"] == "ok":
+              assert (build / "life-ustc-static.sqlite").is_file()
+          if builders["rss"]["status"] == "ok":
+              feeds = list((build / "rss").glob("*.xml"))
+              assert feeds and all(feed.stat().st_size > 0 for feed in feeds)
+          PY
       - name: Prepare Pages artifact
         run: touch build/.nojekyll
       - name: Upload static files as artifact
         id: deployment
         uses: actions/upload-pages-artifact@v4
         with:
           path: build/
+          retention-days: 7
 
   deploy:
     if: github.ref == 'refs/heads/master'

.github/workflows/ruff.yml

@@ -21,3 +21,7 @@ jobs:
         run: uv sync --group dev
       - name: Ruff check
         run: uv run ruff check . --output-format=github
+      - name: Ruff format
+        run: uv run ruff format --check .
+      - name: Run tests
+        run: uv run python -m unittest discover -s tests -p 'test_*.py'

main.py

@@ -1,13 +1,76 @@
 import argparse
 import asyncio
+import json
 import logging
 import shutil
+import tempfile
+from collections.abc import Awaitable, Callable
+from datetime import UTC, datetime
 from pathlib import Path
 
 from src import make_curriculum, make_rss, make_young_events
+from src.sqlite_store import GUESSES_FILENAME, SNAPSHOT_FILENAME
 from tools.upstream_schemas import export_upstream_schemas
 
 
+def _copy_output(source: Path, destination: Path) -> None:
+    if source.is_dir():
+        shutil.copytree(source, destination)
+    elif source.exists():
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(source, destination)
+
+
+def _restore_outputs(output_paths: tuple[Path, ...], backup_dir: Path) -> None:
+    for index, output_path in enumerate(output_paths):
+        if output_path.is_dir():
+            shutil.rmtree(output_path)
+        elif output_path.exists():
+            output_path.unlink()
+        _copy_output(backup_dir / str(index), output_path)
+
+
+async def _run_builders(
+    builders: list[tuple[str, Callable[[], Awaitable[None]], tuple[Path, ...]]],
+    *,
+    status_path: Path,
+) -> dict[str, dict[str, str]]:
+    results: dict[str, dict[str, str]] = {}
+    for name, builder, output_paths in builders:
+        with tempfile.TemporaryDirectory(prefix=f"static-{name}-") as temporary_dir:
+            backup_dir = Path(temporary_dir)
+            for index, output_path in enumerate(output_paths):
+                _copy_output(output_path, backup_dir / str(index))
+
+            try:
+                await builder()
+            except Exception as error:
+                _restore_outputs(output_paths, backup_dir)
+                logging.exception("%s builder failed; restored previous output", name)
+                print(
+                    f"::error title={name} builder failed::"
+                    f"{type(error).__name__}: {error}"
+                )
+                results[name] = {"status": "failed", "error": type(error).__name__}
+            else:
+                results[name] = {"status": "ok"}
+
+    status_path.parent.mkdir(parents=True, exist_ok=True)
+    status_path.write_text(
+        json.dumps(
+            {
+                "generated_at": datetime.now(UTC).isoformat(),
+                "builders": results,
+            },
+            ensure_ascii=False,
+            indent=2,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    return results
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(description="Life-USTC static builders")
     parser.add_argument(
@@ -43,15 +106,29 @@ def main() -> None:
     run_curriculum = args.curriculum or run_all
     run_young = args.young or run_all
 
-    async def _run():
-        if run_rss:
-            await make_rss()
-        if run_curriculum:
-            await make_curriculum()
-        if run_young:
-            await make_young_events()
+    builders: list[tuple[str, Callable[[], Awaitable[None]], tuple[Path, ...]]] = []
+    if run_rss:
+        builders.append(("rss", make_rss, (build_dir / "rss",)))
+    if run_curriculum:
+        builders.append(
+            (
+                "curriculum",
+                make_curriculum,
+                (
+                    build_dir / SNAPSHOT_FILENAME,
+                    build_dir / GUESSES_FILENAME,
+                ),
+            )
+        )
+    if run_young:
+        builders.append(("young", make_young_events, (build_dir / SNAPSHOT_FILENAME,)))
+
+    results = asyncio.run(
+        _run_builders(builders, status_path=build_dir / "build-status.json")
+    )
+    if results and all(result["status"] == "failed" for result in results.values()):
+        raise RuntimeError("All selected static builders failed")
 
-    asyncio.run(_run())
     if run_curriculum or run_young:
         schema_paths = export_upstream_schemas(build_dir / "schemas" / "upstream")
         logging.info("Exported %s upstream JSON schemas", len(schema_paths))

src/curriculum.py

@@ -2,6 +2,7 @@
 import time
 from json import JSONDecodeError
 
+import httpx
 from tqdm import tqdm
 
 from .guesses import SQLiteGuessStore
@@ -34,7 +35,9 @@
 
 CATALOG_SEMESTER_URL = "https://catalog.ustc.edu.cn/api/teach/semester/list"
 CATALOG_DEPARTMENT_URL = "https://catalog.ustc.edu.cn/api/teach/department/college-tree"
-CATALOG_LESSON_URL_PREFIX = "https://catalog.ustc.edu.cn/api/teach/lesson/list-for-teach"
+CATALOG_LESSON_URL_PREFIX = (
+    "https://catalog.ustc.edu.cn/api/teach/lesson/list-for-teach"
+)
 CATALOG_EXAM_URL_PREFIX = "https://catalog.ustc.edu.cn/api/teach/exam/list"
 JW_SCHEDULE_TABLE_URL = "https://jw.ustc.edu.cn/ws/schedule-table/datum"
 MIN_CATALOG_LESSON_SEMESTER_ID = 221
@@ -170,9 +173,7 @@ def _catalog_lesson_chunk_count(
         """,
         (fetch[0],),
     ).fetchone()[0]
-    return (
-        int(lesson_count) + JW_SCHEDULE_CHUNK_SIZE - 1
-    ) // JW_SCHEDULE_CHUNK_SIZE
+    return (int(lesson_count) + JW_SCHEDULE_CHUNK_SIZE - 1) // JW_SCHEDULE_CHUNK_SIZE
 
 
 def _expected_jw_schedule_chunk_count(
@@ -298,6 +299,9 @@ def _delete_cached_semester(
 
 
 def _is_skippable_exam_fetch_error(error: Exception) -> bool:
+    if isinstance(error, httpx.HTTPStatusError):
+        return error.response.status_code in {502, 504}
+
     message = str(error).lower()
     return any(
         marker in message
@@ -570,9 +574,7 @@ async def make_curriculum() -> None:
                     "selected_semester_count": len(selected_semesters),
                     "refreshed_semester_count": len(refreshed_semesters),
                     "cached_ended_semester_count": len(cached_ended_semester_ids),
-                    "cached_ended_semester_ids": ",".join(
-                        cached_ended_semester_ids
-                    ),
+                    "cached_ended_semester_ids": ",".join(cached_ended_semester_ids),
                     "catalog_lesson_min_semester_id": MIN_CATALOG_LESSON_SEMESTER_ID,
                     "catalog_lesson_skipped_legacy_semester_count": len(
                         skipped_catalog_lesson_semester_ids

src/rss.py

@@ -6,6 +6,7 @@
 import feedgenerator
 import feedparser
 import html2text
+import httpx
 import yaml
 from tqdm import tqdm
 
@@ -16,9 +17,26 @@
 
 
 async def get_and_clean_feed(url: str, path_to_save: Path):
-    feed = feedparser.parse(url)
+    try:
+        response = httpx.get(url, timeout=60, follow_redirects=True)
+        response.raise_for_status()
+    except httpx.HTTPError as error:
+        logger.warning(
+            "Keeping cached %s after feed fetch failed: %s", path_to_save, error
+        )
+        return
+
+    feed = feedparser.parse(response.content)
+
+    if feed.bozo:
+        logger.warning("Feed %s reported a parse warning: %s", url, feed.bozo_exception)
 
     if not feed.entries:
+        logger.warning(
+            "Keeping cached %s because the fetched feed has no entries%s",
+            path_to_save,
+            f": {feed.bozo_exception}" if feed.bozo else "",
+        )
         return
 
     feed_title = getattr(feed.feed, "title", "RSS Feed")
@@ -33,6 +51,7 @@ async def get_and_clean_feed(url: str, path_to_save: Path):
     handler.ignore_links = True
     handler.ignore_images = True
 
+    written_items = 0
     for entry in tqdm(
         feed.entries,
         position=0,
@@ -54,10 +73,18 @@ async def get_and_clean_feed(url: str, path_to_save: Path):
                 description=description,
                 pubdate=date,
             )
+            written_items += 1
         except Exception as e:
             logger.exception("Failed to process feed entry: %s", e)
 
-    with open(path_to_save, "w") as f:
+    if written_items == 0:
+        logger.warning(
+            "Keeping cached %s because no fetched entries could be parsed",
+            path_to_save,
+        )
+        return
+
+    with open(path_to_save, "w", encoding="utf-8") as f:
         new_feed.write(f, "utf-8")
 
 

src/sqlite_store.py

@@ -242,7 +242,7 @@ def store_response(
         response: BaseModel,
         fetch_id: int,
         context: Mapping[str, Scalar | None] | None = None,
-        ) -> int:
+    ) -> int:
         context = context or {}
 
         if isinstance(response, RootModel):
@@ -261,9 +261,7 @@ def store_response(
                         count += 1
                 return count
             if isinstance(root, BaseModel):
-                self._insert_model(
-                    table_name, root, fetch_id=fetch_id, context=context
-                )
+                self._insert_model(table_name, root, fetch_id=fetch_id, context=context)
                 return 1
             return 0
 

src/utils/auth.py

@@ -46,7 +46,7 @@ class LoginConfig:
 
     @classmethod
     def from_env(cls) -> "LoginConfig":
-        timeout_ms = int(os.getenv("USTC_TIMEOUT_MS", "0"))
+        timeout_ms = int(os.getenv("USTC_TIMEOUT_MS", "60000"))
         attempts = max(int(os.getenv("USTC_LOGIN_ATTEMPTS", "3")), 1)
         state_max_turns = max(int(os.getenv("USTC_LOGIN_STATE_MAX_TURNS", "10")), 1)
         turn_wait_ms = max(int(os.getenv("USTC_LOGIN_TURN_WAIT_MS", "5000")), 0)
@@ -172,14 +172,12 @@ def __init__(
         page: Page | None,
         timeout_ms: int = 10 * 60 * 1000,
         fail_on_status_code: bool = True,
-        max_retries: int = 100,
         transient_retries: int = 3,
     ):
         self.client = client
         self.page = page
         self.timeout_ms = timeout_ms
         self.fail_on_status_code = fail_on_status_code
-        self.max_retries = max_retries
         self.transient_retries = transient_retries
         self.logger = logging.getLogger(__name__)
 

src/utils/jw.py

@@ -13,9 +13,7 @@
 from .tools import compose_start_end, join_nonempty
 
 _jw_user_id_cache: dict[int, str] = {}
-JW_SSO_URL = (
-    "https://jw.ustc.edu.cn/ucas-sso/login"
-)
+JW_SSO_URL = "https://jw.ustc.edu.cn/ucas-sso/login"
 
 indexStartTimes: dict[int, int] = {
     1: 7 * 60 + 50,
@@ -301,7 +299,7 @@ async def fetch_jw_schedule_table_json(
         url=url,
         data={"lessonIds": course_id_list},
         timeout=30_000,
-        max_retries=2,
+        transient_retries=2,
     )