diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1eb84a8eb9..ad07519fd3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,6 +19,7 @@ permissions: jobs: update: runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: Checkout uses: actions/checkout@v6 @@ -35,32 +36,35 @@ jobs: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - run_id="$(gh run list \ + mapfile -t run_ids < <(gh run list \ --repo "$GITHUB_REPOSITORY" \ --workflow build.yml \ --branch master \ --status success \ - --limit 1 \ + --limit 7 \ --json databaseId \ - --jq '.[0].databaseId // empty')" - if [ -z "$run_id" ]; then + --jq '.[].databaseId') + if [ "${#run_ids[@]}" -eq 0 ]; then echo "No previous successful build artifact found" exit 0 fi - mkdir -p /tmp/previous-pages build - if ! gh run download "$run_id" \ - --repo "$GITHUB_REPOSITORY" \ - --name github-pages \ - --dir /tmp/previous-pages; then - echo "Previous build artifact $run_id is unavailable" - exit 0 - fi - - if [ -f /tmp/previous-pages/artifact.tar ]; then - tar -xf /tmp/previous-pages/artifact.tar -C build - echo "Restored previous build artifact $run_id" - fi + mkdir -p build + for run_id in "${run_ids[@]}"; do + rm -rf /tmp/previous-pages + mkdir -p /tmp/previous-pages + if gh run download "$run_id" \ + --repo "$GITHUB_REPOSITORY" \ + --name github-pages \ + --dir /tmp/previous-pages && + test -f /tmp/previous-pages/artifact.tar; then + tar -xf /tmp/previous-pages/artifact.tar -C build + echo "Restored previous build artifact $run_id" + exit 0 + fi + echo "::warning title=Previous build cache unavailable::Artifact from run $run_id could not be restored" + done + echo "::warning title=Incremental cache unavailable::No recent successful Pages artifact could be restored; running a full rebuild" - name: Run Program env: TZ: "Asia/Shanghai" @@ -71,10 +75,24 @@ jobs: uv run --no-dev python main.py - name: Verify SQLite snapshots and schemas run: | - test -f build/life-ustc-static.sqlite - test -f build/life-ustc-static-guesses.sqlite + test -s build/build-status.json test -d build/schemas/upstream test "$(find build/schemas/upstream -name '*.schema.json' | wc -l)" -ge 5 + uv run --no-dev python - <<'PY' + import json + from pathlib import Path + + build = Path("build") + builders = json.loads((build / "build-status.json").read_text())["builders"] + if builders["curriculum"]["status"] == "ok": + assert (build / "life-ustc-static.sqlite").is_file() + assert (build / "life-ustc-static-guesses.sqlite").is_file() + if builders["young"]["status"] == "ok": + assert (build / "life-ustc-static.sqlite").is_file() + if builders["rss"]["status"] == "ok": + feeds = list((build / "rss").glob("*.xml")) + assert feeds and all(feed.stat().st_size > 0 for feed in feeds) + PY - name: Prepare Pages artifact run: touch build/.nojekyll - name: Upload static files as artifact @@ -82,6 +100,7 @@ jobs: uses: actions/upload-pages-artifact@v4 with: path: build/ + retention-days: 7 deploy: if: github.ref == 'refs/heads/master' diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index a65f0eada6..6c104059be 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -21,3 +21,7 @@ jobs: run: uv sync --group dev - name: Ruff check run: uv run ruff check . --output-format=github + - name: Ruff format + run: uv run ruff format --check . + - name: Run tests + run: uv run python -m unittest discover -s tests -p 'test_*.py' diff --git a/main.py b/main.py index e9f42f0de1..5c184f97d1 100644 --- a/main.py +++ b/main.py @@ -1,13 +1,76 @@ import argparse import asyncio +import json import logging import shutil +import tempfile +from collections.abc import Awaitable, Callable +from datetime import UTC, datetime from pathlib import Path from src import make_curriculum, make_rss, make_young_events +from src.sqlite_store import GUESSES_FILENAME, SNAPSHOT_FILENAME from tools.upstream_schemas import export_upstream_schemas +def _copy_output(source: Path, destination: Path) -> None: + if source.is_dir(): + shutil.copytree(source, destination) + elif source.exists(): + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + + +def _restore_outputs(output_paths: tuple[Path, ...], backup_dir: Path) -> None: + for index, output_path in enumerate(output_paths): + if output_path.is_dir(): + shutil.rmtree(output_path) + elif output_path.exists(): + output_path.unlink() + _copy_output(backup_dir / str(index), output_path) + + +async def _run_builders( + builders: list[tuple[str, Callable[[], Awaitable[None]], tuple[Path, ...]]], + *, + status_path: Path, +) -> dict[str, dict[str, str]]: + results: dict[str, dict[str, str]] = {} + for name, builder, output_paths in builders: + with tempfile.TemporaryDirectory(prefix=f"static-{name}-") as temporary_dir: + backup_dir = Path(temporary_dir) + for index, output_path in enumerate(output_paths): + _copy_output(output_path, backup_dir / str(index)) + + try: + await builder() + except Exception as error: + _restore_outputs(output_paths, backup_dir) + logging.exception("%s builder failed; restored previous output", name) + print( + f"::error title={name} builder failed::" + f"{type(error).__name__}: {error}" + ) + results[name] = {"status": "failed", "error": type(error).__name__} + else: + results[name] = {"status": "ok"} + + status_path.parent.mkdir(parents=True, exist_ok=True) + status_path.write_text( + json.dumps( + { + "generated_at": datetime.now(UTC).isoformat(), + "builders": results, + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + return results + + def main() -> None: parser = argparse.ArgumentParser(description="Life-USTC static builders") parser.add_argument( @@ -43,15 +106,29 @@ def main() -> None: run_curriculum = args.curriculum or run_all run_young = args.young or run_all - async def _run(): - if run_rss: - await make_rss() - if run_curriculum: - await make_curriculum() - if run_young: - await make_young_events() + builders: list[tuple[str, Callable[[], Awaitable[None]], tuple[Path, ...]]] = [] + if run_rss: + builders.append(("rss", make_rss, (build_dir / "rss",))) + if run_curriculum: + builders.append( + ( + "curriculum", + make_curriculum, + ( + build_dir / SNAPSHOT_FILENAME, + build_dir / GUESSES_FILENAME, + ), + ) + ) + if run_young: + builders.append(("young", make_young_events, (build_dir / SNAPSHOT_FILENAME,))) + + results = asyncio.run( + _run_builders(builders, status_path=build_dir / "build-status.json") + ) + if results and all(result["status"] == "failed" for result in results.values()): + raise RuntimeError("All selected static builders failed") - asyncio.run(_run()) if run_curriculum or run_young: schema_paths = export_upstream_schemas(build_dir / "schemas" / "upstream") logging.info("Exported %s upstream JSON schemas", len(schema_paths)) diff --git a/src/curriculum.py b/src/curriculum.py index 2d9ceea68f..be0291dd09 100644 --- a/src/curriculum.py +++ b/src/curriculum.py @@ -2,6 +2,7 @@ import time from json import JSONDecodeError +import httpx from tqdm import tqdm from .guesses import SQLiteGuessStore @@ -34,7 +35,9 @@ CATALOG_SEMESTER_URL = "https://catalog.ustc.edu.cn/api/teach/semester/list" CATALOG_DEPARTMENT_URL = "https://catalog.ustc.edu.cn/api/teach/department/college-tree" -CATALOG_LESSON_URL_PREFIX = "https://catalog.ustc.edu.cn/api/teach/lesson/list-for-teach" +CATALOG_LESSON_URL_PREFIX = ( + "https://catalog.ustc.edu.cn/api/teach/lesson/list-for-teach" +) CATALOG_EXAM_URL_PREFIX = "https://catalog.ustc.edu.cn/api/teach/exam/list" JW_SCHEDULE_TABLE_URL = "https://jw.ustc.edu.cn/ws/schedule-table/datum" MIN_CATALOG_LESSON_SEMESTER_ID = 221 @@ -170,9 +173,7 @@ def _catalog_lesson_chunk_count( """, (fetch[0],), ).fetchone()[0] - return ( - int(lesson_count) + JW_SCHEDULE_CHUNK_SIZE - 1 - ) // JW_SCHEDULE_CHUNK_SIZE + return (int(lesson_count) + JW_SCHEDULE_CHUNK_SIZE - 1) // JW_SCHEDULE_CHUNK_SIZE def _expected_jw_schedule_chunk_count( @@ -298,6 +299,9 @@ def _delete_cached_semester( def _is_skippable_exam_fetch_error(error: Exception) -> bool: + if isinstance(error, httpx.HTTPStatusError): + return error.response.status_code in {502, 504} + message = str(error).lower() return any( marker in message @@ -570,9 +574,7 @@ async def make_curriculum() -> None: "selected_semester_count": len(selected_semesters), "refreshed_semester_count": len(refreshed_semesters), "cached_ended_semester_count": len(cached_ended_semester_ids), - "cached_ended_semester_ids": ",".join( - cached_ended_semester_ids - ), + "cached_ended_semester_ids": ",".join(cached_ended_semester_ids), "catalog_lesson_min_semester_id": MIN_CATALOG_LESSON_SEMESTER_ID, "catalog_lesson_skipped_legacy_semester_count": len( skipped_catalog_lesson_semester_ids diff --git a/src/rss.py b/src/rss.py index 7909b659f8..779fe67823 100644 --- a/src/rss.py +++ b/src/rss.py @@ -6,6 +6,7 @@ import feedgenerator import feedparser import html2text +import httpx import yaml from tqdm import tqdm @@ -16,9 +17,26 @@ async def get_and_clean_feed(url: str, path_to_save: Path): - feed = feedparser.parse(url) + try: + response = httpx.get(url, timeout=60, follow_redirects=True) + response.raise_for_status() + except httpx.HTTPError as error: + logger.warning( + "Keeping cached %s after feed fetch failed: %s", path_to_save, error + ) + return + + feed = feedparser.parse(response.content) + + if feed.bozo: + logger.warning("Feed %s reported a parse warning: %s", url, feed.bozo_exception) if not feed.entries: + logger.warning( + "Keeping cached %s because the fetched feed has no entries%s", + path_to_save, + f": {feed.bozo_exception}" if feed.bozo else "", + ) return feed_title = getattr(feed.feed, "title", "RSS Feed") @@ -33,6 +51,7 @@ async def get_and_clean_feed(url: str, path_to_save: Path): handler.ignore_links = True handler.ignore_images = True + written_items = 0 for entry in tqdm( feed.entries, position=0, @@ -54,10 +73,18 @@ async def get_and_clean_feed(url: str, path_to_save: Path): description=description, pubdate=date, ) + written_items += 1 except Exception as e: logger.exception("Failed to process feed entry: %s", e) - with open(path_to_save, "w") as f: + if written_items == 0: + logger.warning( + "Keeping cached %s because no fetched entries could be parsed", + path_to_save, + ) + return + + with open(path_to_save, "w", encoding="utf-8") as f: new_feed.write(f, "utf-8") diff --git a/src/sqlite_store.py b/src/sqlite_store.py index 596cad02af..8b3dafdca3 100644 --- a/src/sqlite_store.py +++ b/src/sqlite_store.py @@ -242,7 +242,7 @@ def store_response( response: BaseModel, fetch_id: int, context: Mapping[str, Scalar | None] | None = None, - ) -> int: + ) -> int: context = context or {} if isinstance(response, RootModel): @@ -261,9 +261,7 @@ def store_response( count += 1 return count if isinstance(root, BaseModel): - self._insert_model( - table_name, root, fetch_id=fetch_id, context=context - ) + self._insert_model(table_name, root, fetch_id=fetch_id, context=context) return 1 return 0 diff --git a/src/utils/auth.py b/src/utils/auth.py index 758d78e25d..b77163ab66 100644 --- a/src/utils/auth.py +++ b/src/utils/auth.py @@ -46,7 +46,7 @@ class LoginConfig: @classmethod def from_env(cls) -> "LoginConfig": - timeout_ms = int(os.getenv("USTC_TIMEOUT_MS", "0")) + timeout_ms = int(os.getenv("USTC_TIMEOUT_MS", "60000")) attempts = max(int(os.getenv("USTC_LOGIN_ATTEMPTS", "3")), 1) state_max_turns = max(int(os.getenv("USTC_LOGIN_STATE_MAX_TURNS", "10")), 1) turn_wait_ms = max(int(os.getenv("USTC_LOGIN_TURN_WAIT_MS", "5000")), 0) @@ -172,14 +172,12 @@ def __init__( page: Page | None, timeout_ms: int = 10 * 60 * 1000, fail_on_status_code: bool = True, - max_retries: int = 100, transient_retries: int = 3, ): self.client = client self.page = page self.timeout_ms = timeout_ms self.fail_on_status_code = fail_on_status_code - self.max_retries = max_retries self.transient_retries = transient_retries self.logger = logging.getLogger(__name__) diff --git a/src/utils/jw.py b/src/utils/jw.py index b91692f2e5..a6aeaf568e 100644 --- a/src/utils/jw.py +++ b/src/utils/jw.py @@ -13,9 +13,7 @@ from .tools import compose_start_end, join_nonempty _jw_user_id_cache: dict[int, str] = {} -JW_SSO_URL = ( - "https://jw.ustc.edu.cn/ucas-sso/login" -) +JW_SSO_URL = "https://jw.ustc.edu.cn/ucas-sso/login" indexStartTimes: dict[int, int] = { 1: 7 * 60 + 50, @@ -301,7 +299,7 @@ async def fetch_jw_schedule_table_json( url=url, data={"lessonIds": course_id_list}, timeout=30_000, - max_retries=2, + transient_retries=2, ) diff --git a/src/utils/tj_rss.py b/src/utils/tj_rss.py index 5c0dc52156..fa60bf6d31 100644 --- a/src/utils/tj_rss.py +++ b/src/utils/tj_rss.py @@ -74,7 +74,8 @@ def tj_ustc_RSS(output_dir: Path | str): s = requests.Session() s.mount("http://", HTTPAdapter(max_retries=3)) - r = s.get(url, headers=headers) + r = s.get(url, headers=headers, timeout=60) + r.raise_for_status() r.encoding = "utf-8" html = r.text diff --git a/tests/test_auth.py b/tests/test_auth.py index e694f028d3..2970d90f23 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -5,7 +5,12 @@ import httpx -from src.utils.auth import RequestSession, USTCSession, _create_request_http_client +from src.utils.auth import ( + LoginConfig, + RequestSession, + USTCSession, + _create_request_http_client, +) class RequestSessionTest(unittest.IsolatedAsyncioTestCase): @@ -77,8 +82,39 @@ async def handler(request: httpx.Request) -> httpx.Response: finally: await client.aclose() + async def test_post_json_honors_per_request_transient_retries(self) -> None: + attempts = 0 + + async def handler(request: httpx.Request) -> httpx.Response: + nonlocal attempts + attempts += 1 + return httpx.Response(503, request=request) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + session = RequestSession(client=client, page=None) + with ( + patch.object(session, "_request_retry_wait_ms", return_value=0), + self.assertRaises(httpx.HTTPStatusError), + ): + await session.post_json( + "https://jw.ustc.edu.cn/ws/schedule-table/datum", + data={"lessonIds": ["1"]}, + transient_retries=2, + ) + finally: + await client.aclose() + + self.assertEqual(attempts, 3) + class USTCSessionConfigTest(unittest.TestCase): + def test_login_timeout_defaults_to_one_minute(self) -> None: + with patch.dict(os.environ, {}, clear=True): + config = LoginConfig.from_env() + + self.assertEqual(config.timeout_ms, 60_000) + def test_after_login_services_default_to_enabled(self) -> None: with patch.dict( os.environ, diff --git a/tests/test_curriculum.py b/tests/test_curriculum.py index 1098bf211f..52b5cf6236 100644 --- a/tests/test_curriculum.py +++ b/tests/test_curriculum.py @@ -2,9 +2,12 @@ from json import JSONDecodeError from unittest.mock import AsyncMock, MagicMock, patch +import httpx + from src.curriculum import ( _cached_complete_semester_ids, _has_cached_jw_schedule, + _is_skippable_exam_fetch_error, _jw_schedule_expected_chunk_count_key, _refresh_curriculum_semesters, _selected_curriculum_semesters, @@ -169,6 +172,32 @@ async def test_failed_chunk_aborts_refresh_and_is_not_complete(self) -> None: class CatalogExamFetchTest(unittest.TestCase): + def test_skips_structured_bad_gateway_errors(self) -> None: + request = httpx.Request( + "GET", "https://catalog.ustc.edu.cn/api/teach/exam/list/401" + ) + for status_code in (502, 504): + response = httpx.Response(status_code, request=request) + error = httpx.HTTPStatusError( + "Bad Gateway", + request=request, + response=response, + ) + self.assertTrue(_is_skippable_exam_fetch_error(error)) + + def test_does_not_skip_other_structured_http_errors(self) -> None: + request = httpx.Request( + "GET", "https://catalog.ustc.edu.cn/api/teach/exam/list/401" + ) + response = httpx.Response(500, request=request) + error = httpx.HTTPStatusError( + "Server Error", + request=request, + response=response, + ) + + self.assertFalse(_is_skippable_exam_fetch_error(error)) + def test_skips_semesters_below_minimum_exam_id(self) -> None: self.assertFalse(_should_fetch_catalog_exams("221")) self.assertFalse(_should_fetch_catalog_exams("362")) diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000000..b3b222b26e --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,40 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from main import _run_builders + + +class BuilderIsolationTest(unittest.IsolatedAsyncioTestCase): + async def test_failed_builder_restores_output_and_later_builder_runs(self) -> None: + with tempfile.TemporaryDirectory() as temporary_dir: + root = Path(temporary_dir) + shared_output = root / "snapshot.sqlite" + status_path = root / "build-status.json" + shared_output.write_text("cached", encoding="utf-8") + later_builder_ran = False + + async def failing_builder() -> None: + shared_output.write_text("partial", encoding="utf-8") + raise RuntimeError("upstream unavailable") + + async def succeeding_builder() -> None: + nonlocal later_builder_ran + later_builder_ran = True + + results = await _run_builders( + [ + ("curriculum", failing_builder, (shared_output,)), + ("young", succeeding_builder, (shared_output,)), + ], + status_path=status_path, + ) + + status = json.loads(status_path.read_text(encoding="utf-8")) + + self.assertEqual(shared_output.read_text(encoding="utf-8"), "cached") + self.assertTrue(later_builder_ran) + self.assertEqual(results["curriculum"]["status"], "failed") + self.assertEqual(results["young"]["status"], "ok") + self.assertEqual(status["builders"], results) diff --git a/tests/test_rss.py b/tests/test_rss.py new file mode 100644 index 0000000000..6d6583b3bc --- /dev/null +++ b/tests/test_rss.py @@ -0,0 +1,47 @@ +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +import httpx + +from src.rss import get_and_clean_feed + + +class RssCacheSafetyTest(unittest.IsolatedAsyncioTestCase): + async def test_fetch_failure_keeps_cached_feed(self) -> None: + with tempfile.TemporaryDirectory() as temporary_dir: + feed_path = Path(temporary_dir) / "feed.xml" + feed_path.write_text("cached", encoding="utf-8") + with patch( + "src.rss.httpx.get", + side_effect=httpx.ConnectError("offline"), + ): + await get_and_clean_feed("https://example.com/feed", feed_path) + + self.assertEqual(feed_path.read_text(encoding="utf-8"), "cached") + + async def test_unparseable_entries_keep_cached_feed(self) -> None: + response = MagicMock() + response.content = b"" + parsed_feed = MagicMock() + parsed_feed.entries = [ + { + "title": "Entry", + "link": "https://example.com/entry", + "published": "unsupported date", + "description": "Description", + } + ] + parsed_feed.feed = {"title": "Feed"} + + with tempfile.TemporaryDirectory() as temporary_dir: + feed_path = Path(temporary_dir) / "feed.xml" + feed_path.write_text("cached", encoding="utf-8") + with ( + patch("src.rss.httpx.get", return_value=response), + patch("src.rss.feedparser.parse", return_value=parsed_feed), + ): + await get_and_clean_feed("https://example.com/feed", feed_path) + + self.assertEqual(feed_path.read_text(encoding="utf-8"), "cached") diff --git a/tests/test_sqlite_store.py b/tests/test_sqlite_store.py index a8e24d2b3e..77681aae9c 100644 --- a/tests/test_sqlite_store.py +++ b/tests/test_sqlite_store.py @@ -10,7 +10,9 @@ def test_can_reopen_existing_store_without_resetting_it(self) -> None: with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "snapshot.sqlite" store = SQLiteModelStore(path) - store.record_fetch(source="source", method="GET", url="https://example.test") + store.record_fetch( + source="source", method="GET", url="https://example.test" + ) store.close() reopened = SQLiteModelStore(path, reset=False) diff --git a/tests/test_young.py b/tests/test_young.py index 02f5fac6a1..808b3ca539 100644 --- a/tests/test_young.py +++ b/tests/test_young.py @@ -139,11 +139,7 @@ async def get_json(self, url: str): self.urls.append(url) params = parse_qs(urlsplit(url).query) page_no = int(params["pageNo"][0]) - records = ( - [{"id": "1"}, {"id": "2"}] - if page_no == 1 - else [{"id": "3"}] - ) + records = [{"id": "1"}, {"id": "2"}] if page_no == 1 else [{"id": "3"}] return { "success": True, "result": { @@ -168,10 +164,7 @@ async def get_json(self, url: str): ["1", "2", "3"], ) self.assertEqual( - [ - parse_qs(urlsplit(url).query)["pageNo"][0] - for url in session.urls - ], + [parse_qs(urlsplit(url).query)["pageNo"][0] for url in session.urls], ["1", "2"], )