diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b4100c2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +# Local dev/media artifacts must not end up in the image (uploads persist +# on /mnt/data/GCAtlas via bind mount). +media/ +.venv/ +.git/ +data/backups/ diff --git a/.gitignore b/.gitignore index a18385a..efa1240 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,7 @@ local_settings.py db.sqlite3 db.sqlite3-journal staticfiles +media/ # Flask stuff: instance/ diff --git a/Containerfile b/Containerfile index cb6ee1c..f238562 100644 --- a/Containerfile +++ b/Containerfile @@ -7,12 +7,19 @@ COPY . /app RUN UV_PROJECT_ENVIRONMENT=/usr/local \ uv sync --project=/app --frozen --compile-bytecode --no-dev --no-editable --no-managed-python -FROM gcr.io/distroless/base-debian13:nonroot +RUN python django_manage.py collectstatic --noinput + +FROM ghcr.io/astral-sh/uv:python3.14-trixie-slim + +RUN useradd --create-home --shell /bin/bash nonroot COPY --from=builder /usr/local /usr/local COPY --from=builder /app /app +# Media uploads land here; make the named volume writable by nonroot. +RUN mkdir -p /app/media && chown -R nonroot:nonroot /app/media + WORKDIR /app USER nonroot diff --git a/apps/auth/utils.py b/apps/auth/utils.py index 3e63695..4411e5c 100644 --- a/apps/auth/utils.py +++ b/apps/auth/utils.py @@ -19,7 +19,7 @@ AUTH_SETTINGS = settings.AUTH PASSWORD_LENGTH_MIN = AUTH_SETTINGS["PASSWORD_LENGTH_MIN"] PASSWORD_LENGTH_MAX = AUTH_SETTINGS["PASSWORD_LENGTH_MAX"] -OTP_TIMEOUT = AUTH_SETTINGS["OTP_TIMEOUT"] +OTP_TIMEOUT = int(AUTH_SETTINGS["OTP_TIMEOUT"]) EMAIL_DOMAIN_NAME = AUTH_SETTINGS["EMAIL_DOMAIN_NAME"] QUEST_SETTINGS = settings.QUEST @@ -46,7 +46,7 @@ def get_survey_details(action: str) -> dict[str, Any] | None: try: question_id = int(action_details.get("QUESTIONID")) - except ValueError, TypeError: + except (ValueError, TypeError): # fmt: skip logger.error( "Could not parse 'QUESTIONID' for action '%s'. Check your settings.", action ) @@ -59,35 +59,75 @@ def get_survey_details(action: str) -> dict[str, Any] | None: } +# Dedicated network timeout for the siteverify HTTP call. Kept short and +# separate from OTP_TIMEOUT: the OTP window is a business rule, not a +# network deadline. +TURNSTILE_VERIFY_TIMEOUT = 15 +TURNSTILE_VERIFY_RETRIES = 3 + + async def verify_turnstile_token( turnstile_token, client_ip ) -> tuple[bool, Response | None]: """Helper function to verify Turnstile token with Cloudflare's API""" - try: - async with httpx.AsyncClient(timeout=OTP_TIMEOUT) as client: - response = await client.post( - "https://challenges.cloudflare.com/turnstile/v0/siteverify", - data={ - "secret": settings.TURNSTILE_SECRET_KEY, - "response": turnstile_token, - "remoteip": client_ip, - }, + last_error: Exception | None = None + for attempt in range(TURNSTILE_VERIFY_RETRIES): + try: + async with httpx.AsyncClient(timeout=TURNSTILE_VERIFY_TIMEOUT) as client: + response = await client.post( + "https://challenges.cloudflare.com/turnstile/v0/siteverify", + data={ + "secret": settings.TURNSTILE_SECRET_KEY, + "response": turnstile_token, + "remoteip": client_ip, + }, + ) + try: + data = response.json() + except ValueError: + logger.error( + "Turnstile siteverify returned non-JSON: status=%s body=%s", + response.status_code, + response.text[:200], + ) + return False, Response( + {"error": "Turnstile verification error"}, status=502 + ) + if not data.get("success"): + logger.warning("Turnstile verification failed: %s", data) + return False, Response( + {"error": "Turnstile verification failed"}, status=403 + ) + return True, None + except httpx.TimeoutException as e: + last_error = e + logger.warning( + "Turnstile verification timed out (attempt %d/%d)", + attempt + 1, + TURNSTILE_VERIFY_RETRIES, ) - if not response.json().get("success"): - logger.warning("Turnstile verification failed: %s", response.json()) + except httpx.HTTPError as e: + # ConnectError / ReadError etc.: transient network failures. + last_error = e + logger.warning( + "Turnstile verification network error (attempt %d/%d): %s", + attempt + 1, + TURNSTILE_VERIFY_RETRIES, + e, + ) + except Exception: + logger.exception("Turnstile verification error") return False, Response( - {"error": "Turnstile verification failed"}, status=403 + {"error": "Turnstile verification error"}, status=500 ) - return True, None - except httpx.TimeoutException: - logger.error("Turnstile verification timed out") - return False, Response( - {"error": "Turnstile verification timed out"}, status=504 - ) - except Exception: - logger.error("Turnstile verification error") - return False, Response({"error": "Turnstile verification error"}, status=500) + + logger.error( + "Turnstile verification failed after %d attempts: %s", + TURNSTILE_VERIFY_RETRIES, + last_error, + ) + return False, Response({"error": "Turnstile verification timed out"}, status=504) async def get_latest_answer( @@ -133,7 +173,7 @@ async def get_latest_answer( full_url_path = f"{QUEST_BASE_URL}/{quest_api}/json" try: - async with httpx.AsyncClient(timeout=OTP_TIMEOUT) as client: + async with httpx.AsyncClient(timeout=TURNSTILE_VERIFY_TIMEOUT) as client: response = await client.get( full_url_path, params=final_query_params, @@ -153,7 +193,10 @@ async def get_latest_answer( status=500, ) except Exception: - logger.error("An unexpected error occurred") + logger.exception( + "Questionnaire API returned unexpected response: %s", + response.text[:200] if "response" in locals() else "(no response)", + ) return None, Response({"error": "An unexpected error occurred"}, status=500) # Filter and return only the required fields from the first row diff --git a/apps/auth/views.py b/apps/auth/views.py index 01aafb0..26b8ef8 100644 --- a/apps/auth/views.py +++ b/apps/auth/views.py @@ -25,7 +25,7 @@ AUTH_SETTINGS = settings.AUTH -OTP_TIMEOUT = AUTH_SETTINGS["OTP_TIMEOUT"] +OTP_TIMEOUT = int(AUTH_SETTINGS["OTP_TIMEOUT"]) TEMP_TOKEN_TIMEOUT = AUTH_SETTINGS["TEMP_TOKEN_TIMEOUT"] ACTION_LIST = AUTH_SETTINGS["ACTION_LIST"] TOKEN_RATE_LIMIT = AUTH_SETTINGS["TOKEN_RATE_LIMIT"] @@ -240,7 +240,7 @@ def verify_callback_api(request): otp_data = json.loads(otp_data_raw.decode("utf-8")) expected_temp_token = otp_data.get("temp_token") initiated_at = otp_data.get("initiated_at") - except json.JSONDecodeError, AttributeError: + except (json.JSONDecodeError, AttributeError): # fmt: skip logger.error("Invalid OTP data format in verify_callback_api") return Response({"error": "Invalid OTP data format"}, status=401) @@ -261,15 +261,36 @@ def verify_callback_api(request): submitted_at = dateutil.parser.parse(submitted_at_str).timestamp() - # Additional validation: check submission is after initiation and within window - if submitted_at < initiated_at or (submitted_at - initiated_at) > OTP_TIMEOUT: + # Additional validation: check submission is after initiation and within window. + # The WJ platform's server clock is measurably slow and drifts over time + # (measured ~39s slow on 2026-08-13, ~156s slow on 2026-08-29, drifting + # ~7s/day). Tolerance raised to 220s to absorb current drift; revisit if + # drift continues to grow (see dynamic calibration as the durable fix). + timestamp_tolerance = 220 + if ( + submitted_at + timestamp_tolerance < initiated_at + or (submitted_at - initiated_at) > OTP_TIMEOUT + ): + logger.warning( + "Submission timestamp outside validity window: " + "submitted_at=%s initiated_at=%s diff=%.1fs", + submitted_at, + initiated_at, + submitted_at - initiated_at, + ) return Response( {"error": "Submission timestamp outside validity window"}, status=401, ) - except ValueError, TypeError: - logger.error("Error parsing submission timestamp") + except (ValueError, TypeError) as e: + logger.error( + "Error parsing submission timestamp: submitted_at_str=%r " + "initiated_at=%r exception=%s", + locals().get("submitted_at_str"), + locals().get("initiated_at"), + e, + ) return Response({"error": "Invalid submission timestamp"}, status=401) # Step 7: Update state to verified and add user details diff --git a/apps/spider/crawlers/gc_offerings.py b/apps/spider/crawlers/gc_offerings.py index 7b5137f..fd1ef56 100644 --- a/apps/spider/crawlers/gc_offerings.py +++ b/apps/spider/crawlers/gc_offerings.py @@ -3,13 +3,11 @@ import requests from bs4 import BeautifulSoup -from django.db import transaction +from django.db import models, transaction -from apps.web.models import Course +from apps.web.models import Course, CourseOffering, Instructor -OFFERINGS_URL = ( - "https://gc.sjtu.edu.cn/academics/courses/present-course-offerings/" -) +OFFERINGS_URL = "https://gc.sjtu.edu.cn/academics/courses/present-course-offerings/" HEADING_RE = re.compile( r"Courses\s+Offered\s+in\s+(Spring|Summer|Fall)\s+(20\d{2})", re.IGNORECASE, @@ -21,6 +19,86 @@ TERM_CODES = {"spring": "SP", "summer": "SU", "fall": "FA"} MIN_EXPECTED_OFFERINGS = 20 +# --------------------------------------------------------------------------- +# Instructor name canonicalization +# +# The GC page itself is inconsistent: the same person appears under different +# spellings across terms (case, hyphens, middle names, term annotations like +# "(Fall)", CJK annotations like "闫旭", or even full-name/short-name +# alternation). Importing each cell verbatim with get_or_create(name) silently +# forks one teacher into several Instructor rows every time the page rotates. +# The pure-string cleaning/matching helpers live in lib.name_normalization +# (shared with the review API); the functions here resolve against the live +# Instructor table. +# --------------------------------------------------------------------------- + +from lib.name_normalization import ( # noqa: E402 + INSTRUCTOR_SPLITS, + JUNK_INSTRUCTOR_NAMES, + best_name_match, + clean_instructor_name, +) + + +def _best_instructor_match(clean_name, existing): + """Find the Instructor row a cleaned name should map to. + + Candidates are ordered by usage (most offerings first) so ties resolve to + the most-used spelling as canonical. Returns None when nothing plausibly + matches. + """ + ranked = sorted( + existing, + key=lambda r: (-_offering_count(r), r.id), + ) + matched = best_name_match(clean_name, [r.name for r in ranked]) + if matched is None: + return None + return next((r for r in ranked if r.name == matched), None) + + +def _offering_count(row): + """Offering count for tie-breaking, honoring a prefetched annotation.""" + count = getattr(row, "_offering_count", None) + if count is not None: + return count + return row.courseoffering_set.count() + + +def resolve_instructor(clean_name, existing=None): + """Return the Instructor row for a cleaned name, reusing existing rows + across spelling variants. Creates a new row only when no existing + instructor plausibly matches.""" + existing = ( + list(existing) if existing is not None else list(Instructor.objects.all()) + ) + match = _best_instructor_match(clean_name, existing) + if match is not None: + return match + instructor, _ = Instructor.objects.get_or_create(name=clean_name) + return instructor + + +def expand_instructor_names(raw_names, existing): + """Resolve raw GC cell text to Instructor rows, self-defensively. + + Applies cleaning, junk filtering, and curated multi-person splits so the + importer behaves correctly even when handed payloads stored before the + canonicalization logic existed. Returns a deduped list of Instructor rows. + """ + resolved = [] + for raw in raw_names: + cleaned = clean_instructor_name(raw) + if not cleaned or cleaned.lower() in JUNK_INSTRUCTOR_NAMES: + continue + for name in INSTRUCTOR_SPLITS.get(cleaned, [cleaned]): + inst = resolve_instructor(name, existing) + if all(row.id != inst.id for row in existing): + existing.append(inst) + if not any(row.id == inst.id for row in resolved): + resolved.append(inst) + return resolved + class GCOfferingsParseError(ValueError): pass @@ -84,9 +162,7 @@ def _coalesce_course_metadata(offerings): for (_, course_code), items in by_code.items(): for field in fields: - populated = { - item[field] for item in items if item[field] not in (None, "") - } + populated = {item[field] for item in items if item[field] not in (None, "")} if len(populated) > 1: raise GCOfferingsParseError( f"conflicting {field} values for {course_code}: " @@ -97,11 +173,7 @@ def _coalesce_course_metadata(offerings): item[field] = value crosslisted_codes = sorted( - { - code - for item in items - for code in item.get("crosslisted_codes", []) - } + {code for item in items for code in item.get("crosslisted_codes", [])} ) for item in items: item["crosslisted_codes"] = crosslisted_codes @@ -197,31 +269,49 @@ def _parse_course_cells(values, source_url): def _parse_instructors(value): - value = value.strip() - if not value or value in {"-", "–", "—"}: - return [] - return [name for name in INSTRUCTOR_SEPARATOR_RE.split(value) if name] + names = [] + for raw in INSTRUCTOR_SEPARATOR_RE.split(value or ""): + cleaned = clean_instructor_name(raw) + if not cleaned or cleaned.lower() in JUNK_INSTRUCTOR_NAMES: + continue + # Expand cells that cram several instructors together (curated). + names.extend(INSTRUCTOR_SPLITS.get(cleaned, [cleaned])) + return names @transaction.atomic def import_gc_courses(offerings): if not offerings: raise ValueError("refusing to import an empty course list") - imported_codes = set() + courses_by_code = {} + existing = list( + Instructor.objects.annotate(_offering_count=models.Count("courseoffering")) + ) for item in offerings: - if item["course_code"] in imported_codes: - continue - Course.objects.update_or_create( - course_code=item["course_code"], - defaults={ - "course_title": item["course_title"], - "department": item["department"], - "number": item["number"], - "course_credits": item["course_credits"], - "url": item["url"], - }, + course = courses_by_code.get(item["course_code"]) + if course is None: + course, _ = Course.objects.update_or_create( + course_code=item["course_code"], + defaults={ + "course_title": item["course_title"], + "department": item["department"], + "number": item["number"], + "course_credits": item["course_credits"], + "url": item["url"], + }, + ) + courses_by_code[item["course_code"]] = course + + instructors = expand_instructor_names(item["instructors"], existing) + # Sections are numbered by row order within the GC page table, not by + # the registrar's section numbers (the page has no such column). + offering, _ = CourseOffering.objects.get_or_create( + course=course, + term=item["term"], + section=item["section"], + defaults={"period": ""}, ) - imported_codes.add(item["course_code"]) + offering.instructors.set(instructors) - return len(imported_codes) + return len(courses_by_code) diff --git a/apps/spider/crawlers/orc.py b/apps/spider/crawlers/orc.py index bf55907..7ff99b9 100644 --- a/apps/spider/crawlers/orc.py +++ b/apps/spider/crawlers/orc.py @@ -1,16 +1,20 @@ import re from urllib.parse import urljoin +from django.db import models + +from lib.name_normalization import JUNK_INSTRUCTOR_NAMES, clean_instructor_name +from apps.spider.crawlers.gc_offerings import expand_instructor_names from apps.spider.utils import retrieve_soup # parse_number_and_subnumber, from apps.web.models import Course, CourseOffering, Instructor from lib.constants import CURRENT_TERM -BASE_URL = "https://www.ji.sjtu.edu.cn/" +BASE_URL = "https://gc.sjtu.edu.cn/" ORC_BASE_URL = urljoin(BASE_URL, "/academics/courses/courses-by-number/") # ORC_UNDERGRAD_SUFFIX = "Departments-Programs-Undergraduate" # ORC_GRADUATE_SUFFIX = "Departments-Programs-Graduate" COURSE_DETAIL_URL_PREFIX = ( - "https://www.ji.sjtu.edu.cn/academics/courses/courses-by-number/course-info/?id=" + "https://gc.sjtu.edu.cn/academics/courses/courses-by-number/course-info/?id=" ) UNDERGRAD_URL = ORC_BASE_URL INSTRUCTOR_TERM_REGEX = re.compile(r"^(?P\w*)\s?(\((?P\w*)\))?") @@ -119,9 +123,10 @@ def _crawl_course_data(course_url): course_topics = list(section_lines.get("Course Topics:", [])) instructors = [] for name_line in section_lines.get("Instructors:", []): - instructors.extend( - name.strip() for name in name_line.split(";") if name.strip() - ) + for raw in name_line.split(";"): + cleaned = clean_instructor_name(raw) + if cleaned and cleaned.lower() not in JUNK_INSTRUCTOR_NAMES: + instructors.append(cleaned) return { "course_code": course_code, @@ -143,6 +148,9 @@ def _crawl_course_data(course_url): def import_department(department_data): + existing = list( + Instructor.objects.annotate(_offering_count=models.Count("courseoffering")) + ) for course_data in department_data: # Skip pages whose structure was not recognized: importing them would # overwrite stored course fields with empty values. @@ -169,17 +177,17 @@ def import_department(department_data): }, ) - # Handle instructors + # Handle instructors: resolve against existing rows so spelling drift + # on the catalog page never forks one teacher into several rows. if "instructors" in course_data and course_data["instructors"]: - for instructor_name in course_data["instructors"]: - instructor, _ = Instructor.objects.get_or_create(name=instructor_name) - # Create a course offering for the current term if it doesn't exist - offering, _ = CourseOffering.objects.get_or_create( - course=course, - term=CURRENT_TERM, - defaults={"section": 1, "period": ""}, - ) - offering.instructors.add(instructor) + instructors = expand_instructor_names(course_data["instructors"], existing) + # Create a course offering for the current term if it doesn't exist + offering, _ = CourseOffering.objects.get_or_create( + course=course, + term=CURRENT_TERM, + defaults={"section": 1, "period": ""}, + ) + offering.instructors.add(*instructors) def extract_prerequisites(pre_requisites): diff --git a/apps/spider/tests/test_gc_offerings.py b/apps/spider/tests/test_gc_offerings.py index f372e5c..6496e79 100644 --- a/apps/spider/tests/test_gc_offerings.py +++ b/apps/spider/tests/test_gc_offerings.py @@ -3,10 +3,11 @@ from apps.spider.crawlers.gc_offerings import ( GCOfferingsParseError, + clean_instructor_name, import_gc_courses, parse_gc_offerings, ) -from apps.web.models import Course, CourseOffering +from apps.web.models import Course, CourseOffering, Instructor SAMPLE_HTML = """ @@ -114,12 +115,12 @@ def test_parse_gc_offerings_reads_all_semester_tables(): reason="project migrations use PostgreSQL-only ArrayField columns", ) @pytest.mark.django_db -def test_import_gc_courses_updates_courses_without_changing_offerings(): +def test_import_gc_courses_updates_courses_and_creates_offerings(): rows = parse_gc_offerings(SAMPLE_HTML) stale_course = Course.objects.create( course_code="OLD1000J", course_title="Old", department="OLD", number=1000 ) - CourseOffering.objects.create( + stale_offering = CourseOffering.objects.create( course=stale_course, term="26SU", section=1, period="" ) @@ -128,7 +129,169 @@ def test_import_gc_courses_updates_courses_without_changing_offerings(): physics = Course.objects.get(course_code="PHYS1500J") assert physics.course_title == "Physics I" assert physics.course_credits == 4 - assert not physics.courseoffering_set.exists() - assert CourseOffering.objects.filter( - course=stale_course, term="26SU" - ).exists() + # Unrelated offerings are untouched. + assert CourseOffering.objects.get(pk=stale_offering.pk).course == stale_course + + offerings = list(physics.courseoffering_set.order_by("section")) + assert [(o.term, o.section) for o in offerings] == [("26SU", 1), ("26SU", 2)] + assert [o.instructors_string() for o in offerings] == [ + "Richard Grumitt", + "Mesli Abdelmadjid", + ] + + # Crosslisted code imports under its primary code only. + assert Course.objects.filter(course_code="VK335").count() == 0 + + # A row with no instructor ("–") still gets an offering, without instructors. + materials = Course.objects.get(course_code="MSE3350J") + assert materials.courseoffering_set.get().instructors.count() == 0 + + +@pytest.mark.skipif( + "postgresql" not in settings.DATABASES["default"]["ENGINE"], + reason="project migrations use PostgreSQL-only ArrayField columns", +) +@pytest.mark.django_db +def test_import_gc_courses_is_idempotent(): + rows = parse_gc_offerings(SAMPLE_HTML) + + assert import_gc_courses(rows) == 5 + assert import_gc_courses(rows) == 5 + + assert CourseOffering.objects.count() == 6 + assert Instructor.objects.count() == 6 # 6 unique names across 6 rows + physics = Course.objects.get(course_code="PHYS1500J") + assert physics.courseoffering_set.count() == 2 + + +@pytest.mark.skipif( + "postgresql" not in settings.DATABASES["default"]["ENGINE"], + reason="project migrations use PostgreSQL-only ArrayField columns", +) +@pytest.mark.django_db +def test_import_gc_courses_syncs_changed_instructors(): + rows = parse_gc_offerings(SAMPLE_HTML) + assert import_gc_courses(rows) == 5 + + physics_rows = [row for row in rows if row["course_code"] == "PHYS1500J"] + physics_rows[0]["instructors"] = ["New Professor"] + assert import_gc_courses(rows) == 5 + + physics = Course.objects.get(course_code="PHYS1500J") + section_one = physics.courseoffering_set.get(term="26SU", section=1) + assert [i.name for i in section_one.instructors.all()] == ["New Professor"] + section_two = physics.courseoffering_set.get(term="26SU", section=2) + assert [i.name for i in section_two.instructors.all()] == ["Mesli Abdelmadjid"] + # Instructor rows are never deleted, only unbound. + assert Instructor.objects.filter(name="Richard Grumitt").exists() + + +def test_clean_instructor_name_strips_page_annotations(): + assert clean_instructor_name("Sung-Liang Chen (Fall)") == "Sung-Liang Chen" + assert clean_instructor_name("Rui Yang (Summer).") == "Rui Yang" + assert clean_instructor_name("YAN Xu 闫旭") == "YAN Xu" + assert clean_instructor_name("Qiong Yu (余琼)") == "Qiong Yu" + assert clean_instructor_name("Jaehyung “Joshua” Ju") == "Jaehyung Joshua Ju" + assert clean_instructor_name("Dr. Lin Yun") == "Lin Yun" + assert clean_instructor_name("Albert Shih (UM)") == "Albert Shih" + # Junk / empty cells collapse to "" and are dropped upstream. + assert clean_instructor_name("教师") == "" + assert clean_instructor_name(",") == "" + assert clean_instructor_name(" ") == "" + + +@pytest.mark.skipif( + "postgresql" not in settings.DATABASES["default"]["ENGINE"], + reason="project migrations use PostgreSQL-only ArrayField columns", +) +@pytest.mark.django_db +def test_import_reuses_instructor_across_spelling_variants(): + """A name drift on the GC page must rebind to the existing row, never fork.""" + rows = parse_gc_offerings(SAMPLE_HTML) + assert import_gc_courses(rows) == 5 + assert Instructor.objects.count() == 6 + + physics = Course.objects.get(course_code="PHYS1500J") + section_one = physics.courseoffering_set.get(term="26SU", section=1) + section_two = physics.courseoffering_set.get(term="26SU", section=2) + original_instructor = section_one.instructors.get() + mesli_instructor = section_two.instructors.get() + + # GC rotates to a word-reversed spelling for the same person. + rows[0]["instructors"] = ["Grumitt Richard"] + assert import_gc_courses(rows) == 5 + + section_one.refresh_from_db() + # No new Instructor row; the offering stays bound to the original one. + assert Instructor.objects.count() == 6 + assert section_one.instructors.get().pk == original_instructor.pk + # Case/punctuation drift is absorbed the same way. + rows[1]["instructors"] = ["mesli-abdelmadjid"] + assert import_gc_courses(rows) == 5 + assert Instructor.objects.count() == 6 + section_two.refresh_from_db() + assert section_two.instructors.get().pk == mesli_instructor.pk + + +@pytest.mark.skipif( + "postgresql" not in settings.DATABASES["default"]["ENGINE"], + reason="project migrations use PostgreSQL-only ArrayField columns", +) +@pytest.mark.django_db +def test_import_filters_junk_and_expands_merged_cells(): + """Junk cells (教师, punctuation) vanish; merged cells split into people.""" + html = """ +

Courses Offered in Summer 2026

+ + + + + +
Course CodeCourse Title -CHNCourse Title -ENGCrsInstructor(s)
TEST1000J测试Test Course4Dr. Lin Yun, 教师, YAN Xu 闫旭
+ """ + rows = parse_gc_offerings(html) + assert rows[0]["instructors"] == ["Lin Yun", "YAN Xu"] + + assert import_gc_courses(rows) == 1 + offering = CourseOffering.objects.get() + assert {i.name for i in offering.instructors.all()} == {"Lin Yun", "YAN Xu"} + + # Merged multi-person cell (no separator) splits into both people. + html2 = html.replace("Dr. Lin Yun, 教师, YAN Xu 闫旭", "Zhaoguang Wang Ting Sun") + rows2 = parse_gc_offerings(html2) + assert rows2[0]["instructors"] == ["Zhaoguang Wang", "Ting Sun"] + + +@pytest.mark.skipif( + "postgresql" not in settings.DATABASES["default"]["ENGINE"], + reason="project migrations use PostgreSQL-only ArrayField columns", +) +@pytest.mark.django_db +def test_import_splits_merged_cells_from_stale_payloads(): + """Payloads stored before canonicalization carry unsplit merged cells; + the importer must still split them instead of collapsing to one person.""" + rows = parse_gc_offerings(SAMPLE_HTML) + rows.append( + { + "course_code": "ME3950J", + "crosslisted_codes": [], + "course_title": "Laboratory I", + "course_title_chn": "", + "department": "ME", + "number": 3950, + "course_credits": 4, + "url": "https://gc.sjtu.edu.cn/", + "term": "25FA", + "section": 1, + "instructors": ["Zhaoguang Wang Ting Sun"], + } + ) + assert import_gc_courses(rows) == 6 + + offering = CourseOffering.objects.get( + course__course_code="ME3950J", term="25FA", section=1 + ) + assert {i.name for i in offering.instructors.all()} == { + "Zhaoguang Wang", + "Ting Sun", + } diff --git a/apps/web/admin.py b/apps/web/admin.py index 48724f0..84c8b95 100644 --- a/apps/web/admin.py +++ b/apps/web/admin.py @@ -6,6 +6,7 @@ from django import forms from django.contrib import admin from django.core.exceptions import PermissionDenied +from django.db import models from django.core.management import call_command from django.core.management.base import CommandError from django.shortcuts import render @@ -20,6 +21,8 @@ Review, ReviewVote, Student, + Syllabus, + SyllabusFile, Vote, ) @@ -89,7 +92,9 @@ def import_legacy_reviews_view(self, request): if form.is_valid(): upload = form.cleaned_data["csv_file"] if upload.size > self.max_upload_size: - form.add_error("csv_file", "CSV files must be no larger than 1 MiB.") + form.add_error( + "csv_file", "CSV files must be no larger than 1 MiB." + ) else: try: csv_text = upload.read().decode("utf-8-sig") @@ -150,3 +155,44 @@ def _run_import(self, csv_text, expected_count, *, execute): admin.site.register(ReviewVote) admin.site.register(Vote) admin.site.register(Student) + + +@admin.register(SyllabusFile) +class SyllabusFileAdmin(admin.ModelAdmin): + list_display = ("id", "original_filename", "sha256", "size", "created_at") + search_fields = ("original_filename", "sha256") + readonly_fields = ("sha256", "original_filename", "size", "created_at") + fields = ("file", "sha256", "original_filename", "size", "content_type") + + +@admin.register(Syllabus) +class SyllabusAdmin(admin.ModelAdmin): + list_display = ( + "id", + "course", + "instructor", + "status", + "is_primary", + "uploaded_by", + "created_at", + ) + list_filter = ("status", "is_primary") + search_fields = ("course__course_code", "course__course_title", "instructor__name") + actions = ("reject_syllabi",) + + def formfield_for_dbfield(self, db_field, **kwargs): + form_field = super().formfield_for_dbfield(db_field, **kwargs) + if isinstance(db_field, models.TextField) and db_field.name == "summary_md": + form_field.widget = forms.Textarea(attrs={"rows": 10, "cols": 80}) + return form_field + + def save_model(self, request, obj, form, change): + super().save_model(request, obj, form, change) + if obj.is_primary: + Syllabus.objects.filter( + course=obj.course, instructor=obj.instructor + ).exclude(pk=obj.pk).update(is_primary=False) + + @admin.action(description="Reject selected syllabi (mark failed, unset primary)") + def reject_syllabi(self, request, queryset): + queryset.update(status=Syllabus.Status.FAILED, is_primary=False) diff --git a/apps/web/management/commands/import_legacy_reviews.py b/apps/web/management/commands/import_legacy_reviews.py index d82eaae..00419da 100644 --- a/apps/web/management/commands/import_legacy_reviews.py +++ b/apps/web/management/commands/import_legacy_reviews.py @@ -1,13 +1,14 @@ import csv import os -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError from django.db import connection, transaction -from apps.web.models import Course, Review +from apps.web.models import Course, Instructor, Review +from lib.name_normalization import canonicalize_professor IMPORT_USERNAME = "LegacyReviewImporter" @@ -49,16 +50,6 @@ def handle(self, *args, **options): f"Eligible count is {eligible}, expected {options['expected_count']}; aborting." ) - unique_rows = [] - seen = set() - duplicate_in_csv = 0 - for row in rows: - if row.duplicate_key in seen: - duplicate_in_csv += 1 - continue - seen.add(row.duplicate_key) - unique_rows.append(row) - course_codes = sorted({row.course_code for row in rows}) courses = {} unmatched = [] @@ -68,13 +59,50 @@ def handle(self, *args, **options): except Course.DoesNotExist: unmatched.append(course_code) + # Canonicalize legacy professor names against each course's instructors + # (same matching rules as the review-submission path) so reversed / + # misspelled CSV variants land on the canonical name and dedupe + # correctly against both the CSV and the database. + course_instructor_names = { + code: list( + Instructor.objects.filter(courseoffering__course=course) + .values_list("name", flat=True) + .distinct() + ) + for code, course in courses.items() + } + canonical_rows = [] + for row in rows: + if row.course_code not in courses: + canonical_rows.append(row) + continue + canonical = canonicalize_professor( + row.professor, course_instructor_names[row.course_code] + ) + canonical_rows.append(replace(row, professor=canonical or row.professor)) + + unique_rows = [] + seen = set() + duplicate_in_csv = 0 + for row in canonical_rows: + if row.duplicate_key in seen: + duplicate_in_csv += 1 + continue + seen.add(row.duplicate_key) + unique_rows.append(row) + matched_rows = [row for row in unique_rows if row.course_code in courses] unmatched_rows = [row for row in unique_rows if row.course_code not in courses] + # A review already exists if any row for the same course carries the + # same comment text. Legacy comments are unique per (course, comment) + # (verified for the shipped CSV), and professor spellings were + # consolidated to canonical names after import — so matching on the + # exact professor string would re-insert duplicate comments whose CSV + # variant did not machine-canonicalize (e.g. single-word "Manuel"). existing_keys = set() for row in matched_rows: if Review.objects.filter( course=courses[row.course_code], - professor=row.professor, comments=row.comments, ).exists(): existing_keys.add(row.duplicate_key) diff --git a/apps/web/migrations/0014_syllabusfile_syllabus.py b/apps/web/migrations/0014_syllabusfile_syllabus.py new file mode 100644 index 0000000..39ace87 --- /dev/null +++ b/apps/web/migrations/0014_syllabusfile_syllabus.py @@ -0,0 +1,127 @@ +# Generated by Django 6.0.5 on 2026-09-05 05:51 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("web", "0013_expand_review_term"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="SyllabusFile", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("file", models.FileField(upload_to="syllabi/")), + ("sha256", models.CharField(db_index=True, max_length=64, unique=True)), + ( + "content_type", + models.CharField(blank=True, default="", max_length=100), + ), + ( + "original_filename", + models.CharField(blank=True, default="", max_length=255), + ), + ("size", models.PositiveBigIntegerField(default=0)), + ("extracted_text", models.TextField(blank=True, default="")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ], + ), + migrations.CreateModel( + name="Syllabus", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "status", + models.CharField( + choices=[ + ("pending", "Pending"), + ("processing", "Processing"), + ("analyzed", "Analyzed"), + ("failed", "Failed"), + ], + db_index=True, + default="pending", + max_length=16, + ), + ), + ("summary_md", models.TextField(blank=True, default="")), + ("verdict", models.JSONField(blank=True, null=True)), + ("comparison", models.JSONField(blank=True, null=True)), + ("is_primary", models.BooleanField(default=False)), + ("error_message", models.TextField(blank=True, default="")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "course", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="syllabi", + to="web.course", + ), + ), + ( + "instructor", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="syllabi", + to="web.instructor", + ), + ), + ( + "uploaded_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="uploaded_syllabi", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "file", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="syllabi", + to="web.syllabusfile", + ), + ), + ], + options={ + "ordering": ["-created_at"], + "indexes": [ + models.Index( + fields=["course", "instructor", "status"], + name="web_syllabu_course__a2f772_idx", + ) + ], + "constraints": [ + models.UniqueConstraint( + fields=("course", "instructor", "file"), + name="unique_course_instructor_syllabus_file", + ) + ], + }, + ), + ] diff --git a/apps/web/migrations/0015_syllabus_status_rejected.py b/apps/web/migrations/0015_syllabus_status_rejected.py new file mode 100644 index 0000000..de97f21 --- /dev/null +++ b/apps/web/migrations/0015_syllabus_status_rejected.py @@ -0,0 +1,28 @@ +# Generated by Claude 2026-09-05: add "rejected" status to Syllabus + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("web", "0014_syllabusfile_syllabus"), + ] + + operations = [ + migrations.AlterField( + model_name="syllabus", + name="status", + field=models.CharField( + choices=[ + ("pending", "Pending"), + ("processing", "Processing"), + ("analyzed", "Analyzed"), + ("rejected", "Rejected"), + ("failed", "Failed"), + ], + db_index=True, + default="pending", + max_length=16, + ), + ), + ] diff --git a/apps/web/models/__init__.py b/apps/web/models/__init__.py index 6c7fa92..0c1c5eb 100644 --- a/apps/web/models/__init__.py +++ b/apps/web/models/__init__.py @@ -5,6 +5,8 @@ from .instructor import Instructor from .review import Review from .student import Student +from .syllabus import Syllabus +from .syllabus_file import SyllabusFile from .vote import Vote from .vote_for_review import ReviewVote @@ -16,6 +18,8 @@ "Instructor", "Review", "Student", + "Syllabus", + "SyllabusFile", "Vote", "ReviewVote", ] diff --git a/apps/web/models/syllabus.py b/apps/web/models/syllabus.py new file mode 100644 index 0000000..0201619 --- /dev/null +++ b/apps/web/models/syllabus.py @@ -0,0 +1,82 @@ +from __future__ import unicode_literals + +from django.conf import settings +from django.db import models +from django.db.models.signals import post_delete +from django.dispatch import receiver + + +class Syllabus(models.Model): + """A syllabus uploaded for one course + instructor pairing. + + `file` points at a shared SyllabusFile (deduped by sha256). The AI + analysis writes `summary_md`, `verdict` and `comparison`; exactly one + analyzed Syllabus per (course, instructor) is `is_primary`. + """ + + class Status: + PENDING = "pending" + PROCESSING = "processing" + ANALYZED = "analyzed" + REJECTED = "rejected" + FAILED = "failed" + + STATUS_CHOICES = [ + (Status.PENDING, "Pending"), + (Status.PROCESSING, "Processing"), + (Status.ANALYZED, "Analyzed"), + (Status.REJECTED, "Rejected"), + (Status.FAILED, "Failed"), + ] + + course = models.ForeignKey( + "Course", on_delete=models.CASCADE, related_name="syllabi" + ) + instructor = models.ForeignKey( + "Instructor", on_delete=models.CASCADE, related_name="syllabi" + ) + file = models.ForeignKey( + "SyllabusFile", on_delete=models.PROTECT, related_name="syllabi" + ) + uploaded_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="uploaded_syllabi", + ) + status = models.CharField( + max_length=16, choices=STATUS_CHOICES, default=Status.PENDING, db_index=True + ) + summary_md = models.TextField(blank=True, default="") + verdict = models.JSONField(null=True, blank=True) + comparison = models.JSONField(null=True, blank=True) + is_primary = models.BooleanField(default=False) + error_message = models.TextField(blank=True, default="") + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["course", "instructor", "file"], + name="unique_course_instructor_syllabus_file", + ) + ] + indexes = [models.Index(fields=["course", "instructor", "status"])] + ordering = ["-created_at"] + + def __str__(self): + return f"{self.course_id} / {self.instructor_id} / {self.status}" + + +@receiver(post_delete, sender=Syllabus) +def recycle_orphan_syllabus_file(sender, instance, **kwargs): + """After a Syllabus row is deleted, move its now-unreferenced file to recycle. + + Files are deduped by sha256 and shared across syllabi; a file is moved + only when this was the last reference. Imported lazily to avoid a cycle. + """ + from apps.web.syllabus_files import recycle_file_if_unreferenced + + recycle_file_if_unreferenced(instance.file) diff --git a/apps/web/models/syllabus_file.py b/apps/web/models/syllabus_file.py new file mode 100644 index 0000000..47b7dc8 --- /dev/null +++ b/apps/web/models/syllabus_file.py @@ -0,0 +1,24 @@ +from __future__ import unicode_literals + +from django.db import models + + +class SyllabusFile(models.Model): + """One uploaded syllabus document, deduplicated by content sha256. + + The same file (identical bytes) uploaded for different courses or + instructors shares a single SyllabusFile row; `extracted_text` is cached + here so re-analysis never re-extracts or re-OCRs. + """ + + file = models.FileField(upload_to="syllabi/") + sha256 = models.CharField(max_length=64, unique=True, db_index=True) + content_type = models.CharField(max_length=100, blank=True, default="") + original_filename = models.CharField(max_length=255, blank=True, default="") + size = models.PositiveBigIntegerField(default=0) + extracted_text = models.TextField(blank=True, default="") + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.sha256[:16] diff --git a/apps/web/serializers.py b/apps/web/serializers.py index b4b243c..40d52ec 100644 --- a/apps/web/serializers.py +++ b/apps/web/serializers.py @@ -9,9 +9,12 @@ DistributiveRequirement, Instructor, Review, + Syllabus, + SyllabusFile, Vote, ) from lib import constants +from lib.name_normalization import canonicalize_professor from lib.terms import is_valid_term, normalize_term @@ -29,6 +32,74 @@ class Meta: fields = ("term", "section", "period", "limit", "instructors") +class InstructorSerializer(serializers.ModelSerializer): + class Meta: + model = Instructor + fields = ("id", "name") + + +class SyllabusFileSerializer(serializers.ModelSerializer): + class Meta: + model = SyllabusFile + fields = ("id", "original_filename", "size", "content_type") + + +class SyllabusSerializer(serializers.ModelSerializer): + instructor = InstructorSerializer(read_only=True) + file = SyllabusFileSerializer(read_only=True) + uploaded_by = serializers.StringRelatedField(read_only=True) + + class Meta: + model = Syllabus + fields = ( + "id", + "course", + "instructor", + "file", + "uploaded_by", + "status", + "summary_md", + "verdict", + "comparison", + "is_primary", + "error_message", + "created_at", + "updated_at", + ) + + +class SyllabusCreateSerializer(serializers.Serializer): + file = serializers.FileField() + instructor = serializers.IntegerField() + + def validate_file(self, value): + allowed = settings.SYLLABUS["ALLOWED_EXTENSIONS"] + name = value.name.lower() + if not any(name.endswith(ext) for ext in allowed): + raise serializers.ValidationError( + f"Unsupported file type. Allowed: {', '.join(allowed)}" + ) + if value.size > settings.SYLLABUS["MAX_UPLOAD_SIZE"]: + raise serializers.ValidationError("File exceeds the 20 MB upload limit") + return value + + def validate_instructor(self, value): + course = self.context["course"] + teaches = Instructor.objects.filter( + courseoffering__course=course, pk=value + ).exists() + if not teaches: + raise serializers.ValidationError("Instructor does not teach this course") + return value + + +class SyllabusAdminUpdateSerializer(serializers.ModelSerializer): + class Meta: + model = Syllabus + fields = ("summary_md", "verdict", "comparison", "is_primary") + read_only_fields = ("comparison",) + + class ReviewSerializer(serializers.ModelSerializer): # user = serializers.StringRelatedField() term = serializers.CharField() @@ -83,15 +154,32 @@ def validate_term(self, value): ) def validate_professor(self, value): - """Validate professor name format""" - names = value.split(" ") + """Validate and canonicalize professor name. + + The name is matched against the course's canonical instructors + (from course offerings): reversed, misspelled, or annotated variants + are corrected to the canonical spelling. Only when nothing matches is + the submitted name kept as its own professor. + """ + course = self.context.get("course") + if course is None and self.instance is not None: + course = self.instance.course + + candidate_names = [] + if course is not None: + candidate_names = ( + Instructor.objects.filter(courseoffering__course=course) + .values_list("name", flat=True) + .distinct() + ) + normalized = canonicalize_professor(value, candidate_names) - if len(names) < 2: + if len(normalized.split()) < 2: raise serializers.ValidationError( "Please use a valid professor name, e.g. John Smith" ) - return " ".join([n.capitalize() for n in names]) + return normalized def validate_comments(self, value): """Validate review minimum length""" @@ -334,9 +422,13 @@ def get_can_write_review(self, obj): return False def get_instructors(self, obj): - """Return a list of instructor names for the course""" - instructors = obj.get_instructors() - return [instructor.name for instructor in instructors] + """Return instructor {id, name} pairs so clients can key uploads on them. + + Term-agnostic: a syllabus can exist for any instructor who ever taught + the course, and uploads should not be blocked by term bookkeeping. + """ + instructors = obj.get_instructors(term=None) + return [{"id": i.id, "name": i.name} for i in instructors] def get_course_topics(self, obj): return obj.course_topics diff --git a/apps/web/syllabus_analysis.py b/apps/web/syllabus_analysis.py new file mode 100644 index 0000000..b09cf9f --- /dev/null +++ b/apps/web/syllabus_analysis.py @@ -0,0 +1,258 @@ +"""Syllabus text extraction and local Ollama analysis helpers. + +Pure functions (settings-only dependencies) so they can be unit-tested and +reused from the Celery task in apps/web/tasks.py. +""" + +from __future__ import annotations + +import base64 +import io +import json +import logging +import re + +import httpx +from django.conf import settings + +logger = logging.getLogger(__name__) + +# Below this many characters a PDF is treated as a scan and OCR'd via vision. +TEXT_OCR_THRESHOLD = 300 +# Prompts are capped so a long syllabus never blows the context window. +MAX_SYLLABUS_CHARS = 60_000 +MAX_SUMMARY_CHARS = 2_000 + + +class SyllabusAnalysisError(Exception): + """Raised when extraction, OCR or model analysis fails.""" + + +def extract_pdf_text(pdf_bytes: bytes) -> str: + """Extract embedded text from a PDF; '' for scanned/image-only PDFs.""" + from pypdf import PdfReader + + reader = PdfReader(io.BytesIO(pdf_bytes)) + return "\n".join( + (page.extract_text() or "") for page in reader.pages if page.extract_text() + ) + + +def extract_docx_text(docx_bytes: bytes) -> str: + """Extract paragraphs + table cells from a .docx file.""" + import docx + + document = docx.Document(io.BytesIO(docx_bytes)) + parts = [p.text for p in document.paragraphs] + for table in document.tables: + for row in table.rows: + parts.append(" | ".join(cell.text for cell in row.cells)) + return "\n".join(parts) + + +def render_pdf_pages(pdf_bytes: bytes, max_pages: int | None = None) -> list[bytes]: + """Render PDF pages to PNG bytes for vision OCR (no system deps).""" + import pypdfium2 as pdfium + + max_pages = max_pages or settings.OLLAMA["MAX_PAGES"] + pdf = pdfium.PdfDocument(pdf_bytes) + try: + n_pages = min(len(pdf), max_pages) + images: list[bytes] = [] + for i in range(n_pages): + page = pdf[i] + bitmap = page.render(scale=150 / 72) # ~150 dpi + pil_image = bitmap.to_pil() + buffer = io.BytesIO() + pil_image.save(buffer, format="PNG") + images.append(buffer.getvalue()) + pil_image.close() + return images + finally: + pdf.close() + + +def _image_message(page_png: bytes, page_no: int) -> dict: + encoded = base64.b64encode(page_png).decode("ascii") + return { + "type": "text", + "text": f"Page {page_no}:", + }, { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{encoded}"}, + } + + +def ocr_pages(images: list[bytes]) -> str: + """OCR rendered pages with the local vision model.""" + if not images: + return "" + parts: list[dict] = [ + {"type": "text", "text": "Transcribe all text in each page exactly, in order."} + ] + for i, page_png in enumerate(images, start=1): + parts.extend(_image_message(page_png, i)) + response = ollama_chat( + [{"role": "user", "content": parts}], + format_json=False, + ) + return (response.get("message") or {}).get("content", "").strip() + + +def ollama_chat(messages: list[dict], format_json: bool = False) -> dict: + """POST /api/chat to the local Ollama; returns the parsed response dict. + + Retries transient connection/read-timeout failures with backoff. + """ + ollama = settings.OLLAMA + payload: dict = { + "model": ollama["MODEL"], + "messages": messages, + "stream": False, + "options": { + "num_ctx": ollama["NUM_CTX"], + "temperature": 0.2, + # qwen3 thinking models: with JSON format enabled the final answer + # lands in `message.thinking` and `content` comes back empty. + "think": False, + }, + } + if format_json: + payload["format"] = "json" + url = f"{ollama['BASE_URL'].rstrip('/')}/api/chat" + timeout = ollama["TIMEOUT"] + last_error: Exception | None = None + for attempt, backoff in enumerate((0, 5, 15)): + if backoff: + import time + + time.sleep(backoff) + try: + with httpx.Client(timeout=timeout) as client: + response = client.post(url, json=payload) + response.raise_for_status() + data = response.json() + if not (data.get("message") or {}).get("content"): + logger.warning( + "Ollama returned empty content: done=%s eval=%d ctx=%d model=%s", + data.get("done_reason"), + data.get("eval_count"), + data.get("prompt_eval_count"), + payload["model"], + ) + return data + except (httpx.ConnectError, httpx.ReadTimeout) as exc: + last_error = exc + logger.warning("Ollama unreachable (attempt %d/3): %s", attempt + 1, exc) + except httpx.HTTPStatusError as exc: + raise SyllabusAnalysisError( + f"Ollama returned HTTP {exc.response.status_code}" + ) from exc + raise SyllabusAnalysisError(f"Ollama unreachable: {last_error}") + + +def parse_json_response(content: str) -> dict: + """Parse model output to dict: strip fences, strict parse, regex fallback.""" + if not content: + raise SyllabusAnalysisError("Empty model response") + text = content.strip() + text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.MULTILINE) + try: + parsed = json.loads(text) + if isinstance(parsed, dict): + return parsed + except json.JSONDecodeError: + # Not clean JSON yet; fall through to fence-stripping / regex below. + pass + match = re.search(r"\{.*\}", text, flags=re.DOTALL) + if match: + try: + return json.loads(match.group(0)) + except json.JSONDecodeError: + # Regex extraction failed too; the final raise reports the failure. + pass + raise SyllabusAnalysisError("Model output was not parseable JSON") + + +def _truncate(text: str, limit: int) -> str: + return text if len(text) <= limit else text[:limit] + "\n...[truncated]" + + +def build_analysis_prompt(course, instructor, syllabus_text: str) -> str: + """Prompt asking the model to check the syllabus against known course info.""" + topics = course.course_topics or [] + topic_block = ( + "\n".join(f"- {topic}" for topic in topics) + if topics + else "(no known topics on file)" + ) + return f"""You are verifying a course syllabus for authenticity against known course data. + +Course code: {course.course_code} +Course title: {course.course_title} +Department: {course.department} +Known description: {course.description or "(none)"} +Known topics on file: +{topic_block} +Instructor (claimed): {instructor.name} + +Syllabus text to analyze: +--- +{syllabus_text} +--- + +Task: +1. Judge whether this syllabus plausibly belongs to this course and instructor (match). +2. Judge whether it looks like a real, legitimate syllabus (not fabricated, clearly + copied from another course, or garbled). +3. Produce a concise markdown summary of the syllabus: structure, grading scheme, + schedule highlights, textbooks, policies. + +Reply with JSON only, shape: +{{"match_score": , "matches_course_content": , "is_legitimate": , +"flags": [], "summary_md": ""}} +""" + + +def build_comparison_prompt(course, instructor, new_text: str, old_text: str) -> str: + return f"""Two syllabus versions exist for {course.course_code} - {course.course_title} +taught by {instructor.name}. Decide which one is newer, matches the course better, +and is more authentic/complete. Existing (primary) version first, new upload second. + +Existing version: +--- +{old_text} +--- + +New version: +--- +{new_text} +--- + +Reply with JSON only, shape: +{{"newer": , "better_match": , "more_authentic": , +"recommendation": "keep_old"|"keep_new", "notes": ""}} +""" + + +def analyze(course, instructor, syllabus_text: str) -> dict: + """Run the analysis prompt and return the structured verdict dict.""" + prompt = build_analysis_prompt( + course, instructor, _truncate(syllabus_text, MAX_SYLLABUS_CHARS) + ) + response = ollama_chat([{"role": "user", "content": prompt}], format_json=True) + verdict = parse_json_response((response.get("message") or {}).get("content") or "") + # summary_md is nested inside the verdict JSON by the model. + return verdict + + +def compare(course, instructor, new_text: str, old_text: str) -> dict: + """Run the comparison prompt; returns the comparison dict.""" + prompt = build_comparison_prompt( + course, + instructor, + _truncate(new_text, MAX_SYLLABUS_CHARS), + _truncate(old_text, MAX_SYLLABUS_CHARS), + ) + response = ollama_chat([{"role": "user", "content": prompt}], format_json=True) + return parse_json_response((response.get("message") or {}).get("content") or "") diff --git a/apps/web/syllabus_files.py b/apps/web/syllabus_files.py new file mode 100644 index 0000000..6564c45 --- /dev/null +++ b/apps/web/syllabus_files.py @@ -0,0 +1,55 @@ +"""Filesystem helpers for syllabus files: recycle (soft-delete) storage. + +Files live in MEDIA_ROOT/syllabi/. and are shared by +Syllabus rows via sha256 dedup. When the last referencing syllabus is +rejected or deleted the file moves to MEDIA_ROOT/recycle/ instead of +being destroyed, so an admin can audit and restore it. +""" + +from __future__ import unicode_literals + +import os +import shutil +from datetime import datetime, timezone + +from django.core.files.storage import default_storage + +RECYCLE_DIR = "recycle" + + +def recycle_file_if_unreferenced(file_obj, exclude_syllabus=None): + """Move ``file_obj`` (a SyllabusFile row) to the recycle dir if orphaned. + + A file counts as referenced when any Syllabus other than + ``exclude_syllabus`` still points at it; callers pass the syllabus being + rejected/deleted so its own reference does not block the move. Returns + the new relative name, or None when nothing was moved. + """ + referencing = file_obj.syllabi.all() + if exclude_syllabus is not None: + referencing = referencing.exclude(pk=exclude_syllabus.pk) + if referencing.exists(): + return None + + storage = default_storage + current_name = file_obj.file.name + if not current_name or current_name.startswith(f"{RECYCLE_DIR}/"): + return None + + base = current_name.rsplit("/", 1)[-1] + dest_name = f"{RECYCLE_DIR}/{base}" + if storage.exists(dest_name): + stamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S") + dest_name = f"{RECYCLE_DIR}/{stamp}_{base}" + + if hasattr(storage, "path"): + dest_dir = os.path.dirname(storage.path(dest_name)) + os.makedirs(dest_dir, exist_ok=True) + shutil.move(storage.path(current_name), storage.path(dest_name)) + else: + storage.save(dest_name, file_obj.file) # remote fallback + storage.delete(current_name) + + file_obj.file.name = dest_name + file_obj.save(update_fields=["file", "updated_at"]) + return dest_name diff --git a/apps/web/tasks.py b/apps/web/tasks.py new file mode 100644 index 0000000..efa7275 --- /dev/null +++ b/apps/web/tasks.py @@ -0,0 +1,135 @@ +import logging + +from celery import shared_task +from django.conf import settings +from django.db import transaction + +from apps.web.models import Syllabus, SyllabusFile +from apps.web.syllabus_files import recycle_file_if_unreferenced +from apps.web.syllabus_analysis import ( + TEXT_OCR_THRESHOLD, + SyllabusAnalysisError, + analyze, + compare, + extract_docx_text, + extract_pdf_text, + ocr_pages, + render_pdf_pages, +) + +logger = logging.getLogger(__name__) + + +@shared_task +def process_syllabus(syllabus_id): + """Extract text, analyze against course data, and resolve the primary copy. + + Uploads whose verdict match_score is below the configured threshold are + rejected (status ``rejected``, kept for audit, hidden from course pages) + and their file moved to the recycle dir when no other syllabus shares it. + Comparison only decides which of the duplicates is marked primary. + + Failure bookkeeping lives outside the atomic block: a raise inside it + would roll the FAILED status back with the rest of the work. + """ + try: + _analyze_and_resolve(syllabus_id) + except Exception as exc: # noqa: BLE001 - any failure marks the syllabus failed + logger.exception("Syllabus analysis failed for syllabus %s", syllabus_id) + Syllabus.objects.filter(pk=syllabus_id).update( + status=Syllabus.Status.FAILED, error_message=str(exc)[:1000] + ) + raise + + +@transaction.atomic +def _analyze_and_resolve(syllabus_id): + syllabus = ( + Syllabus.objects.select_for_update() + .select_related("file", "course", "instructor") + .get(pk=syllabus_id) + ) + syllabus.status = Syllabus.Status.PROCESSING + syllabus.save(update_fields=["status", "updated_at"]) + + text = _extract_or_ocr(syllabus.file) + siblings = list( + Syllabus.objects.filter( + course=syllabus.course, + instructor=syllabus.instructor, + status=Syllabus.Status.ANALYZED, + ) + .exclude(pk=syllabus.pk) + .order_by("-created_at") + ) + verdict = analyze(syllabus.course, syllabus.instructor, text) + summary_md = str(verdict.get("summary_md", "")).strip() + if not summary_md: + raise SyllabusAnalysisError("Model returned no summary") + + # Reject uploads that clearly do not belong to this course: keep the row + # for audit (admin can still see and restore it) but hide it from the + # course page, and recycle the file once nothing references it. + match_score = verdict.get("match_score") + if isinstance(match_score, (int, float)) and ( + match_score < settings.SYLLABUS["MIN_MATCH_SCORE"] + ): + syllabus.summary_md = summary_md + syllabus.verdict = verdict + syllabus.status = Syllabus.Status.REJECTED + syllabus.error_message = "" + syllabus.save() + recycle_file_if_unreferenced(syllabus.file, exclude_syllabus=syllabus) + return + + syllabus.summary_md = summary_md + syllabus.verdict = verdict + syllabus.status = Syllabus.Status.ANALYZED + syllabus.error_message = "" + syllabus.save() + + if not siblings: + syllabus.is_primary = True + syllabus.save(update_fields=["is_primary", "updated_at"]) + return + + # Compare against the current primary sibling (if still one). + primary_sibling = next((s for s in siblings if s.is_primary), siblings[0]) + comparison = compare( + syllabus.course, + syllabus.instructor, + text, + primary_sibling.file.extracted_text, + ) + syllabus.comparison = comparison + syllabus.save(update_fields=["comparison", "updated_at"]) + if comparison.get("recommendation") == "keep_new": + primary_sibling.is_primary = False + primary_sibling.save(update_fields=["is_primary", "updated_at"]) + syllabus.is_primary = True + syllabus.save(update_fields=["is_primary", "updated_at"]) + + +def _extract_or_ocr(file_obj: SyllabusFile) -> str: + """Return extracted text, using the cached copy when present.""" + if file_obj.extracted_text: + return file_obj.extracted_text + with file_obj.file.open("rb") as handle: + content = handle.read() + name = file_obj.file.name.lower() + if name.endswith(".pdf"): + text = extract_pdf_text(content) + if len(text.strip()) < TEXT_OCR_THRESHOLD: + logger.info("PDF %s looks scanned; OCR via vision model", file_obj.pk) + images = render_pdf_pages(content) + text = ocr_pages(images) + elif name.endswith(".docx"): + text = extract_docx_text(content) + else: + raise SyllabusAnalysisError(f"Unsupported syllabus file type: {name}") + + if len(text.strip()) < 10: + raise SyllabusAnalysisError("Could not extract any readable text from the file") + file_obj.extracted_text = text + file_obj.save(update_fields=["extracted_text", "updated_at"]) + return text diff --git a/apps/web/tests/conftest.py b/apps/web/tests/conftest.py index bc55dae..60adb82 100644 --- a/apps/web/tests/conftest.py +++ b/apps/web/tests/conftest.py @@ -28,6 +28,40 @@ def auth_client(user, base_client): return base_client +@pytest.fixture +def staff_user(db): + """Returns a saved staff user instance.""" + return factories.UserFactory(is_staff=True) + + +@pytest.fixture +def staff_client(staff_user): + """Returns its own API client authenticated as a staff user.""" + client = APIClient() + client.force_authenticate(user=staff_user) + return client + + +@pytest.fixture +def eager_media(tmp_path, settings): + """Scratch media root + inline (eager) Celery tasks for syllabus tests. + + website/celery.py snapshots Django settings at import time, so + settings-only overrides race with module import order. Push the flags + straight onto the app config too, then restore them after each test. + """ + from website.celery import app as celery_app + + conf = celery_app.conf + previous = (conf.get("task_always_eager"), conf.get("task_eager_propagates")) + settings.MEDIA_ROOT = tmp_path / "media" + settings.CELERY_TASK_ALWAYS_EAGER = True + settings.CELERY_TASK_EAGER_PROPAGATES = True + conf.update(task_always_eager=True, task_eager_propagates=True) + yield tmp_path / "media" + conf.update(task_always_eager=previous[0], task_eager_propagates=previous[1]) + + # ------------------------------------------------------------------------- # 2. Data Fixtures (Models) # ------------------------------------------------------------------------- diff --git a/apps/web/tests/factories.py b/apps/web/tests/factories.py index 53c32ee..e6fcadd 100644 --- a/apps/web/tests/factories.py +++ b/apps/web/tests/factories.py @@ -1,12 +1,17 @@ +import hashlib + import factory import factory.fuzzy from django.contrib.auth.models import User # Import models from their individual files from apps.web.models.course import Course +from apps.web.models.course_offering import CourseOffering +from apps.web.models.instructor import Instructor from apps.web.models.review import Review from apps.web.models.student import Student -from apps.web.models.course_offering import CourseOffering +from apps.web.models.syllabus import Syllabus +from apps.web.models.syllabus_file import SyllabusFile class UserFactory(factory.django.DjangoModelFactory): @@ -77,3 +82,49 @@ class Meta: model = "web.DistributiveRequirement" name = factory.Sequence(lambda n: f"Dist{n}") + + +class InstructorFactory(factory.django.DjangoModelFactory): + class Meta: + model = Instructor + + name = factory.Sequence(lambda n: f"Prof{n} Name{n}") + + +def syllabus_pdf_bytes(seed: str = "syllabus") -> bytes: + """Minimal valid-enough PDF bytes; distinct seed => distinct sha256.""" + return ( + f"%PDF-1.4\n1 0 obj<>endobj\ntrailer<>\n%%EOF" + ).encode() + + +class SyllabusFileFactory(factory.django.DjangoModelFactory): + class Meta: + model = SyllabusFile + + content_type = "application/pdf" + original_filename = factory.Sequence(lambda n: f"syllabus-{n}.pdf") + # Unique per created file (unique sha256 constraint). Content hash + # consistency is the upload view's job, not the factory's. + sha256 = factory.Sequence( + lambda n: hashlib.sha256(f"factory-seed-{n}".encode()).hexdigest() + ) + size = factory.LazyFunction(lambda: len(syllabus_pdf_bytes())) + file = factory.django.FileField(data=syllabus_pdf_bytes(), filename="syllabus.pdf") + + +class SyllabusFactory(factory.django.DjangoModelFactory): + class Meta: + model = Syllabus + + course = factory.SubFactory(CourseFactory) + instructor = factory.SubFactory(InstructorFactory) + file = factory.SubFactory(SyllabusFileFactory) + status = Syllabus.Status.ANALYZED + summary_md = "# Syllabus\n\nGrading: 40% homework, 60% final." + verdict = { + "match_score": 90, + "matches_course_content": True, + "is_legitimate": True, + "flags": [], + } diff --git a/apps/web/tests/test_admin_legacy_review_import.py b/apps/web/tests/test_admin_legacy_review_import.py index 3ab4dc0..768d8e5 100644 --- a/apps/web/tests/test_admin_legacy_review_import.py +++ b/apps/web/tests/test_admin_legacy_review_import.py @@ -10,7 +10,9 @@ from apps.web.models import Course, Review -def review_csv(*, course_code="TEST1000J", professor="Professor", term="", comment="Comment"): +def review_csv( + *, course_code="TEST1000J", professor="Professor", term="", comment="Comment" +): content = StringIO() writer = csv.DictWriter( content, fieldnames=["course_code", "professor", "term", "comment"] @@ -25,7 +27,9 @@ def review_csv(*, course_code="TEST1000J", professor="Professor", term="", comme } ) return SimpleUploadedFile( - "legacy_reviews.csv", content.getvalue().encode("utf-8"), content_type="text/csv" + "legacy_reviews.csv", + content.getvalue().encode("utf-8"), + content_type="text/csv", ) @@ -52,9 +56,7 @@ def test_only_superusers_can_access_legacy_review_import(client, import_url): @pytest.mark.django_db -def test_admin_preview_then_confirm_imports_uploaded_csv( - client, superuser, import_url -): +def test_admin_preview_then_confirm_imports_uploaded_csv(client, superuser, import_url): Course.objects.create(course_code="TEST1000J") client.force_login(superuser) diff --git a/apps/web/tests/test_professor_normalization.py b/apps/web/tests/test_professor_normalization.py new file mode 100644 index 0000000..bde69c6 --- /dev/null +++ b/apps/web/tests/test_professor_normalization.py @@ -0,0 +1,157 @@ +"""Professor-name canonicalization on review submission. + +New reviews must attribute the professor using the course's canonical +Instructor names (from course offerings): reversed / misspelled / annotated +variants are corrected automatically; only names that match nothing are kept +as their own professor. +""" + +import pytest +from rest_framework import status + +from apps.web.models import Review +from apps.web.tests import factories +from lib.name_normalization import canonicalize_professor + +# --------------------------------------------------------------------------- +# Pure lib tests +# --------------------------------------------------------------------------- + + +def test_canonicalize_matches_reversed_and_misspelled(): + assert canonicalize_professor("Lin Zibo", ["Zibo Lin"]) == "Zibo Lin" + assert canonicalize_professor("lin zibo", ["Zibo Lin"]) == "Zibo Lin" + assert ( + canonicalize_professor("Manuel Charlamagne", ["Manuel Charlemagne"]) + == "Manuel Charlemagne" + ) + assert ( + canonicalize_professor("Aline Chevelier", ["Aline Chevalier"]) + == "Aline Chevalier" + ) + assert canonicalize_professor("Kwee-yan Teh", ["Kwee-Yan Teh"]) == "Kwee-Yan Teh" + assert canonicalize_professor("Horst Hohberger", ["Horst Harold Hohberger"]) == ( + "Horst Harold Hohberger" + ) + assert ( + canonicalize_professor("Nick Welchbolen", ["Nicholas Scott Welch-Bolen"]) + == "Nicholas Scott Welch-Bolen" + ) + + +def test_canonicalize_unmatched_names_are_kept(): + assert canonicalize_professor("John Smith", ["Zibo Lin"]) == "John Smith" + assert canonicalize_professor("olga danilkina", []) == "Olga Danilkina" + # Title + single name is not a valid new name; preserved verbatim. + assert canonicalize_professor("Dr. Testing", []) == "Dr. Testing" + assert canonicalize_professor("", ["Zibo Lin"]) == "" + + +def test_canonicalize_does_not_force_unverifiable_typos(): + # Jayhang/Jaehyung is a 4-edit drift; without other evidence it must stay. + assert canonicalize_professor("Jayhang Ju", ["Jaehyung Ju"]) == "Jayhang Ju" + + +# --------------------------------------------------------------------------- +# API tests: professor corrected to the course's instructor name +# --------------------------------------------------------------------------- + + +@pytest.fixture +def course_with_instructor(course): + offering = factories.CourseOfferingFactory(course=course, term="26SU") + offering.instructors.add(factories.InstructorFactory(name="Zibo Lin")) + return course, offering.instructors.get() + + +@pytest.mark.django_db +def test_create_review_corrects_reversed_professor( + auth_client, course_reviews_url, min_len, course_with_instructor +): + course, _ = course_with_instructor + response = auth_client.post( + course_reviews_url, + { + "term": "26SU", + "professor": "Lin Zibo", + "comments": "c" * min_len, + }, + format="json", + ) + assert response.status_code == status.HTTP_201_CREATED + assert Review.objects.get(course=course).professor == "Zibo Lin" + + +@pytest.mark.django_db +def test_create_review_corrects_misspelled_professor( + auth_client, course_reviews_url, course, min_len +): + offering = factories.CourseOfferingFactory(course=course, term="26SU") + offering.instructors.add(factories.InstructorFactory(name="Manuel Charlemagne")) + response = auth_client.post( + course_reviews_url, + { + "term": "26SU", + "professor": "Manuel Charlamagne", + "comments": "c" * min_len, + }, + format="json", + ) + assert response.status_code == status.HTTP_201_CREATED + assert Review.objects.get(course=course).professor == "Manuel Charlemagne" + + +@pytest.mark.django_db +def test_create_review_keeps_unmatched_professor( + auth_client, course_reviews_url, course, min_len +): + offering = factories.CourseOfferingFactory(course=course, term="26SU") + offering.instructors.add(factories.InstructorFactory(name="Zibo Lin")) + response = auth_client.post( + course_reviews_url, + { + "term": "26SU", + "professor": "John Smith", + "comments": "c" * min_len, + }, + format="json", + ) + assert response.status_code == status.HTTP_201_CREATED + assert Review.objects.get(course=course).professor == "John Smith" + + +@pytest.mark.django_db +def test_update_review_corrects_professor_via_instance_course( + auth_client, course, user, min_len +): + """PUT has no course in context; it must fall back to the review's course.""" + from django.urls import reverse + + offering = factories.CourseOfferingFactory(course=course, term="26SU") + offering.instructors.add(factories.InstructorFactory(name="Zibo Lin")) + review = factories.ReviewFactory( + course=course, user=user, professor="Zibo Lin", comments="c" * min_len + ) + url = reverse("user_review_api", kwargs={"review_id": review.id}) + response = auth_client.put( + url, + {"term": "26SU", "professor": "Lin Zibo", "comments": "c" * min_len}, + format="json", + ) + assert response.status_code == status.HTTP_200_OK + review.refresh_from_db() + assert review.professor == "Zibo Lin" + + +@pytest.mark.django_db +def test_single_word_professor_still_rejected( + auth_client, course_reviews_url, course, min_len +): + offering = factories.CourseOfferingFactory(course=course, term="26SU") + offering.instructors.add(factories.InstructorFactory(name="Zibo Lin")) + response = auth_client.post( + course_reviews_url, + {"term": "26SU", "professor": "Zibo", "comments": "c" * min_len}, + format="json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST diff --git a/apps/web/tests/test_syllabus.py b/apps/web/tests/test_syllabus.py new file mode 100644 index 0000000..109ef04 --- /dev/null +++ b/apps/web/tests/test_syllabus.py @@ -0,0 +1,533 @@ +import json + +import pytest +from django.core.files.uploadedfile import SimpleUploadedFile +from django.urls import reverse +from rest_framework.test import APIClient + +from apps.web.models import Syllabus, SyllabusFile +from apps.web.tests.factories import ( + CourseOfferingFactory, + InstructorFactory, + SyllabusFactory, + SyllabusFileFactory, +) + +pytestmark = pytest.mark.django_db + + +@pytest.fixture(autouse=True) +def _scratch_media(eager_media): + """Every test here runs against a scratch MEDIA_ROOT with eager Celery.""" + return eager_media + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _offering_with_instructor(course, instructor=None): + offering = CourseOfferingFactory(course=course) + instructor = instructor or InstructorFactory() + offering.instructors.add(instructor) + return instructor + + +def _make_pdf_upload(content=b"%PDF fake syllabus", name="syllabus.pdf"): + return SimpleUploadedFile(name, content, content_type="application/pdf") + + +@pytest.fixture +def syllabus_urls(course): + return { + "list": reverse("course_syllabi_api", kwargs={"course_id": course.id}), + "detail": lambda sid: reverse( + "syllabus_detail_api", kwargs={"syllabus_id": sid} + ), + "download": lambda sid: reverse( + "syllabus_download_api", kwargs={"syllabus_id": sid} + ), + } + + +@pytest.fixture +def fake_ollama(monkeypatch): + """Runs analysis inline with a canned verdict; comparison switches on prompt. + + Tests can mutate ``chat.state["match_score"]`` (etc.) to steer the verdict. + """ + + def _chat(messages, format_json=False): + prompt = messages[0]["content"] + if isinstance(prompt, list): + prompt = "\n".join(p.get("text", "") for p in prompt) + if "Two syllabus versions exist" in prompt: + result = { + "newer": True, + "better_match": True, + "more_authentic": True, + "recommendation": "keep_new", + "notes": "mock", + } + else: + result = { + "match_score": _chat.state["match_score"], + "matches_course_content": True, + "is_legitimate": True, + "flags": [], + "summary_md": "## Summary\n\nMock grading summary.", + } + # Ollama /api/chat nests content under message.content. + return {"message": {"content": json.dumps(result)}} + + _chat.state = {"match_score": 88} + monkeypatch.setattr("apps.web.syllabus_analysis.ollama_chat", _chat) + return _chat + + +@pytest.fixture +def fake_extraction(monkeypatch): + """Skips real PDF parsing in the eager task.""" + + def _extract(file_obj): + return "This course covers calculus and linear algebra topics." + + monkeypatch.setattr("apps.web.tasks._extract_or_ocr", _extract) + + +# --------------------------------------------------------------------------- +# Upload endpoint +# --------------------------------------------------------------------------- + + +class TestUpload: + def test_upload_requires_auth(self, base_client, syllabus_urls): + response = base_client.post(syllabus_urls["list"], {}, format="multipart") + assert response.status_code in (401, 403) + + def test_unknown_course_404(self, auth_client, course, db): + url = reverse("course_syllabi_api", kwargs={"course_id": 999999}) + response = auth_client.post(url, {}, format="multipart") + assert response.status_code == 404 + + def test_rejects_unsupported_extension(self, auth_client, course, syllabus_urls): + instructor = _offering_with_instructor(course) + upload = _make_pdf_upload(name="syllabus.exe") + response = auth_client.post( + syllabus_urls["list"], + {"file": upload, "instructor": instructor.id}, + format="multipart", + ) + assert response.status_code == 400 + assert "file" in response.data + + def test_rejects_oversized_file(self, auth_client, course, syllabus_urls): + instructor = _offering_with_instructor(course) + upload = SimpleUploadedFile( + "big.pdf", b"x" * 21 * 1024 * 1024, content_type="application/pdf" + ) + response = auth_client.post( + syllabus_urls["list"], + {"file": upload, "instructor": instructor.id}, + format="multipart", + ) + assert response.status_code == 400 + + def test_rejects_instructor_not_teaching_course( + self, auth_client, course, syllabus_urls + ): + stranger = InstructorFactory() + upload = _make_pdf_upload() + response = auth_client.post( + syllabus_urls["list"], + {"file": upload, "instructor": stranger.id}, + format="multipart", + ) + assert response.status_code == 400 + + def test_successful_upload_analyzes_and_stores_by_hash( + self, + auth_client, + course, + user, + syllabus_urls, + fake_ollama, + fake_extraction, + ): + instructor = _offering_with_instructor(course) + upload = _make_pdf_upload() + response = auth_client.post( + syllabus_urls["list"], + {"file": upload, "instructor": instructor.id}, + format="multipart", + ) + assert response.status_code == 201 + data = response.data + assert data["status"] == Syllabus.Status.ANALYZED # eager task ran + assert data["is_primary"] is True + assert data["uploaded_by"] == user.username + assert "Summary" in data["summary_md"] + assert data["verdict"]["match_score"] == 88 + + stored = SyllabusFile.objects.get(pk=data["file"]["id"]) + assert stored is not None + assert len(stored.sha256) == 64 # sha256 of the uploaded bytes + + # Stored under the .pdf name and scoped to the test media root + assert stored.file.name.startswith("syllabi/") + assert stored.file.name.endswith(".pdf") + + def test_same_bytes_share_file_different_syllabus_rows( + self, auth_client, course, syllabus_urls, fake_ollama, fake_extraction + ): + instructor_a = _offering_with_instructor(course) + instructor_b = InstructorFactory() + offering_b = CourseOfferingFactory(course=course) + offering_b.instructors.add(instructor_b) + + # Fresh upload object per POST — a consumed SimpleUploadedFile can't + # be re-encoded by the multipart client. + first = auth_client.post( + syllabus_urls["list"], + {"file": _make_pdf_upload(), "instructor": instructor_a.id}, + format="multipart", + ) + second = auth_client.post( + syllabus_urls["list"], + {"file": _make_pdf_upload(), "instructor": instructor_b.id}, + format="multipart", + ) + assert first.status_code == second.status_code == 201 + assert first.data["file"]["id"] == second.data["file"]["id"] + assert first.data["id"] != second.data["id"] + assert SyllabusFile.objects.count() == 1 + assert Syllabus.objects.count() == 2 + + def test_identical_reupload_is_idempotent( + self, auth_client, course, syllabus_urls, fake_ollama, fake_extraction + ): + instructor = _offering_with_instructor(course) + first = auth_client.post( + syllabus_urls["list"], + {"file": _make_pdf_upload(), "instructor": instructor.id}, + format="multipart", + ) + second = auth_client.post( + syllabus_urls["list"], + {"file": _make_pdf_upload(), "instructor": instructor.id}, + format="multipart", + ) + assert first.status_code == 201 + assert second.status_code == 200 + assert first.data["id"] == second.data["id"] + assert Syllabus.objects.count() == 1 + + +# --------------------------------------------------------------------------- +# List / detail / download +# --------------------------------------------------------------------------- + + +class TestRead: + def test_list_is_public(self, base_client, syllabus_urls): + response = base_client.get(syllabus_urls["list"]) + assert response.status_code == 200 + + def test_second_version_triggers_comparison_and_primary_handover( + self, auth_client, course, syllabus_urls, fake_ollama, fake_extraction + ): + instructor = _offering_with_instructor(course) + first = auth_client.post( + syllabus_urls["list"], + { + "file": _make_pdf_upload(content=b"%PDF v1", name="v1.pdf"), + "instructor": instructor.id, + }, + format="multipart", + ) + assert first.data["is_primary"] is True + + second = auth_client.post( + syllabus_urls["list"], + { + "file": _make_pdf_upload(content=b"%PDF v2 longer", name="v2.pdf"), + "instructor": instructor.id, + }, + format="multipart", + ) + assert second.status_code == 201 + # Mock comparison recommends keep_new -> primary handover + assert second.data["is_primary"] is True + assert second.data["comparison"]["recommendation"] == "keep_new" + assert Syllabus.objects.get(pk=first.data["id"]).is_primary is False + + def test_download_requires_login( + self, base_client, auth_client, user, course, syllabus_urls + ): + instructor = _offering_with_instructor(course) + syllabus = SyllabusFactory( + course=course, instructor=instructor, uploaded_by=user + ) + anon = APIClient() # base_client is shared with auth_client + response = anon.get(syllabus_urls["download"](syllabus.id)) + assert response.status_code in (401, 403) + authed = auth_client.get(syllabus_urls["download"](syllabus.id)) + assert authed.status_code == 200 + assert "attachment" in authed["Content-Disposition"] + streamed = b"".join(authed.streaming_content) + assert streamed == syllabus.file.file.read() + + def test_staff_patch_clears_sibling_primary( + self, auth_client, staff_client, user, course, syllabus_urls + ): + instructor = _offering_with_instructor(course) + primary = SyllabusFactory( + course=course, instructor=instructor, uploaded_by=user, is_primary=True + ) + other = SyllabusFactory( + course=course, instructor=instructor, uploaded_by=user, is_primary=False + ) + # Non-staff cannot patch + denied = auth_client.patch( + syllabus_urls["detail"](other.id), + {"summary_md": "hacked", "is_primary": True}, + format="json", + ) + assert denied.status_code == 403 + + updated = staff_client.patch( + syllabus_urls["detail"](other.id), + {"summary_md": "Curated by admin", "is_primary": True}, + format="json", + ) + assert updated.status_code == 200 + primary.refresh_from_db() + other.refresh_from_db() + assert primary.is_primary is False + assert other.is_primary is True + assert other.summary_md == "Curated by admin" + + +# --------------------------------------------------------------------------- +# Task failure + OCR path +# --------------------------------------------------------------------------- + + +class TestTaskEdgeCases: + def test_ollama_failure_marks_syllabus_failed( + self, auth_client, course, syllabus_urls, fake_extraction, monkeypatch + ): + from apps.web.syllabus_analysis import SyllabusAnalysisError + + instructor = _offering_with_instructor(course) + + def _chat_down(messages, format_json=False): + raise SyllabusAnalysisError("model down") + + monkeypatch.setattr("apps.web.syllabus_analysis.ollama_chat", _chat_down) + upload = _make_pdf_upload() + response = auth_client.post( + syllabus_urls["list"], + {"file": upload, "instructor": instructor.id}, + format="multipart", + ) + assert response.status_code == 201 + syllabus = Syllabus.objects.get(pk=response.data["id"]) + assert syllabus.status == Syllabus.Status.FAILED + assert "model down" in syllabus.error_message + + def test_ocr_path_called_for_textless_pdf( + self, auth_client, course, syllabus_urls, fake_ollama, monkeypatch + ): + instructor = _offering_with_instructor(course) + + calls = {} + + def fake_extract(content: bytes) -> str: + calls["extract"] = True + return "" + + def fake_render(content: bytes, max_pages=None): + calls["render"] = True + return [b"fake-png-page"] + + def fake_ocr(images): + calls["ocr"] = True + return "OCR'd syllabus text" + + import apps.web.tasks as tasks_mod + + monkeypatch.setattr(tasks_mod, "extract_pdf_text", fake_extract) + monkeypatch.setattr(tasks_mod, "render_pdf_pages", fake_render) + monkeypatch.setattr(tasks_mod, "ocr_pages", fake_ocr) + upload = _make_pdf_upload() + response = auth_client.post( + syllabus_urls["list"], + {"file": upload, "instructor": instructor.id}, + format="multipart", + ) + assert response.status_code == 201 + assert calls.get("extract") and calls.get("render") and calls.get("ocr") + syllabus = Syllabus.objects.get(pk=response.data["id"]) + assert syllabus.status == Syllabus.Status.ANALYZED + assert "OCR'd" in syllabus.file.extracted_text + + +# --------------------------------------------------------------------------- +# Instructor serialization + staff flag +# --------------------------------------------------------------------------- + + +class TestPayloads: + def test_course_instructors_endpoint_returns_objects(self, base_client, course): + instructor = _offering_with_instructor(course) + url = reverse("course_instructors", kwargs={"course_id": course.id}) + response = base_client.get(url) + assert response.status_code == 200 + assert {"id": instructor.id, "name": instructor.name} in response.data[ + "instructors" + ] + + def test_course_detail_instructors_are_objects(self, base_client, course): + instructor = _offering_with_instructor(course) + url = reverse("course_detail_api", kwargs={"course_id": course.id}) + response = base_client.get(url) + assert response.status_code == 200 + instructors = response.data["instructors"] + assert any(i["id"] == instructor.id for i in instructors) + assert all(isinstance(i, dict) for i in instructors) + + def test_user_status_exposes_staff(self, auth_client, staff_client): + response = auth_client.get(reverse("user_status")) + assert response.data["is_staff"] is False + response = staff_client.get(reverse("user_status")) + assert response.data["is_staff"] is True + anonymous = APIClient() + response = anonymous.get(reverse("user_status")) + assert response.data["isAuthenticated"] is False + + +# --------------------------------------------------------------------------- +# Low-match rejection + recycle +# --------------------------------------------------------------------------- + + +class TestLowMatchRejection: + def test_low_match_score_rejects_and_recycles_file( + self, auth_client, course, syllabus_urls, fake_ollama, fake_extraction + ): + fake_ollama.state["match_score"] = 15 + instructor = _offering_with_instructor(course) + response = auth_client.post( + syllabus_urls["list"], + {"file": _make_pdf_upload(), "instructor": instructor.id}, + format="multipart", + ) + assert response.status_code == 201 + data = response.data + assert data["status"] == Syllabus.Status.REJECTED + assert data["is_primary"] is False + assert data["verdict"]["match_score"] == 15 + + file_obj = SyllabusFile.objects.get(pk=data["file"]["id"]) + assert file_obj.file.name.startswith("recycle/") + + def test_boundary_score_60_is_analyzed( + self, auth_client, course, syllabus_urls, fake_ollama, fake_extraction + ): + fake_ollama.state["match_score"] = 60 + instructor = _offering_with_instructor(course) + response = auth_client.post( + syllabus_urls["list"], + {"file": _make_pdf_upload(), "instructor": instructor.id}, + format="multipart", + ) + assert response.status_code == 201 + assert response.data["status"] == Syllabus.Status.ANALYZED + file_obj = SyllabusFile.objects.get(pk=response.data["file"]["id"]) + assert file_obj.file.name.startswith("syllabi/") + + def test_shared_file_not_recycled_while_referenced( + self, auth_client, course, syllabus_urls, fake_ollama, fake_extraction + ): + instructor_a = _offering_with_instructor(course) + instructor_b = InstructorFactory() + offering_b = CourseOfferingFactory(course=course) + offering_b.instructors.add(instructor_b) + + first = auth_client.post( + syllabus_urls["list"], + {"file": _make_pdf_upload(), "instructor": instructor_a.id}, + format="multipart", + ) + assert first.status_code == 201 + + fake_ollama.state["match_score"] = 5 + second = auth_client.post( + syllabus_urls["list"], + {"file": _make_pdf_upload(), "instructor": instructor_b.id}, + format="multipart", + ) + assert second.status_code == 201 + assert second.data["status"] == Syllabus.Status.REJECTED + assert second.data["file"]["id"] == first.data["file"]["id"] + + # Still referenced by the analyzed first syllabus -> not recycled. + file_obj = SyllabusFile.objects.get(pk=first.data["file"]["id"]) + assert file_obj.file.name.startswith("syllabi/") + + +# --------------------------------------------------------------------------- +# Deletion (API + signal-driven recycle) +# --------------------------------------------------------------------------- + + +class TestSyllabusDelete: + def test_delete_requires_staff(self, auth_client, user, course, syllabus_urls): + instructor = _offering_with_instructor(course) + syllabus = SyllabusFactory( + course=course, instructor=instructor, uploaded_by=user + ) + response = auth_client.delete(syllabus_urls["detail"](syllabus.id)) + assert response.status_code == 403 + assert Syllabus.objects.filter(pk=syllabus.pk).exists() + + def test_staff_delete_recycles_orphan_file( + self, staff_client, user, course, syllabus_urls + ): + instructor = _offering_with_instructor(course) + syllabus = SyllabusFactory( + course=course, instructor=instructor, uploaded_by=user + ) + file_obj = syllabus.file + assert file_obj.file.name.startswith("syllabi/") + + response = staff_client.delete(syllabus_urls["detail"](syllabus.id)) + assert response.status_code == 204 + assert not Syllabus.objects.filter(pk=syllabus.pk).exists() + file_obj.refresh_from_db() + assert file_obj.file.name.startswith("recycle/") + + def test_queryset_delete_recycles_each_orphan_and_keeps_shared(self, course): + """QuerySet.delete (admin delete_selected path) fires post_delete.""" + instructor_a = _offering_with_instructor(course) + instructor_b = InstructorFactory() + offering_b = CourseOfferingFactory(course=course) + offering_b.instructors.add(instructor_b) + + shared_file = SyllabusFileFactory() + first = SyllabusFactory( + course=course, instructor=instructor_a, file=shared_file + ) + second = SyllabusFactory( + course=course, instructor=instructor_b, file=shared_file + ) + + Syllabus.objects.filter(pk=second.pk).delete() + shared_file.refresh_from_db() + assert shared_file.file.name.startswith("syllabi/") # first still refs it + + Syllabus.objects.filter(pk=first.pk).delete() + shared_file.refresh_from_db() + assert shared_file.file.name.startswith("recycle/") diff --git a/apps/web/urls.py b/apps/web/urls.py index 6915bd3..7349694 100644 --- a/apps/web/urls.py +++ b/apps/web/urls.py @@ -32,6 +32,21 @@ views.CoursesReviewsAPI.as_view(), name="course_review_api", ), + re_path( + r"^courses/(?P[0-9]+)/syllabi/$", + views.CourseSyllabiAPI.as_view(), + name="course_syllabi_api", + ), + re_path( + r"^syllabi/(?P[0-9]+)/download/$", + views.syllabus_download, + name="syllabus_download_api", + ), + re_path( + r"^syllabi/(?P[0-9]+)/$", + views.SyllabusDetailAPI.as_view(), + name="syllabus_detail_api", + ), re_path(r"^reviews/?$", views.UserReviewsAPI.as_view(), name="user_reviews_api"), re_path( r"^reviews/(?P[0-9]+)/$", diff --git a/apps/web/views.py b/apps/web/views.py index 8e55d9e..955f10a 100644 --- a/apps/web/views.py +++ b/apps/web/views.py @@ -1,13 +1,18 @@ +import hashlib import logging from django.conf import settings +from django.core.files.base import ContentFile +from django.db import IntegrityError from django.db.models import Count, Prefetch, Q +from django.http import FileResponse +from django.views.decorators.csrf import ensure_csrf_cookie from rest_framework import generics, mixins, pagination, status from rest_framework.decorators import ( api_view, permission_classes, ) -from rest_framework.permissions import AllowAny, IsAuthenticated +from rest_framework.permissions import AllowAny, IsAdminUser, IsAuthenticated from rest_framework.response import Response from apps.web.models import ( @@ -16,6 +21,8 @@ Instructor, Review, ReviewVote, + Syllabus, + SyllabusFile, Vote, ) from apps.web.serializers import ( @@ -24,7 +31,11 @@ CourseVoteSerializer, ReviewSerializer, ReviewVoteSerializer, + SyllabusAdminUpdateSerializer, + SyllabusCreateSerializer, + SyllabusSerializer, ) +from apps.web.tasks import process_syllabus from lib.departments import get_department_name from lib.grades import numeric_value_for_grade from lib.terms import numeric_value_of_term @@ -36,6 +47,7 @@ class CoursesPagination(pagination.PageNumberPagination): page_size = settings.WEB["COURSE"]["PAGE_SIZE"] +@ensure_csrf_cookie @api_view(["GET"]) def user_status(request): """ @@ -43,12 +55,19 @@ def user_status(request): Input: - None Output: - - Authenticated user: {"isAuthenticated": true, "username": "string"} + - Authenticated user: {"isAuthenticated": true, "username": "string", + "is_staff": bool} - Anonymous user: {"isAuthenticated": false} """ if request.user.is_authenticated: logger.info("User is authenticated") - return Response({"isAuthenticated": True, "username": request.user.username}) + return Response( + { + "isAuthenticated": True, + "username": request.user.username, + "is_staff": request.user.is_staff, + } + ) else: logger.info("User is not authenticated") return Response({"isAuthenticated": False}) @@ -134,7 +153,7 @@ def _filter_by_score(self, queryset): try: threshold = int(param_value) queryset = queryset.filter(**{f"{field_name}__gte": threshold}) - except ValueError, TypeError: + except (ValueError, TypeError): pass return queryset @@ -282,7 +301,7 @@ def post(self, request, *args, **kwargs): ) # Validate and save review using ReviewSerializer - serializer = ReviewSerializer(data=request.data) + serializer = ReviewSerializer(data=request.data, context={"course": course}) if not serializer.is_valid(): logger.warning("Review serializer errors: %s", serializer.errors) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) @@ -474,17 +493,21 @@ def course_professors(request, course_id): @permission_classes([AllowAny]) def course_instructors(request, course_id): """ - Unused API. + List instructors for a course (term-agnostic), as {id, name} objects. """ try: course = Course.objects.get(pk=course_id) - instructors = course.get_instructors() - return Response( - {"instructors": [instructor.name for instructor in instructors]}, status=200 - ) except Course.DoesNotExist: logger.warning("Course with id %d not found for instructors API", course_id) return Response({"error": "Course not found"}, status=404) + instructors = ( + Instructor.objects.filter(courseoffering__course=course) + .distinct() + .order_by("name") + ) + return Response( + {"instructors": [{"id": i.id, "name": i.name} for i in instructors]}, status=200 + ) @api_view(["POST"]) @@ -584,3 +607,178 @@ def review_vote_api(request, review_id): "user_vote": user_vote, } ) + + +class CourseSyllabiAPI( + generics.GenericAPIView, mixins.ListModelMixin, mixins.CreateModelMixin +): + """ + List or upload syllabi for a course. + + GET: public, lists SyllabusSerializer rows (newest first). + POST: authenticated, multipart {file, instructor}; dedupes by sha256. + """ + + def get_permissions(self): + if self.request.method == "POST": + return [IsAuthenticated()] + return [AllowAny()] + + def get_queryset(self): + return Syllabus.objects.filter(course_id=self.kwargs["course_id"]) + + def get_serializer_class(self): + if self.request.method == "POST": + return SyllabusCreateSerializer + return SyllabusSerializer + + def get(self, request, *args, **kwargs): + return self.list(request, *args, **kwargs) + + def post(self, request, *args, **kwargs): + return self.create(request, *args, **kwargs) + + def list(self, request, *args, **kwargs): + course_id = self.kwargs.get("course_id") + if not Course.objects.filter(pk=course_id).exists(): + return Response( + {"detail": "Course not found"}, status=status.HTTP_404_NOT_FOUND + ) + queryset = self.get_queryset().select_related( + "instructor", "file", "uploaded_by" + ) + serializer = SyllabusSerializer(queryset, many=True) + return Response(serializer.data) + + def create(self, request, *args, **kwargs): + course_id = self.kwargs.get("course_id") + try: + course = Course.objects.get(pk=course_id) + except Course.DoesNotExist: + return Response( + {"detail": "Course not found"}, status=status.HTTP_404_NOT_FOUND + ) + + serializer = SyllabusCreateSerializer( + data=request.data, context={"course": course} + ) + if not serializer.is_valid(): + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + upload = serializer.validated_data["file"] + instructor_id = serializer.validated_data["instructor"] + content = upload.read() + digest = hashlib.sha256(content).hexdigest() + + # Reuse an existing SyllabusFile (same bytes already on disk). + file_obj = SyllabusFile.objects.filter(sha256=digest).first() + if file_obj is None: + name = upload.name.lower() + ext = ".pdf" if name.endswith(".pdf") else ".docx" + file_obj = SyllabusFile( + sha256=digest, + content_type=upload.content_type or "", + original_filename=upload.name, + size=len(content), + ) + try: + file_obj.file.save(f"{digest}{ext}", ContentFile(content), save=True) + except IntegrityError: + # Lost a concurrent identical upload race; reuse the winner. + file_obj = SyllabusFile.objects.get(sha256=digest) + + # Idempotent re-upload of the same file for the same course+instructor. + existing = Syllabus.objects.filter( + course=course, instructor_id=instructor_id, file=file_obj + ).first() + if existing: + return Response( + SyllabusSerializer(existing).data, status=status.HTTP_200_OK + ) + + syllabus = Syllabus.objects.create( + course=course, + instructor_id=instructor_id, + file=file_obj, + uploaded_by=request.user, + status=Syllabus.Status.PENDING, + ) + try: + process_syllabus.delay(syllabus.id) + except Exception: # noqa: BLE001 - broker down; task stays pending + logger.exception("Failed to enqueue syllabus analysis %d", syllabus.id) + # Eager execution (tests) may already have analyzed this row. + syllabus.refresh_from_db() + return Response( + SyllabusSerializer(syllabus).data, status=status.HTTP_201_CREATED + ) + + +class SyllabusDetailAPI( + generics.GenericAPIView, + mixins.RetrieveModelMixin, + mixins.UpdateModelMixin, + mixins.DestroyModelMixin, +): + """ + Syllabus detail. GET public; PATCH/DELETE staff-only (summary/verdict/ + primary; deletion also recycles the file when unreferenced). + """ + + serializer_class = SyllabusSerializer + queryset = Syllabus.objects.select_related("instructor", "file", "uploaded_by") + lookup_field = "pk" + lookup_url_kwarg = "syllabus_id" + + def get_serializer_class(self): + if self.request.method in ("PUT", "PATCH"): + return SyllabusAdminUpdateSerializer + return SyllabusSerializer + + def get_permissions(self): + if self.request.method in ("PUT", "PATCH", "DELETE"): + return [IsAdminUser()] + return [AllowAny()] + + def get(self, request, *args, **kwargs): + return self.retrieve(request, *args, **kwargs) + + def delete(self, request, *args, **kwargs): + return self.destroy(request, *args, **kwargs) + + def patch(self, request, *args, **kwargs): + syllabus = self.get_object() + serializer = SyllabusAdminUpdateSerializer( + syllabus, data=request.data, partial=True + ) + if not serializer.is_valid(): + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + syllabus = serializer.save() + if serializer.validated_data.get("is_primary"): + Syllabus.objects.filter( + course=syllabus.course, + instructor=syllabus.instructor, + ).exclude(pk=syllabus.pk).update(is_primary=False) + return Response(SyllabusSerializer(syllabus).data) + + +@api_view(["GET"]) +@permission_classes([IsAuthenticated]) +def syllabus_download(request, syllabus_id): + """Stream the original uploaded file; logged-in users only.""" + try: + syllabus = Syllabus.objects.select_related("file").get(pk=syllabus_id) + except Syllabus.DoesNotExist: + # The URL pattern already restricts syllabus_id to digits, but strip + # CR/LF anyway so the value can never forge log lines. + safe_id = str(syllabus_id).replace("\r", "").replace("\n", "") + logger.warning("Syllabus %s not found for download", safe_id) + return Response({"detail": "Syllabus not found"}, status=404) + file_obj = syllabus.file + response = FileResponse( + file_obj.file.open("rb"), + as_attachment=True, + filename=file_obj.original_filename, + content_type=file_obj.content_type or "application/octet-stream", + ) + return response diff --git a/compose.dev.yaml b/compose.dev.yaml index 440bc30..84d79f7 100644 --- a/compose.dev.yaml +++ b/compose.dev.yaml @@ -20,6 +20,29 @@ services: REDIS__URL: ${REDIS__URL:-redis://cache:6379/0} volumes: - ./config.yaml:/app/config.yaml:ro + - syllabus_media:/app/media + + worker: + build: + context: . + dockerfile: Containerfile + command: ["celery", "-A", "website.celery", "worker", "--loglevel=info"] + env_file: + - .env + environment: + DATABASE__URL: ${DATABASE__URL:-postgres://admin:test@db:5432/coursereview} + REDIS__URL: ${REDIS__URL:-redis://cache:6379/0} + # Local Ollama runs on the host; podman host-gateway hostname. + OLLAMA__BASE_URL: ${OLLAMA__BASE_URL:-http://host.containers.internal:11434} + volumes: + - ./config.yaml:/app/config.yaml:ro + - syllabus_media:/app/media + depends_on: + db: + condition: service_healthy + cache: + condition: service_healthy + restart: unless-stopped migrate: build: @@ -32,3 +55,6 @@ services: REDIS__URL: ${REDIS__URL:-redis://cache:6379/0} volumes: - ./config.yaml:/app/config.yaml:ro + +volumes: + syllabus_media: diff --git a/compose.yaml b/compose.yaml index a20016d..ba5f676 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,7 +2,7 @@ services: db: image: postgres:18-alpine volumes: - - postgres18_data:/var/lib/postgresql/data + - postgres18_data:/var/lib/postgresql environment: POSTGRES_DB: ${POSTGRES_DB:-coursereview} POSTGRES_USER: ${POSTGRES_USER:-admin} @@ -23,8 +23,12 @@ services: retries: 5 restart: unless-stopped - backend: + CourseReview: image: coursereview-backend + volumes: + - ./config.yaml:/app/config.yaml:ro + - ./.env:/app/.env:ro + - /mnt/data/GCAtlas:/app/media depends_on: db: condition: service_healthy @@ -57,5 +61,44 @@ services: command: ["python", "django_manage.py", "migrate"] restart: "no" + worker: + image: coursereview-backend + volumes: + - ./config.yaml:/app/config.yaml:ro + - ./.env:/app/.env:ro + - /mnt/data/GCAtlas:/app/media + depends_on: + db: + condition: service_healthy + cache: + condition: service_healthy + environment: + PYTHONUNBUFFERED: "1" + OLLAMA__BASE_URL: http://host.docker.internal:11434 + extra_hosts: + - "host.docker.internal:host-gateway" + # distroless image runs non-root with an unwritable CWD; the default + # ./celerybeat-schedule would raise PermissionError and kill beat. + command: + [ + "celery", + "-A", + "website.celery", + "worker", + "--beat", + "--loglevel=info", + "--pool=solo", + "--schedule=/tmp/celerybeat-schedule", + ] + restart: unless-stopped + + tunnel: + image: cloudflare/cloudflared:latest + command: tunnel run --protocol http2 --token ${TUNNEL_TOKEN} + restart: unless-stopped + depends_on: + CourseReview: + condition: service_healthy + volumes: postgres18_data: diff --git a/config.yaml.example b/config.yaml.example index 2890816..847e727 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -61,3 +61,14 @@ QUEST: URL: "https://wj.sjtu.edu.cn/q/dummy2" QUESTIONID: 10000002 # AUTO_IMPORT_CRAWLED_DATA: true + +# SYLLABUS: +# MAX_UPLOAD_SIZE: 20971520 # 20 MB +# ALLOWED_EXTENSIONS: [".pdf", ".docx"] + +# OLLAMA: # local LLM used for syllabus analysis/OCR (env: OLLAMA__BASE_URL etc.) +# BASE_URL: "http://127.0.0.1:11434" +# MODEL: "qwen3.8:latest" +# TIMEOUT: 600 +# NUM_CTX: 262144 +# MAX_PAGES: 30 diff --git a/lib/name_normalization.py b/lib/name_normalization.py new file mode 100644 index 0000000..3d7eb3f --- /dev/null +++ b/lib/name_normalization.py @@ -0,0 +1,208 @@ +"""Instructor / professor name canonicalization shared by crawlers and review I/O. + +GC (gc.sjtu.edu.cn) is the college's own course site, and its pages are the +single source of truth for instructor names. The pages themselves are +inconsistent across terms (case, hyphens, middle names, term/CJK annotations, +given-vs-family order, nickname variants, occasional typos), so any place that +accepts an instructor name from outside — crawler imports or user-submitted +reviews — must normalize it against the canonical Instructor rows before +storing, and only fall back to creating/keeping a new name when nothing +matches. + +All helpers here are pure string functions (no Django imports) so both the +spider apps and the web review API can share them without import cycles. +""" + +import re + +CJK_RE = re.compile(r"[\u4e00-\u9fff\u3400-\u4dbf]+") +PAREN_RE = re.compile(r"[\(（][^)）]*[\)）]") +TITLE_RE = re.compile(r"^(dr|prof|ms|mr|mrs|miss)\.?\s+", re.IGNORECASE) +QUOTE_RE = re.compile(r"[\"'“”‘’]") +TRAILING_PUNCT_RE = re.compile(r"[\s.,;:]+$") + +# Cells that are not instructor names at all and must never be stored. +JUNK_INSTRUCTOR_NAMES = { + ",", + "，", + ";", + "；", + "-", + "–", + "—", + ".", + "教师", + "教授", + "老师", + "staff", + "tbd", + "tba", +} + +# Source cells that cram several instructors into one string without +# separators (curated; grows as the site produces new cases). +INSTRUCTOR_SPLITS = { + "Zhaoguang Wang Ting Sun": ["Zhaoguang Wang", "Ting Sun"], +} + +# Token-level nicknames that plain subsequence matching cannot catch +# (Nick/Nicholas is not a prefix relationship). +TOKEN_ALIASES = {"nick": "nicholas"} + + +def clean_instructor_name(name): + """Strip page annotations so names are clean and matchable. + + Removes leading titles (Dr./Prof./...), parenthetical annotations + ("(Fall)", "(Summer).", "(余琼)", "(UM)"), trailing CJK annotations + ("YAN Xu 闫旭"), quotes ("Jaehyung “Joshua” Ju"), and stray trailing + punctuation. Returns "" when nothing meaningful remains. + """ + n = (name or "").replace("\u00a0", " ") + n = TITLE_RE.sub("", n) + n = PAREN_RE.sub(" ", n) + n = CJK_RE.sub(" ", n) + n = QUOTE_RE.sub("", n) + n = TRAILING_PUNCT_RE.sub("", n) + n = re.sub(r"\s+", " ", n).strip() + return n + + +def name_tokens(name): + """Lowercased letter tokens; hyphens/punctuation inside a token are merged + (e.g. 'Welch-Bolen' -> 'welchbolen') so hyphen variants compare equal.""" + tokens = [] + for raw in name.split(): + token = re.sub(r"[^a-z]", "", raw.lower()) + if token: + tokens.append(token) + return tokens + + +def _levenshtein(a, b): + if a == b: + return 0 + if not a: + return len(b) + if not b: + return len(a) + prev = list(range(len(b) + 1)) + for i, ca in enumerate(a, 1): + cur = [i] + for j, cb in enumerate(b, 1): + cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))) + prev = cur + return prev[-1] + + +def tokens_equivalent(a, b): + """Token equality allowing curated nicknames (nick ~ nicholas).""" + return a == b or TOKEN_ALIASES.get(a, a) == b or a == TOKEN_ALIASES.get(b, b) + + +def _fuzzy_pair(a, b): + """True when a token pair is a plausible typo: same-ish length, small + edit distance. Short tokens are never fuzzy-matched (surname confusions + like Bo/Po must not collapse).""" + if len(a) < 5 or len(b) < 5: + return False + if abs(len(a) - len(b)) > 2: + return False + distance = _levenshtein(a, b) + return distance <= 2 and distance <= len(a) // 3 + 1 + + +def is_token_subsequence(short_tokens, long_tokens): + """Ordered subsequence with alias/fuzzy awareness. + + Used to catch middle-name/extra-token variants ('Horst Harold Hohberger' + vs 'Horst Hohberger'); fuzzy pairs are only tolerated for the shared run + of tokens, never to skip extra tokens. + """ + it = iter(long_tokens) + for token in short_tokens: + for candidate in it: + if tokens_equivalent(token, candidate) or _fuzzy_pair(token, candidate): + break + else: + return False + return True + + +def best_name_match(clean_name, candidate_names): + """Return the candidate a cleaned name refers to, or None. + + Matching ladder, first hit wins: + 1. exact name + 2. identical letter sequence (case/punctuation/hyphen variants) + 3. same word set (given-vs-family order variants) + 4. ordered token subsequence with >=2 tokens (middle names dropped, + nickname aliases) + 5. single-token typo (same token count, one fuzzy edit-distance pair) + Callers should pass candidates ordered by preference (most-used spelling + first) so ties resolve deterministically. + """ + tokens = name_tokens(clean_name) + letters = "".join(tokens) + if not letters: + return None + + for candidate in candidate_names: + candidate_tokens = name_tokens(candidate) + if not candidate_tokens: + continue + candidate_letters = "".join(candidate_tokens) + if candidate == clean_name: + return candidate + if candidate_letters == letters: + return candidate + if ( + len(candidate_tokens) >= 2 + and len(tokens) >= 2 + and set(candidate_tokens) == set(tokens) + ): + return candidate + for candidate in candidate_names: + candidate_tokens = name_tokens(candidate) + if len(candidate_tokens) >= 2 and is_token_subsequence( + candidate_tokens, tokens + ): + return candidate + if len(tokens) >= 2 and is_token_subsequence(tokens, candidate_tokens): + return candidate + for candidate in candidate_names: + candidate_tokens = name_tokens(candidate) + if len(tokens) == len(candidate_tokens) and len(tokens) >= 2: + fuzzy = sum( + 1 + for a, b in zip(tokens, candidate_tokens) + if not tokens_equivalent(a, b) and not _fuzzy_pair(a, b) + ) + if fuzzy == 0 and tokens != candidate_tokens: + return candidate + return None + + +def canonicalize_professor(raw_name, candidate_names): + """Canonicalize a user-supplied professor name against candidate names + (typically the course's instructors). Returns the canonical candidate when + a match is found; otherwise the submitted name is kept as its own + professor (to be resolved later, not guessed). Unmatched names are stored + cleaned of annotations when they still carry a real first+last name; + inputs like "Dr. Testing" (title + single name) are kept verbatim. + """ + cleaned = clean_instructor_name(raw_name) + if not cleaned: + return "" + match = best_name_match(cleaned, list(candidate_names)) + if match is not None: + return match + + def pretty(name): + return " ".join( + token if token.isupper() else token.title() for token in name.split() + ) + + if len(cleaned.split()) >= 2: + return pretty(cleaned) + return pretty(raw_name.strip()) diff --git a/pyproject.toml b/pyproject.toml index bdba2ab..547e5b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,7 @@ version = "0.0.1" dependencies = [ "beautifulsoup4>=4.14.0", "bpython>=0.26", + "celery>=5.6.3", "dj-database-url>=3.1.0", "django>=6.0.0", "django-cors-headers>=4.9.0", @@ -14,12 +15,16 @@ dependencies = [ "gunicorn>=26.0.0", "httpx>=0.28.0", "psycopg[binary]>=3.3.0", + "pypdf>=6.17.0", + "pypdfium2>=5.13.0", "python-dateutil>=2.9.0.post0", + "python-docx>=1.2.0", "python-dotenv>=1.2.0", "pytz>=2026.2", "pyyaml>=6.0.0", "redis>=7.4.0", "requests>=2.34.0", + "whitenoise>=6.12.0", ] [tool.uv] diff --git a/uv.lock b/uv.lock index dc4218e..d9600da 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,18 @@ version = 1 revision = 3 requires-python = "==3.14.*" +[[package]] +name = "amqp" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, +] + [[package]] name = "ansicon" version = "1.89.0" @@ -45,6 +57,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, ] +[[package]] +name = "billiard" +version = "4.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537, upload-time = "2025-11-30T13:28:48.52Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" }, +] + [[package]] name = "blessed" version = "1.42.0" @@ -75,6 +96,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/92/26d8d98de4c1676305e03ec2be67850afaf883b507bf71b917d852585ec8/bpython-0.26-py3-none-any.whl", hash = "sha256:91bdbbe667078677dc6b236493fc03e47a04cd099630a32ca3f72d6d49b71e20", size = 175988, upload-time = "2025-10-28T07:19:40.114Z" }, ] +[[package]] +name = "celery" +version = "5.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "billiard" }, + { name = "click" }, + { name = "click-didyoumean" }, + { name = "click-plugins" }, + { name = "click-repl" }, + { name = "kombu" }, + { name = "python-dateutil" }, + { name = "tzlocal" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/b4/a1233943ab5c8ea05fb877a88a0a0622bf47444b99e4991a8045ac37ea1d/celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912", size = 1742243, upload-time = "2026-03-26T12:14:51.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" }, +] + [[package]] name = "certifi" version = "2026.5.20" @@ -125,6 +166,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, ] +[[package]] +name = "click" +version = "8.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, +] + +[[package]] +name = "click-didyoumean" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" }, +] + +[[package]] +name = "click-plugins" +version = "1.1.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" }, +] + +[[package]] +name = "click-repl" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "prompt-toolkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -141,6 +228,7 @@ source = { virtual = "." } dependencies = [ { name = "beautifulsoup4" }, { name = "bpython" }, + { name = "celery" }, { name = "dj-database-url" }, { name = "django" }, { name = "django-cors-headers" }, @@ -150,12 +238,16 @@ dependencies = [ { name = "gunicorn" }, { name = "httpx" }, { name = "psycopg", extra = ["binary"] }, + { name = "pypdf" }, + { name = "pypdfium2" }, { name = "python-dateutil" }, + { name = "python-docx" }, { name = "python-dotenv" }, { name = "pytz" }, { name = "pyyaml" }, { name = "redis" }, { name = "requests" }, + { name = "whitenoise" }, ] [package.dev-dependencies] @@ -173,6 +265,7 @@ lint = [ requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.14.0" }, { name = "bpython", specifier = ">=0.26" }, + { name = "celery", specifier = ">=5.6.3" }, { name = "dj-database-url", specifier = ">=3.1.0" }, { name = "django", specifier = ">=6.0.0" }, { name = "django-cors-headers", specifier = ">=4.9.0" }, @@ -182,12 +275,16 @@ requires-dist = [ { name = "gunicorn", specifier = ">=26.0.0" }, { name = "httpx", specifier = ">=0.28.0" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.3.0" }, + { name = "pypdf", specifier = ">=6.17.0" }, + { name = "pypdfium2", specifier = ">=5.13.0" }, { name = "python-dateutil", specifier = ">=2.9.0.post0" }, + { name = "python-docx", specifier = ">=1.2.0" }, { name = "python-dotenv", specifier = ">=1.2.0" }, { name = "pytz", specifier = ">=2026.2" }, { name = "pyyaml", specifier = ">=6.0.0" }, { name = "redis", specifier = ">=7.4.0" }, { name = "requests", specifier = ">=2.34.0" }, + { name = "whitenoise", specifier = ">=6.12.0" }, ] [package.metadata.requires-dev] @@ -448,6 +545,65 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/00/b61668fd3b1e43b445979ec9a9e0af4781bf06884937d1e906f6a1be6dff/jinxed-2.0.0-py2.py3-none-any.whl", hash = "sha256:b3df1be5262a37145ef42875a8bbf918f1a563fbd035359650dd9fc0bb2b9294", size = 95364, upload-time = "2026-05-08T21:25:24.536Z" }, ] +[[package]] +name = "kombu" +version = "5.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "amqp" }, + { name = "packaging" }, + { name = "tzdata" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" }, +] + +[[package]] +name = "lxml" +version = "6.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/ad/28ecd7cb894d172f3c9c80a075eeeb2017ac62e3632cee05a5f9493547eb/lxml-6.1.3.tar.gz", hash = "sha256:45222d94ddd511536f3b2f7d9deae3b2339b4ce0f075f1ca25703b07cad9dd21", size = 4211198, upload-time = "2026-09-02T14:48:02.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/15/fc75a70b0af6021d0ea16811f1fc71cc42cd06ce90fe10f007a69b2eed84/lxml-6.1.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2bec13085dc8ef48a3fe62f7dfcacfeda2c785cdf19cc8eeda2bb9ed081da165", size = 8609725, upload-time = "2026-09-02T14:49:00.156Z" }, + { url = "https://files.pythonhosted.org/packages/84/ef/398fcf9018f881ec9aeaafae1ddd6586dfb13314a35d35e899de373dcae0/lxml-6.1.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4f4db7c7e954d289d71878938348b3d91b904a3e8210a11939359fb758a58e7d", size = 4639629, upload-time = "2026-09-02T14:49:02.81Z" }, + { url = "https://files.pythonhosted.org/packages/a7/2d/49b6a6ad7ce8f64b07b9fe852ff0c6d3fcbb26db61bee4f63d4120180a1c/lxml-6.1.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2cae5d5c90a62d9139c512a0cb1aad1d182b022b5740daea2617eb5bf7fc658e", size = 4965074, upload-time = "2026-09-02T14:49:05.133Z" }, + { url = "https://files.pythonhosted.org/packages/66/bc/6230cf80e4331c33383b0b6b73dc31a393dd76edd4cb73d761de5123034d/lxml-6.1.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c6c0c13128a32eb04a51357e56a094e13aa8e6d3d1884de2e9ae923f6915e1a8", size = 5099355, upload-time = "2026-09-02T14:49:07.343Z" }, + { url = "https://files.pythonhosted.org/packages/ac/cf/d1143d9b7717e07a82f158a1fc9ce6e581fdad1226734950af869e3ffde4/lxml-6.1.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2221e88679d1351e9a40aaee54bc65679b9795bbd0160bc3d5e36b163344eb75", size = 5036795, upload-time = "2026-09-02T14:49:09.65Z" }, + { url = "https://files.pythonhosted.org/packages/31/6f/194bb00ffb89712c30f5a7e1b8e685590e140fad6c8261fec172c09a3dc0/lxml-6.1.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cfb398886a7eb4c719161c3efcff2a1248febc53a4d8e5072d2d8a87fed84ac9", size = 5658740, upload-time = "2026-09-02T14:49:11.9Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/27e3cee3dcdb3b7bc09727b642bdbfcd098490ea77df04611db9060d7722/lxml-6.1.3-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7eb78ba28b187e1e9203a55c60fcf70df2d22cb205fe6d51b9383d6097419f0", size = 5245991, upload-time = "2026-09-02T14:49:14.154Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e9/8312560579fc980bbd2233a8a673cc46f7d613d3633f2bf08a21e8f4ad13/lxml-6.1.3-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:ea6b1e9105b4b24a34c722432d9fb578f9ed83af21fa1abda639011e0f22bbb6", size = 5354136, upload-time = "2026-09-02T14:49:16.459Z" }, + { url = "https://files.pythonhosted.org/packages/74/d8/eda60f4f73a9c780b5d6e1175484f66e6c81a2c93346e2906a1fec9c7a02/lxml-6.1.3-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:e8b17e23df3e827a69d25af70990ca2420e92668aaffaeeb3cd2351d7916a023", size = 4704379, upload-time = "2026-09-02T14:49:19.032Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c8/c9cc60057be78ac34bd2b842e45e6e88edbfe5e532e82c3b82381b7aab49/lxml-6.1.3-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b7c37339d7e75cab9a123a04248e243cefefb302ad6db566ea0c77cbcde421e", size = 5258676, upload-time = "2026-09-02T14:49:21.306Z" }, + { url = "https://files.pythonhosted.org/packages/41/7b/66894008fee8d1785b8db129747ae963fd427b68f456918df7f2f24a8b98/lxml-6.1.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:83e3a51e7933db700a0da0db31849db3a24022d9970da9bb73001e1d0326fd92", size = 5090069, upload-time = "2026-09-02T14:49:23.562Z" }, + { url = "https://files.pythonhosted.org/packages/8b/31/c1b60404859f4c3cd1f41f29c65a24e25cea78fde822d9574a21f66810be/lxml-6.1.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9bde9ae026a55b9a192078dfa6e27dd0ca4a050171ab6272e92f97b757dfdf48", size = 4741958, upload-time = "2026-09-02T14:49:26.037Z" }, + { url = "https://files.pythonhosted.org/packages/23/b8/6285f0cf546f14da2554cabdeaf7c2c2ff3190c74807f0de2e8810a786f9/lxml-6.1.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1a635e837b50a1819bebfedaac5916498ea024120969da8790500148fb0a894d", size = 5683245, upload-time = "2026-09-02T14:49:28.438Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f6/2168cab44336dcb15fed0f0b78577225b83297cdf0dee349c95420c3dcb0/lxml-6.1.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d0c5c362bc94f1929dc7e96e715bbe7bd17037f802e6d8f0d1545df9133c0559", size = 5246087, upload-time = "2026-09-02T14:49:30.955Z" }, + { url = "https://files.pythonhosted.org/packages/f5/89/32f5de69a0a31f30e6164981851f87b37ecb2c4ee838e504b88d49d4818e/lxml-6.1.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c59e4265608da6a041f54646ecc0c9ecdbb19aaf14c4c684bb6c2114998cc415", size = 5269352, upload-time = "2026-09-02T14:49:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a1/741d952ed3a7ef7a50055c6415aec3f067015e97f72f4389ce77b09657ba/lxml-6.1.3-cp314-cp314-win32.whl", hash = "sha256:2e62c569ec7531b679b184cbfe335c501c1d13c4b363560013019962eb630e6d", size = 3662783, upload-time = "2026-09-02T14:50:23.751Z" }, + { url = "https://files.pythonhosted.org/packages/0f/bc/5811cc73cac05e324e05ba9b0924e1a163a317a167ede8a9c748b11db30a/lxml-6.1.3-cp314-cp314-win_amd64.whl", hash = "sha256:66299564c046bc7e0cc5de5106601eae907e9fa5904cd68a323380a8502f7861", size = 4073951, upload-time = "2026-09-02T14:50:26.348Z" }, + { url = "https://files.pythonhosted.org/packages/92/18/3768c8b01ac3a9bed1914715e6011711b00e2a11628ffa6f7fa37f8e0269/lxml-6.1.3-cp314-cp314-win_arm64.whl", hash = "sha256:ebd054ad1737a68fb7c5c073d405cef2b88bb824e294de3b4a4e995b47f0e376", size = 3749279, upload-time = "2026-09-02T14:50:28.749Z" }, + { url = "https://files.pythonhosted.org/packages/72/38/84684784738d9451db2b330de2483f496690c3a5c642071df24135739b37/lxml-6.1.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:5a143e6207579de8baeded4eaac9134413200359f1969d636f0bfb98ee8c3c8f", size = 8860296, upload-time = "2026-09-02T14:49:36.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/b7/fc4c50bb1b38e864010ea396046cabe85129bf9e65b11edcfbc37d356241/lxml-6.1.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a1cec0f99b9b914d39176347a93b7610dc09324491aee1cbc57cd291a41a1d55", size = 4755190, upload-time = "2026-09-02T14:49:39.872Z" }, + { url = "https://files.pythonhosted.org/packages/94/e2/ee9aa6ed2b666b2db1f6f7fd48964ff9da39ebe827ef5eac0ab881f639d9/lxml-6.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f6b9d2aad499c769ee8287609ab0e6de99d8bcea99c6e6c2e64945259fd52fb2", size = 4979517, upload-time = "2026-09-02T14:49:42.153Z" }, + { url = "https://files.pythonhosted.org/packages/29/e3/e7763d1661b283ddd4fa36f91b9a497db6b8d2aff55028b16c7f642e0755/lxml-6.1.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a23fefdb345b2d4d0ff2860571b5ff9a89a28b6a120f720e8fb0324d346626", size = 5115270, upload-time = "2026-09-02T14:49:44.493Z" }, + { url = "https://files.pythonhosted.org/packages/2d/cd/22205d5b4d177e3f4156f780412426ee7c7f8107809f119f0dcc40fa51e3/lxml-6.1.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:545ccc14fb05485f48b4439ec35beb16d5b5280eb6c81c658bd4707a2a119414", size = 5032449, upload-time = "2026-09-02T14:49:46.841Z" }, + { url = "https://files.pythonhosted.org/packages/da/43/06a4626c3bb79ef8c501b674afab8100d64e798665bb2a97d1c960636a49/lxml-6.1.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:93476b6514b373fc6ca67d26c442784f7807c86f00635bfe79f935c3eab2af17", size = 5603325, upload-time = "2026-09-02T14:49:49.664Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/733682a0c2de9f5779ba207bbb3f3f6be8c6bda863fc01739b186b38783a/lxml-6.1.3-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8db38ff3fb7aee7d6a82ae4da2eef1178656fe1216841fbd24870062a9d60473", size = 5229023, upload-time = "2026-09-02T14:49:52.447Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8a/e69cdaca3fd33a647942925664f01b20908d41a6968c182305be9c38fb11/lxml-6.1.3-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:25f4118c438f96bb466e83108506d03d5c31b1bd2387e83e5b070bda6ded9c37", size = 5317811, upload-time = "2026-09-02T14:49:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b2/0c397588174403c2ab68fc464abf97e03e7324f9c6cb6a99023104707195/lxml-6.1.3-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:1beb0f9909b26cee938df9ba56b15252a84429b1fc30ce6fca161390b9789a70", size = 4646516, upload-time = "2026-09-02T14:49:57.761Z" }, + { url = "https://files.pythonhosted.org/packages/56/7e/cfea25afafbe49db8b225764f7f74bb37c2a7f5e717d917d3d4a5e098ed4/lxml-6.1.3-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3a27ac6c780c8b8a1cd231b58407634cafc1c4cc28cd6c7141362df0f36351e7", size = 5240626, upload-time = "2026-09-02T14:50:00.279Z" }, + { url = "https://files.pythonhosted.org/packages/a1/75/7a587771bb52ebb0e2c57b6dbe9fd96a70fbb54d72ddd97d54c5f8ec18d5/lxml-6.1.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a1932d7ce78a561367512c594fe66eac2b2ec9b9264cfd9b5f950622f4a116e2", size = 5086619, upload-time = "2026-09-02T14:50:03.245Z" }, + { url = "https://files.pythonhosted.org/packages/1e/01/94c0ebe6d831861542d251e038052e52bf6d33f1d18f1cfffdc82851065a/lxml-6.1.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:7d0f5976aa2701996f759b30172925829867547bb073af0ae67d1307a0f0262c", size = 4758828, upload-time = "2026-09-02T14:50:05.873Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/938d67bd0e5b1fdfa52be28aefdffbad57e1f6b8e921c2aab88542c75f40/lxml-6.1.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e7ce578aa8a80910a72a8ca0bbea3baae10100827249001999726a788456d8", size = 5627083, upload-time = "2026-09-02T14:50:08.555Z" }, + { url = "https://files.pythonhosted.org/packages/d8/65/4e51522f6c214650db0abb7b16ccd11b1238b8a05a8d59aa4ebed59c9f67/lxml-6.1.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d97c5227621af74b111882a290b10f371780a38eef9d9e730408fba2259b52fb", size = 5235170, upload-time = "2026-09-02T14:50:11.255Z" }, + { url = "https://files.pythonhosted.org/packages/92/c2/e73d19365665f6b16ef84df21199befc3b06e4c539046ad2d9595f6fb9ea/lxml-6.1.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:da707f14ea3c35ee463d50acd596d6488e4b2b4ae7cf77a5bf93f55c023d63e8", size = 5252273, upload-time = "2026-09-02T14:50:13.782Z" }, + { url = "https://files.pythonhosted.org/packages/48/a9/7f386c84c9fe2854e1ca6e231c285e1c8f392971ac353c6865e6ec49faff/lxml-6.1.3-cp314-cp314t-win32.whl", hash = "sha256:9efe56a68179f3adc4de41861c9358931db03837c48dd5e1c78077b84dd07f3a", size = 3902712, upload-time = "2026-09-02T14:50:16.171Z" }, + { url = "https://files.pythonhosted.org/packages/82/a6/8a3eb793f7900ef01c7f99e6f5fcbcfbdff35251cfaef66b32a4c16352d6/lxml-6.1.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c9389b3784b56c58d933b5e0aecdf28f901b073ff385358d8a7d40907f6e14b2", size = 4400979, upload-time = "2026-09-02T14:50:18.621Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c4/3807bea283b4fe9e9d9f5dde46a73df91178472b335d2778e10b2a37aa22/lxml-6.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32a409be3190b088f960ac92bfedfbef2f86c49ff940765e1548177592d20026", size = 3823401, upload-time = "2026-09-02T14:50:21.119Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -490,6 +646,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/36/2ab7647fe1e84bba2baae7f04de241197eed62683fb3085e164de266d111/prek-0.4.1-py3-none-win_arm64.whl", hash = "sha256:5b4a348537924b20e208cbd87ef58e96ec37d691c5bec2969209c40de0ecf72e", size = 5423147, upload-time = "2026-05-20T04:27:17.023Z" }, ] +[[package]] +name = "prompt-toolkit" +version = "3.0.53" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/ea/39b988c938f75cb75d7045b5c69f8bfed47ee2152c8837fb403de29d6fb8/prompt_toolkit-3.0.53.tar.gz", hash = "sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6", size = 435492, upload-time = "2026-07-26T20:56:14.758Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/6f/84908cad2d6aa5144abcf7b42709fe4fdb459bc640ec7ac5786e7693dabc/prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2", size = 392288, upload-time = "2026-07-26T20:56:12.512Z" }, +] + [[package]] name = "psycopg" version = "3.3.4" @@ -534,6 +702,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pypdf" +version = "6.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/dc/34857a5e31cf708c163929f61a9ba4bd357a8850e49fc4e846ced527b51f/pypdf-6.17.0.tar.gz", hash = "sha256:097ad0d829778ec5b615aeaa5c6da4b6cac4992f8fd80b56f98a1a8c006573bb", size = 7018352, upload-time = "2026-09-04T11:30:44.256Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/08/1e9731038124a9127e1d27848952b86fb32b2f45f8f1b94adc7f0817a6ac/pypdf-6.17.0-py3-none-any.whl", hash = "sha256:5bd827266a21553b74d910e350131a6227b72f2ab4209bf372814b8195fa11c5", size = 388051, upload-time = "2026-09-04T11:30:42.681Z" }, +] + +[[package]] +name = "pypdfium2" +version = "5.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/78/a52cb80611339ec95f35c7a10d7bfe7a6f97f3b50a35a9f94283d062512e/pypdfium2-5.13.0.tar.gz", hash = "sha256:7ca2d8e31bd8d0d40c496416b7d8bea423388669ffd494929f50e8c3a82326b8", size = 273639, upload-time = "2026-08-13T10:58:15.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/9c/a49050af85055054299c7fab658ac63f8fddde575774aecbf8f71c7a9e5f/pypdfium2-5.13.0-py3-none-android_23_arm64_v8a.whl", hash = "sha256:882f4bbd4b17a335b43603169a14cde9341de12b238acd5c39e690cbca7c4293", size = 3417299, upload-time = "2026-08-13T10:57:40.522Z" }, + { url = "https://files.pythonhosted.org/packages/50/ad/f23027328843ee2bdd05afe16bb101f5906befd0c70de35fa8c53f60a5ff/pypdfium2-5.13.0-py3-none-android_23_armeabi_v7a.whl", hash = "sha256:d96929bde3bd64c771ab3558ca1ffd7704cc4d872ab92cd9f8f8b8a20f7f36b8", size = 2864708, upload-time = "2026-08-13T10:57:42.259Z" }, + { url = "https://files.pythonhosted.org/packages/08/99/1fe58428b69d2722dcbcfaa08ce71834a332c5b518fd58874bcef936b823/pypdfium2-5.13.0-py3-none-macosx_13_0_arm64.whl", hash = "sha256:da5c7b74eebf40b5c1fbe1de01aa1edc8827a79fb1efd999616bc20dcaf77ba4", size = 3507415, upload-time = "2026-08-13T10:57:43.978Z" }, + { url = "https://files.pythonhosted.org/packages/9f/41/06e26da88a4f5b4ed289325868717a186020661b7b221aa6df622711d31b/pypdfium2-5.13.0-py3-none-macosx_13_0_x86_64.whl", hash = "sha256:2abedfb5c70992b19c780ed58d7f7b929e8ce8ee52c9140158f44317c90ec6c7", size = 3670979, upload-time = "2026-08-13T10:57:45.607Z" }, + { url = "https://files.pythonhosted.org/packages/fe/31/f8210d53775f142be934336665b1d60e800c3f176f28c29b4908d945c518/pypdfium2-5.13.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ee8c2bb2e68b396ab4a763215ac100dacb6b96d0da5bebeb239a021aecc3a7e", size = 3676486, upload-time = "2026-08-13T10:57:47.267Z" }, + { url = "https://files.pythonhosted.org/packages/94/50/d339fa09fbe592564b100bfc76833170a1104a764a458ac2abfffcb632f2/pypdfium2-5.13.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07f58e91b8c45ca144a1ff3008faf3c73ef8a5e9fb32988831788363288228cd", size = 3400883, upload-time = "2026-08-13T10:57:49.189Z" }, + { url = "https://files.pythonhosted.org/packages/c3/e0/b10cf41b5e9f0212d014c40635659c6ab95bb4fcc6fc47f5d3c571f8d57f/pypdfium2-5.13.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46b2f5be9e7ae941ee4216e3d20b66f9dc3d81944a3d57756272de5275204709", size = 3803912, upload-time = "2026-08-13T10:57:50.865Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d8/25ba4ce9a9059ece82f4514df0658fde0aa9bbeafe135e76017c052bf56f/pypdfium2-5.13.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d96beb7f379e6c76d874ca93fcd182ac3168dd499056407070f9927fb1061b8e", size = 4218231, upload-time = "2026-08-13T10:57:52.525Z" }, + { url = "https://files.pythonhosted.org/packages/d3/7c/74a2fb48e5b0d2402d9ca64b39074c722d67e9a8a2c58449a843a8c2329a/pypdfium2-5.13.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81df25c1ab4c13ff773102d3cbea1967511d079123b067fc077bd0c4d57d91d8", size = 3730077, upload-time = "2026-08-13T10:57:54.021Z" }, + { url = "https://files.pythonhosted.org/packages/59/12/8c922f00518c26dc47d3676cc09c1d3c95e991c1977e31067d23cc2215cb/pypdfium2-5.13.0-py3-none-manylinux_2_27_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d66a32d89fa5b4a2715810171239eb194df4aba604727483ab760512f3c6a851", size = 4031512, upload-time = "2026-08-13T10:57:55.736Z" }, + { url = "https://files.pythonhosted.org/packages/c6/48/a171d034c2dac01adcc57d3dad3c97ba11f19d916f421176002c9e02c904/pypdfium2-5.13.0-py3-none-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b90b0a5ac310bb34db8eb848e58fcab4e201e124e3cf3cb1ccb7b85293e034af", size = 3995485, upload-time = "2026-08-13T10:57:57.39Z" }, + { url = "https://files.pythonhosted.org/packages/36/2e/dcb24776d409bb9e5b7fb26a0c62a87b98ab0e30dfcca645eaf31e35123b/pypdfium2-5.13.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ada81c36483cd61d07e32bc7814620ee96256b4f421b913f566861bf91800248", size = 5016636, upload-time = "2026-08-13T10:57:59.181Z" }, + { url = "https://files.pythonhosted.org/packages/93/24/1fab8470fc6de6f4481f009c90757b1a1ee0a61d8e864ed273f72ffca855/pypdfium2-5.13.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:3826e521e895648983cb9ee6b934d4bf51552600043984f84e9c2b3b14b696f3", size = 4555251, upload-time = "2026-08-13T10:58:00.753Z" }, + { url = "https://files.pythonhosted.org/packages/cd/ef/6e8dbea1eddcb55cf34172753ffccd39566333c803cc94d43c653f369f2f/pypdfium2-5.13.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5c029d7163a91f264eafab51fb442a84a33efd9fd83d5a06c0136a7857a3cc8d", size = 5263483, upload-time = "2026-08-13T10:58:02.48Z" }, + { url = "https://files.pythonhosted.org/packages/53/fe/2ff673730189a621c01f9193c74b0f6aa70d8740889fdf11949e1c541869/pypdfium2-5.13.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:be2dccbde0ce7efe334ecd8f348df4308db360756ede4f0821d82dfc9a58caa8", size = 5144135, upload-time = "2026-08-13T10:58:04.351Z" }, + { url = "https://files.pythonhosted.org/packages/19/0b/759b9037c007317fa5c990dd3f6eff2b99d3fbced251d1e2512be92f2e2e/pypdfium2-5.13.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:bcd81394fe101405e026eedb3e40bef84635c1e5d974dd6036420eb6937753c6", size = 4648156, upload-time = "2026-08-13T10:58:06.036Z" }, + { url = "https://files.pythonhosted.org/packages/db/3b/ffe29679c52efe8eb02d77aa6656e6d6201395423329af018ebd5923a3d0/pypdfium2-5.13.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:2ed32ff685f8e05e637c990bedbf5fca66727bf27718d8bc33eeab21ce0630d1", size = 5089852, upload-time = "2026-08-13T10:58:07.791Z" }, + { url = "https://files.pythonhosted.org/packages/7b/b6/cebacc1601ddfdcd1e6a1dc321533d215ceccf9b825fa9b91b11c6dc39fb/pypdfium2-5.13.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:9c777edba28d1d5fd15435ed3a78ee2fdb93dd069be37cb53b559bc122793770", size = 5074153, upload-time = "2026-08-13T10:58:09.396Z" }, + { url = "https://files.pythonhosted.org/packages/54/40/cf14c4f534f817788966857afdedb90002198dca5ce4fe2c6ecb031955ae/pypdfium2-5.13.0-py3-none-win32.whl", hash = "sha256:d33ee7077db67478b75efe4b5ea9610fb96c5416a0bc4949227f0f59c34dfcd9", size = 3753164, upload-time = "2026-08-13T10:58:10.97Z" }, + { url = "https://files.pythonhosted.org/packages/5d/99/a37b6b902457569468ed5908c94e56cb6c4032541f02cf89f723d42a9148/pypdfium2-5.13.0-py3-none-win_amd64.whl", hash = "sha256:47dcca2a8d507b5fd24f94c3c9d48fb379430f097bc20f01beff6c963ffbcedb", size = 3885553, upload-time = "2026-08-13T10:58:12.709Z" }, + { url = "https://files.pythonhosted.org/packages/50/7f/d39f6e64375c2ffd50ea100e3c73af79085c880c2791eb7203bc61d8913f/pypdfium2-5.13.0-py3-none-win_arm64.whl", hash = "sha256:554a0b23376460af1410e3c915906895e2dac67a086b9e6ccde0643a795d3b0d", size = 3700026, upload-time = "2026-08-13T10:58:14.206Z" }, +] + [[package]] name = "pytest" version = "9.0.3" @@ -574,6 +780,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-docx" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/f7/eddfe33871520adab45aaa1a71f0402a2252050c14c7e3009446c8f4701c/python_docx-1.2.0.tar.gz", hash = "sha256:7bc9d7b7d8a69c9c02ca09216118c86552704edc23bac179283f2e38f86220ce", size = 5723256, upload-time = "2025-06-16T20:46:27.921Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/00/1e03a4989fa5795da308cd774f05b704ace555a70f9bf9d3be057b680bcf/python_docx-1.2.0-py3-none-any.whl", hash = "sha256:3fd478f3250fbbbfd3b94fe1e985955737c145627498896a8a6bf81f4baf66c7", size = 252987, upload-time = "2025-06-16T20:46:22.506Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.2" @@ -721,6 +940,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, ] +[[package]] +name = "tzlocal" +version = "5.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/5b/879b2f932adfa7a053c360d50bc896c977fa6426109185f7c12ebdd0cb9d/tzlocal-5.4.4.tar.gz", hash = "sha256:8dbb8660838688a7b6ba4fed31d18dedf842afb4d47ca050d6d891c2c15f3be4", size = 31170, upload-time = "2026-06-29T08:03:40.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/a4/017a7a6cbe387d961a688ec31364ae60a5c4e22c96ae9921b79a947c855d/tzlocal-5.4.4-py3-none-any.whl", hash = "sha256:aae09f0126a8a86fa736be266eb4a471380d26a0de3bc14844e7821fee3e2a15", size = 18115, upload-time = "2026-06-29T08:03:38.666Z" }, +] + [[package]] name = "urllib3" version = "2.7.0" @@ -730,6 +961,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] +[[package]] +name = "vine" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" }, +] + [[package]] name = "wcwidth" version = "0.7.0" @@ -738,3 +978,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/2c/ee/afaf0f85a9a18fe47 wheels = [ { url = "https://files.pythonhosted.org/packages/41/52/e465037f5375f43533d1a80b6923955201596a99142ed524d77b571a1418/wcwidth-0.7.0-py3-none-any.whl", hash = "sha256:5d69154c429a82910e241c738cd0e2976fac8a2dd47a1a805f4afed1c0f136f2", size = 110825, upload-time = "2026-05-02T16:04:11.033Z" }, ] + +[[package]] +name = "whitenoise" +version = "6.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/2a/55b3f3a4ec326cd077c1c3defeee656b9298372a69229134d930151acd01/whitenoise-6.12.0.tar.gz", hash = "sha256:f723ebb76a112e98816ff80fcea0a6c9b8ecde835f8ddda25df7a30a3c2db6ad", size = 26841, upload-time = "2026-02-27T00:05:42.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/eb/d5583a11486211f3ebd4b385545ae787f32363d453c19fffd81106c9c138/whitenoise-6.12.0-py3-none-any.whl", hash = "sha256:fc5e8c572e33ebf24795b47b6a7da8da3c00cff2349f5b04c02f28d0cc5a3cc2", size = 20302, upload-time = "2026-02-27T00:05:40.086Z" }, +] diff --git a/website/__init__.py b/website/__init__.py index e69de29..1cb3ae6 100644 --- a/website/__init__.py +++ b/website/__init__.py @@ -0,0 +1,6 @@ +# Imported on every Django process start: makes the configured Celery app the +# current app so @shared_task resolution in web views gets the redis broker +# instead of Celery's default no-op amqp app. +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/website/celery.py b/website/celery.py index 1f030b8..3e272ee 100644 --- a/website/celery.py +++ b/website/celery.py @@ -9,7 +9,6 @@ app = Celery("website") app.config_from_object("django.conf:settings", namespace="CELERY") app.autodiscover_tasks() -print(app.conf.broker_url) app.conf.beat_schedule = { @@ -28,13 +27,9 @@ "task": "apps.spider.tasks.crawl_orc", "schedule": crontab(minute=0, hour=1), # 1AM }, - "crawl_timetable": { - "task": "apps.spider.tasks.crawl_timetable", - "schedule": crontab(minute=30, hour=1), # 1:30AM - }, - "crawl_medians": { - "task": "apps.spider.tasks.crawl_medians", - "schedule": crontab(minute=0, hour=2), # 2AM + "crawl_gc_course_offerings": { + "task": "apps.spider.tasks.crawl_gc_course_offerings", + "schedule": crontab(minute=15, hour=1), # 1:15AM }, "request_term_change": { "task": "apps.analytics.tasks.possibly_request_term_update", diff --git a/website/settings.py b/website/settings.py index d281cf3..5b993c6 100644 --- a/website/settings.py +++ b/website/settings.py @@ -14,6 +14,7 @@ "SECRET_KEY": None, "ALLOWED_HOSTS": ["127.0.0.1", "localhost"], "CORS_ALLOWED_ORIGINS": ["http://localhost:5173", "http://127.0.0.1:5173"], + "CSRF_TRUSTED_ORIGINS": [], "SESSION": { "COOKIE_AGE": 2592000, # 30 days "SAVE_EVERY_REQUEST": True, @@ -54,6 +55,19 @@ }, }, "AUTO_IMPORT_CRAWLED_DATA": True, + "SYLLABUS": { + "MAX_UPLOAD_SIZE": 20971520, # 20 MB + "ALLOWED_EXTENSIONS": [".pdf", ".docx"], + # AI analysis rejects uploads whose match score is below this. + "MIN_MATCH_SCORE": 60, + }, + "OLLAMA": { + "BASE_URL": "http://127.0.0.1:11434", + "MODEL": "qwen3.8:latest", + "TIMEOUT": 600, + "NUM_CTX": 32768, # 27B Q4 ~17GB weights; 262k ctx KV spills model to CPU + "MAX_PAGES": 30, + }, } config = Config(config_path=BASE_DIR / "config.yaml", defaults=DEFAULTS) @@ -68,6 +82,11 @@ DEBUG = config.get("DEBUG", cast=bool) ALLOWED_HOSTS = config.get("ALLOWED_HOSTS", cast=list) CORS_ALLOWED_ORIGINS = config.get("CORS_ALLOWED_ORIGINS", cast=list) +CSRF_TRUSTED_ORIGINS = config.get("CSRF_TRUSTED_ORIGINS", cast=list) + +# Requests arrive via Cloudflare Tunnel over HTTPS; make Django trust the +# forwarded proto so Secure cookies and is_secure() work correctly. +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") # --- Infrastructure --- DATABASES = {"default": dj_database_url.parse(config.get("DATABASE.URL"))} @@ -97,6 +116,12 @@ AUTO_IMPORT_CRAWLED_DATA = config.get("AUTO_IMPORT_CRAWLED_DATA", cast=bool) QUEST = config.get("QUEST") +SYLLABUS = config.get("SYLLABUS") +OLLAMA = config.get("OLLAMA") +CELERY_BROKER_URL = config.get("REDIS.URL") + +# Django admin error emails (empty by default; no mail backend configured) +ADMINS = [] # ============================================================================== @@ -125,6 +150,7 @@ MIDDLEWARE = [ "corsheaders.middleware.CorsMiddleware", "django.middleware.security.SecurityMiddleware", + "whitenoise.middleware.WhiteNoiseMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", @@ -152,7 +178,10 @@ } ] -STATIC_URL = "/dummy/" # Required by Django staticfiles but not used in this setup +STATIC_URL = "/static/" +STATIC_ROOT = "/app/staticfiles" +MEDIA_URL = "/media/" +MEDIA_ROOT = BASE_DIR / "media" # not served directly; downloads go through the API DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" LANGUAGE_CODE = "en-us"