From cac78133a705ea95cafec384fc0c15bfa97b86fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E6=9D=B0=20525370910098?= Date: Fri, 4 Sep 2026 00:35:15 +0800 Subject: [PATCH] fix(spider): parse restructured GC course detail pages GC site moved course detail fields (topics as

s, not

  • s) inside nested wrapper divs, breaking the direct-children parser: only course code/title survived, and each crawl overwrote stored descriptions and topics with empty values. Parse the flattened text stream of the detail block, grouping lines by section marker, so field extraction no longer depends on child layout or list tags. Skip import of pages whose structure was not recognized so future layout changes cannot wipe existing course data again. Verified against live GC pages: 6 random courses parse correctly (up to 18 topics, descriptions, prereqs, instructors, credits); the previous parser extracted none of these fields. Co-Authored-By: Claude Code --- apps/spider/crawlers/orc.py | 155 ++++++++++++++++++++---------------- 1 file changed, 85 insertions(+), 70 deletions(-) diff --git a/apps/spider/crawlers/orc.py b/apps/spider/crawlers/orc.py index aba3fdd..bf55907 100644 --- a/apps/spider/crawlers/orc.py +++ b/apps/spider/crawlers/orc.py @@ -64,80 +64,95 @@ def _crawl_course_data(course_url): return None # Return early if no h2 element found course_heading = course_heading_element.get_text() - if course_heading: - split_course_heading = course_heading.split(" – ") - children = list(soup.find_all(class_="et_pb_text_inner")[3].children) - - course_code = split_course_heading[0] - department = re.findall(r"^([A-Z]{2,4})\d+", course_code)[0] - number = re.findall(r"^[A-Z]{2,4}(\d{3})", course_code)[0] - course_title = split_course_heading[1] - - course_credits = 0 - pre_requisites = "" - description = "" - course_topics = [] - instructors = [] - - for i, child in enumerate(children): - text = child.get_text(strip=True) if hasattr(child, "get_text") else "" - if "Credits:" in text: - course_credits = int(re.findall(r"\d+", text)[0]) - elif "Pre-requisites:" in text: - pre_requisites = extract_prerequisites(text) - elif "Description:" in text: - description = ( - children[i + 2].get_text(strip=True) - if i + 2 < len(children) - else "" - ) - if description == "\n" or "Course Topics" in description: - description = "" - elif "Course Topics:" in text: - course_topics = ( - [li.get_text(strip=True) for li in children[i + 2].find_all("li")] - if i + 2 < len(children) - else [] - ) - elif "Instructors:" in text: - instructors_text = ( - children[i + 2].get_text(strip=True) - if i + 2 < len(children) - else "" - ) - instructors = [ - name.strip() for name in instructors_text.split(";") if name.strip() - ] - - result = { - "course_code": course_code, - "course_title": course_title, - "department": department, - "number": number, - "course_credits": course_credits, - "pre_requisites": pre_requisites, - "description": description, - "course_topics": course_topics, - "instructors": instructors, - "url": course_url, - } - return result - # return { - # "course_code": "QWER1234J", - # "course_title": "Test Course", - # "department": "QWER", - # "number": 1234, - # "course_credits": 4, - # "pre_requisites": None, - # "description": "This is a test course", - # "course_topics": ["Test Topic"], - # "instructors": ["Test Instructor"], - # "url": course_url, - # } + if not course_heading: + return None + split_course_heading = course_heading.split(" – ") + if len(split_course_heading) < 2: + return None + + course_code = split_course_heading[0] + department = re.findall(r"^([A-Z]{2,4})\d+", course_code)[0] + number = re.findall(r"^[A-Z]{2,4}(\d{3})", course_code)[0] + course_title = split_course_heading[1] + + # The GC site wraps the labelled fields in nested divs (and lists topics + # as plain

    s rather than

  • s), so parse the flattened text stream of + # the detail block instead of assuming direct children. + content_sections = soup.find_all(class_="et_pb_text_inner") + if len(content_sections) < 4: + return None + content_lines = [ + line.strip() + for line in content_sections[3].get_text(separator="\n", strip=True).split("\n") + if line.strip() + ] + + section_markers = ( + "Instructors:", + "Credits:", + "Pre-requisites:", + "Description:", + "Course Topics:", + ) + current_section = None + section_lines = {} + structure_recognized = False + for line in content_lines: + marker = next((m for m in section_markers if m in line), None) + if marker is not None: + current_section = marker + structure_recognized = True + elif current_section is not None: + section_lines.setdefault(current_section, []).append(line) + + course_credits = 0 + if "Credits:" in section_lines: + credits_match = re.findall(r"\d+", " ".join(section_lines["Credits:"])) + if credits_match: + course_credits = int(credits_match[0]) + pre_requisites = "" + if "Pre-requisites:" in section_lines: + pre_requisites = extract_prerequisites( + "Pre-requisites: " + " ".join(section_lines["Pre-requisites:"]) + ) + description = " ".join(section_lines.get("Description:", [])).strip() + course_topics = list(section_lines.get("Course Topics:", [])) + instructors = [] + for name_line in section_lines.get("Instructors:", []): + instructors.extend( + name.strip() for name in name_line.split(";") if name.strip() + ) + + return { + "course_code": course_code, + "course_title": course_title, + "department": department, + "number": number, + "course_credits": course_credits, + "pre_requisites": pre_requisites, + "description": description, + "course_topics": course_topics, + "instructors": instructors, + "url": course_url, + # False when the page carried none of the known section markers; used + # by import_department to skip updates so a broken crawl cannot wipe + # existing course data. Stored records without this key default to + # recognized for backward compatibility. + "structure_recognized": structure_recognized, + } def import_department(department_data): for course_data in department_data: + # Skip pages whose structure was not recognized: importing them would + # overwrite stored course fields with empty values. + if not course_data.get("structure_recognized", True): + print( + "Skipping {}: unrecognized page structure".format( + course_data.get("course_code", "") + ) + ) + continue course, created = Course.objects.update_or_create( course_code=course_data["course_code"], defaults={