diff --git a/apps/spider/crawlers/orc.py b/apps/spider/crawlers/orc.py index aba3fdd..bf55907 100644 --- a/apps/spider/crawlers/orc.py +++ b/apps/spider/crawlers/orc.py @@ -64,80 +64,95 @@ def _crawl_course_data(course_url): return None # Return early if no h2 element found course_heading = course_heading_element.get_text() - if course_heading: - split_course_heading = course_heading.split(" – ") - children = list(soup.find_all(class_="et_pb_text_inner")[3].children) - - course_code = split_course_heading[0] - department = re.findall(r"^([A-Z]{2,4})\d+", course_code)[0] - number = re.findall(r"^[A-Z]{2,4}(\d{3})", course_code)[0] - course_title = split_course_heading[1] - - course_credits = 0 - pre_requisites = "" - description = "" - course_topics = [] - instructors = [] - - for i, child in enumerate(children): - text = child.get_text(strip=True) if hasattr(child, "get_text") else "" - if "Credits:" in text: - course_credits = int(re.findall(r"\d+", text)[0]) - elif "Pre-requisites:" in text: - pre_requisites = extract_prerequisites(text) - elif "Description:" in text: - description = ( - children[i + 2].get_text(strip=True) - if i + 2 < len(children) - else "" - ) - if description == "\n" or "Course Topics" in description: - description = "" - elif "Course Topics:" in text: - course_topics = ( - [li.get_text(strip=True) for li in children[i + 2].find_all("li")] - if i + 2 < len(children) - else [] - ) - elif "Instructors:" in text: - instructors_text = ( - children[i + 2].get_text(strip=True) - if i + 2 < len(children) - else "" - ) - instructors = [ - name.strip() for name in instructors_text.split(";") if name.strip() - ] - - result = { - "course_code": course_code, - "course_title": course_title, - "department": department, - "number": number, - "course_credits": course_credits, - "pre_requisites": pre_requisites, - "description": description, - "course_topics": course_topics, - "instructors": instructors, - "url": course_url, - } - return result - # return { - # "course_code": "QWER1234J", - # "course_title": "Test Course", - # "department": "QWER", - # "number": 1234, - # "course_credits": 4, - # "pre_requisites": None, - # "description": "This is a test course", - # "course_topics": ["Test Topic"], - # "instructors": ["Test Instructor"], - # "url": course_url, - # } + if not course_heading: + return None + split_course_heading = course_heading.split(" – ") + if len(split_course_heading) < 2: + return None + + course_code = split_course_heading[0] + department = re.findall(r"^([A-Z]{2,4})\d+", course_code)[0] + number = re.findall(r"^[A-Z]{2,4}(\d{3})", course_code)[0] + course_title = split_course_heading[1] + + # The GC site wraps the labelled fields in nested divs (and lists topics + # as plain

s rather than

  • s), so parse the flattened text stream of + # the detail block instead of assuming direct children. + content_sections = soup.find_all(class_="et_pb_text_inner") + if len(content_sections) < 4: + return None + content_lines = [ + line.strip() + for line in content_sections[3].get_text(separator="\n", strip=True).split("\n") + if line.strip() + ] + + section_markers = ( + "Instructors:", + "Credits:", + "Pre-requisites:", + "Description:", + "Course Topics:", + ) + current_section = None + section_lines = {} + structure_recognized = False + for line in content_lines: + marker = next((m for m in section_markers if m in line), None) + if marker is not None: + current_section = marker + structure_recognized = True + elif current_section is not None: + section_lines.setdefault(current_section, []).append(line) + + course_credits = 0 + if "Credits:" in section_lines: + credits_match = re.findall(r"\d+", " ".join(section_lines["Credits:"])) + if credits_match: + course_credits = int(credits_match[0]) + pre_requisites = "" + if "Pre-requisites:" in section_lines: + pre_requisites = extract_prerequisites( + "Pre-requisites: " + " ".join(section_lines["Pre-requisites:"]) + ) + description = " ".join(section_lines.get("Description:", [])).strip() + course_topics = list(section_lines.get("Course Topics:", [])) + instructors = [] + for name_line in section_lines.get("Instructors:", []): + instructors.extend( + name.strip() for name in name_line.split(";") if name.strip() + ) + + return { + "course_code": course_code, + "course_title": course_title, + "department": department, + "number": number, + "course_credits": course_credits, + "pre_requisites": pre_requisites, + "description": description, + "course_topics": course_topics, + "instructors": instructors, + "url": course_url, + # False when the page carried none of the known section markers; used + # by import_department to skip updates so a broken crawl cannot wipe + # existing course data. Stored records without this key default to + # recognized for backward compatibility. + "structure_recognized": structure_recognized, + } def import_department(department_data): for course_data in department_data: + # Skip pages whose structure was not recognized: importing them would + # overwrite stored course fields with empty values. + if not course_data.get("structure_recognized", True): + print( + "Skipping {}: unrecognized page structure".format( + course_data.get("course_code", "") + ) + ) + continue course, created = Course.objects.update_or_create( course_code=course_data["course_code"], defaults={