diff --git a/apps/spider/crawlers/orc.py b/apps/spider/crawlers/orc.py index aba3fdd..bf55907 100644 --- a/apps/spider/crawlers/orc.py +++ b/apps/spider/crawlers/orc.py @@ -64,80 +64,95 @@ def _crawl_course_data(course_url): return None # Return early if no h2 element found course_heading = course_heading_element.get_text() - if course_heading: - split_course_heading = course_heading.split(" – ") - children = list(soup.find_all(class_="et_pb_text_inner")[3].children) - - course_code = split_course_heading[0] - department = re.findall(r"^([A-Z]{2,4})\d+", course_code)[0] - number = re.findall(r"^[A-Z]{2,4}(\d{3})", course_code)[0] - course_title = split_course_heading[1] - - course_credits = 0 - pre_requisites = "" - description = "" - course_topics = [] - instructors = [] - - for i, child in enumerate(children): - text = child.get_text(strip=True) if hasattr(child, "get_text") else "" - if "Credits:" in text: - course_credits = int(re.findall(r"\d+", text)[0]) - elif "Pre-requisites:" in text: - pre_requisites = extract_prerequisites(text) - elif "Description:" in text: - description = ( - children[i + 2].get_text(strip=True) - if i + 2 < len(children) - else "" - ) - if description == "\n" or "Course Topics" in description: - description = "" - elif "Course Topics:" in text: - course_topics = ( - [li.get_text(strip=True) for li in children[i + 2].find_all("li")] - if i + 2 < len(children) - else [] - ) - elif "Instructors:" in text: - instructors_text = ( - children[i + 2].get_text(strip=True) - if i + 2 < len(children) - else "" - ) - instructors = [ - name.strip() for name in instructors_text.split(";") if name.strip() - ] - - result = { - "course_code": course_code, - "course_title": course_title, - "department": department, - "number": number, - "course_credits": course_credits, - "pre_requisites": pre_requisites, - "description": description, - "course_topics": course_topics, - "instructors": instructors, - "url": course_url, - } - return result - # return { - # "course_code": "QWER1234J", - # "course_title": "Test Course", - # "department": "QWER", - # "number": 1234, - # "course_credits": 4, - # "pre_requisites": None, - # "description": "This is a test course", - # "course_topics": ["Test Topic"], - # "instructors": ["Test Instructor"], - # "url": course_url, - # } + if not course_heading: + return None + split_course_heading = course_heading.split(" – ") + if len(split_course_heading) < 2: + return None + + course_code = split_course_heading[0] + department = re.findall(r"^([A-Z]{2,4})\d+", course_code)[0] + number = re.findall(r"^[A-Z]{2,4}(\d{3})", course_code)[0] + course_title = split_course_heading[1] + + # The GC site wraps the labelled fields in nested divs (and lists topics + # as plain
s rather than