Documentação disponível em /docs
+ + + """ + + +@app.get("/api", tags=["root"]) +def api_info(): """Endpoint raiz com informações da API.""" return { "message": "API de Cursos da UAlg", diff --git a/src/scrape_ualg.py b/src/scrape_ualg.py index 23f1e24..e5104c0 100644 --- a/src/scrape_ualg.py +++ b/src/scrape_ualg.py @@ -15,6 +15,7 @@ import os import time import logging +import re from urllib.parse import urljoin, urlparse from typing import Optional, List, Dict from .config import Config @@ -222,44 +223,143 @@ def parse_course_page(self, soup: BeautifulSoup, url: str) -> Dict: data["title"] = h1.get_text(strip=True) if h1 else "Título não encontrado" # Descrição - desc = soup.select_one(".field--name-body, .text, .course-description") + desc = soup.select_one(".field--name-body, .text, .course-description, .description") data["description"] = desc.get_text(separator="\n", strip=True) if desc else None # Código do curso - code_elem = soup.select_one(".field--name-field-code, .course-code") + code_elem = soup.select_one(".field--name-field-code, .course-code, .code") data["code"] = code_elem.get_text(strip=True) if code_elem else None # Nível (Licenciatura, Mestrado, etc.) - level_elem = soup.select_one(".field--name-field-level, .course-level") + level_elem = soup.select_one(".field--name-field-level, .course-level, .level, .tipo-curso") data["level"] = level_elem.get_text(strip=True) if level_elem else None # Escola/Faculdade - school_elem = soup.select_one(".field--name-field-school, .course-school") + school_elem = soup.select_one(".field--name-field-school, .course-school, .school, .escola") data["school"] = school_elem.get_text(strip=True) if school_elem else None # Idioma - lang_elem = soup.select_one(".field--name-field-language, .course-language") + lang_elem = soup.select_one(".field--name-field-language, .course-language, .language, .idioma") data["language"] = lang_elem.get_text(strip=True) if lang_elem else None # Regime - regime_elem = soup.select_one(".field--name-field-regime, .course-regime") + regime_elem = soup.select_one(".field--name-field-regime, .course-regime, .regime") data["regime"] = regime_elem.get_text(strip=True) if regime_elem else None # Modalidade - modality_elem = soup.select_one(".field--name-field-modality, .course-modality") + modality_elem = soup.select_one(".field--name-field-modality, .course-modality, .modality, .modalidade") data["modality"] = modality_elem.get_text(strip=True) if modality_elem else None + # Áreas de conhecimento + areas = [] + area_selectors = [ + ".field--name-field-area, .course-area, .area, .area-conhecimento", + ".field--name-field-areas", + ".areas-conhecimento", + ] + for selector in area_selectors: + area_elems = soup.select(selector) + for elem in area_elems: + area_text = elem.get_text(strip=True) + if area_text and area_text not in areas: + areas.append(area_text) + data["areas"] = areas + + # Módulos/Unidades Curriculares (UCs) + modules = [] + + # Procurar por tabelas de plano curricular + tables = soup.select("table.plano-curricular, table.curriculum, table.modules, table") + for table in tables: + rows = table.find_all("tr") + for row in rows[1:]: # Skip header + cells = row.find_all(["td", "th"]) + if len(cells) >= 2: + module = { + "code": None, + "title": None, + "ects": None, + "year": None, + "semester": None, + } + + # Extrair informações das células + for i, cell in enumerate(cells): + cell_text = cell.get_text(strip=True) + if not cell_text: + continue + + # Heurísticas para identificar o tipo de dado + if i == 0 or "código" in cell_text.lower() or "code" in cell_text.lower(): + # Possível código + if len(cell_text) < 20 and not module["code"]: + module["code"] = cell_text + elif i == 1 or "disciplina" in cell_text.lower() or "uc" in cell_text.lower(): + # Possível título + if len(cell_text) > 3 and not module["title"]: + module["title"] = cell_text + + # ECTS + if "ects" in cell_text.lower(): + try: + ects_match = re.search(r"(\d+(?:\.\d+)?)", cell_text) + if ects_match: + module["ects"] = float(ects_match.group(1)) + except (ValueError, AttributeError): + pass + + # Ano + if "ano" in cell_text.lower() or "year" in cell_text.lower(): + try: + year_match = re.search(r"(\d+)", cell_text) + if year_match: + module["year"] = int(year_match.group(1)) + except (ValueError, AttributeError): + pass + + # Semestre + if "semestre" in cell_text.lower() or "semester" in cell_text.lower(): + try: + sem_match = re.search(r"(\d+)", cell_text) + if sem_match: + module["semester"] = int(sem_match.group(1)) + except (ValueError, AttributeError): + pass + + # Adicionar módulo se tiver pelo menos título + if module["title"]: + modules.append(module) + + # Também procurar por listas de UCs + uc_lists = soup.select("ul.ucs, ul.modules, .curriculum-list") + for ul in uc_lists: + items = ul.find_all("li") + for item in items: + text = item.get_text(strip=True) + if text and len(text) > 3: + modules.append( + { + "code": None, + "title": text, + "ects": None, + "year": None, + "semester": None, + } + ) + + data["modules"] = modules + # Documentos (PDFs) documents = [] for link in soup.find_all("a", href=True): href = urljoin(url, link["href"]) - if href.lower().endswith(".pdf"): + if href.lower().endswith((".pdf", ".doc", ".docx")): title = link.get_text(strip=True) or "Documento" documents.append((title, href)) data["documents"] = documents - logger.info(f"Curso parseado: {data['title']}") + logger.info(f"Curso parseado: {data['title']} (áreas: {len(areas)}, módulos: {len(modules)}, docs: {len(documents)})") return data def get_course_links(self, soup: BeautifulSoup) -> List[str]: @@ -339,6 +439,60 @@ def scrape_all_courses(self, limit: Optional[int] = None) -> None: course_id = self.save_course(conn, course_data) if course_id: + # Salvar áreas de conhecimento + for area_name in course_data.get("areas", []): + area_id = self.upsert_area(conn, area_name) + if area_id: + # Link curso-área + cur = conn.cursor() + cur.execute( + "INSERT OR IGNORE INTO course_area(course_id, area_id) VALUES (?, ?)", (course_id, area_id) + ) + conn.commit() + + # Salvar módulos/UCs + for module_data in course_data.get("modules", []): + if not module_data.get("title"): + continue + + cur = conn.cursor() + # Inserir ou recuperar módulo + cur.execute( + """ + INSERT OR IGNORE INTO modules(code, title, ects, year, semester) + VALUES (?, ?, ?, ?, ?) + """, + ( + module_data.get("code"), + module_data.get("title"), + module_data.get("ects"), + module_data.get("year"), + module_data.get("semester"), + ), + ) + conn.commit() + + # Recuperar ID do módulo + if module_data.get("code"): + cur.execute( + "SELECT id FROM modules WHERE code=? AND title=?", + (module_data.get("code"), module_data.get("title")), + ) + else: + cur.execute( + "SELECT id FROM modules WHERE title=? AND code IS NULL", (module_data.get("title"),) + ) + + result = cur.fetchone() + if result: + module_id = result[0] + # Link módulo-curso + cur.execute( + "INSERT OR IGNORE INTO module_course(module_id, course_id, mandatory) VALUES (?, ?, ?)", + (module_id, course_id, True), + ) + conn.commit() + # Salvar documentos for title, doc_url in course_data.get("documents", []): self.save_document(conn, course_id, title, doc_url) diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..9f11eaf --- /dev/null +++ b/templates/index.html @@ -0,0 +1,482 @@ + + + + + +Universidade do Algarve - Oferta Formativa
+Descrição do curso de teste
+| Código | +Disciplina | +ECTS | +
|---|---|---|
| CS101 | +Programação I | +6 ECTS | +
| CS102 | +Algoritmos | +5 ECTS | +
Descrição
+ + + """ + + mock_response = Mock() + mock_response.text = listing_html + mock_response.raise_for_status = Mock() + + mock_course_response = Mock() + mock_course_response.text = course_html + mock_course_response.raise_for_status = Mock() + + # Primeira chamada retorna listagem, demais retornam página de curso + mock_get.side_effect = [mock_response] + [mock_course_response] * 3 + + # Executar scraping com limite + scraper.scrape_all_courses(limit=2) + + # Verificar que apenas 2 cursos foram processados (além da página inicial) + assert mock_get.call_count == 3 # 1 listagem + 2 cursos + + @patch("src.scrape_ualg.requests.Session.get") + def test_scrape_all_courses_error_handling(self, mock_get, scraper, test_db_path): + """Testa tratamento de erros durante scraping.""" + scraper.init_db() + + # Mock erro na página inicial (simular exceção de rede) + mock_get.side_effect = requests.RequestException("Connection error") + + # Não deve lançar exceção + scraper.scrape_all_courses(limit=1)