diff --git a/Makefile b/Makefile index cecaed4..026b71f 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ configure: venv - source ./venv/bin/activate && pip install -r requirements.dev.txt + source ./venv/bin/activate && pip install -r requirements.dev.txt -r requirements.txt venv: python3.11 -m venv venv @@ -14,3 +14,9 @@ format: source ./venv/bin/activate && autoflake -r --in-place --remove-all-unused-imports ./migrations source ./venv/bin/activate && isort ./migrations source ./venv/bin/activate && black ./migrations + +run-main: + source ./venv/bin/activate && python3 main.py + +run-autoupdate: + source ./venv/bin/activate && python3 autoupdate.py \ No newline at end of file diff --git a/autoupdate.py b/autoupdate.py index 8f469e0..3165386 100644 --- a/autoupdate.py +++ b/autoupdate.py @@ -32,14 +32,14 @@ lessons = parse_name(results) lessons, places, groups, teachers, subjects = parse_all(lessons) -lessons = manual_edit(lessons) +#lessons = manual_edit(lessons) lessons = multiple_lessons(lessons) lessons = flatten(lessons) lessons = all_to_array(lessons) -completion(groups, places, teachers, headers, "prod") +completion(groups, places, teachers, headers, "prod")# -lessons = to_id(lessons, headers, "prod") +lessons = to_id(lessons, headers, "prod")# conn = engine.connect() conn.execute(sa.text(f""" CREATE TABLE IF NOT EXISTS "{schema}".new( @@ -110,15 +110,15 @@ lessons_for_deleting = pd.read_sql_query(f"""select events_id from "{schema}".diff where action='delete'""", engine) lessons_for_creating = pd.read_sql_query(f"""select id, subject, "start", "end", "group", teacher, place, odd, even, weekday, num from "{schema}".diff where action='create'""", engine) -begin = datetime.datetime(year=2024, month=9, day=1) -end = datetime.datetime(year=2024, month=12, day=30) +begin = datetime.datetime(year=2026, month=9, day=1) +end = datetime.datetime(year=2026, month=12, day=30) begin = begin.strftime("%m/%d/%Y") end = end.strftime("%m/%d/%Y") for i, row in lessons_for_deleting.iterrows(): for id in row["events_id"]: - if check_date(id, "prod", begin): - delete_lesson(headers, id, "prod") -lessons_new = calc_date(lessons_for_creating, begin, end, "09/01/2024") + if check_date(id, "prod", begin):# + delete_lesson(headers, id, "prod")# +lessons_new = calc_date(lessons_for_creating, begin, end, "08/25/2026") a = 1 events = [] for i, event in lessons_new.iterrows(): @@ -135,14 +135,17 @@ batches = [events[i:i + 100] for i in range(0, len(events), 100)] batches_error = [] for num, batch in enumerate(batches): - resp = requests.post("https://api.profcomff.com/timetable/event/bulk", json=batch, headers=headers) + resp = requests.post("https://api.profcomff.com/timetable/event/bulk", json=batch, headers=headers)# if int(resp.status_code) == 502: batches_error.append(batch) print(f"{num+1}/{len(batches)} -- {resp}") time.sleep(0.1) for num, batch in enumerate(batches_error): - resp = requests.post("https://api.profcomff.com/timetable/event/bulk", json=batch, headers=headers) + resp = requests.post("https://api.profcomff.com/timetable/event/bulk", json=batch, headers=headers)# print(f"error-bacthes: {num+1}/{len(batches)} -- {resp}") time.sleep(0.1) + +#DELETE FROM api_timetable.events_rooms WHERE event_id IN (SELECT id FROM api_timetable.event WHERE start_ts >= '2026-09-01' AND start_ts < '2026-12-31'); +#UPDATE api_timetable.event SET is_deleted = true WHERE start_ts >= '2026-09-01' AND start_ts < '2026-12-31'; \ No newline at end of file diff --git a/main.py b/main.py index dde155e..5d89ce7 100644 --- a/main.py +++ b/main.py @@ -2,15 +2,17 @@ import pandas as pd import logging import requests as r +import os, sqlalchemy as sa +from dotenv import load_dotenv # [[курс, поток, количество групп], ...] SOURCES = [ - [1, 1, 6], [1, 2, 6], [1, 3, 6], - [2, 1, 6], [2, 2, 6], [2, 3, 6], + [1, 1, 9], [1, 2, 9], + [2, 1, 9], [2, 2, 9], [3, 1, 10], [3, 2, 8], - [4, 1, 10], [4, 2, 10], - [5, 1, 13], [5, 2, 11], - [6, 1, 11], [6, 2, 10] + [4, 1, 10], [4, 2, 8], + [5, 1, 12], [5, 2, 11], + [6, 1, 13], [6, 2, 10] ] USER_AGENT = "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 " \ @@ -28,13 +30,8 @@ 'raw_html': response.text, }) timetables = pd.DataFrame(data) -results = pd.DataFrame() -for i, row in timetables.iterrows(): - results = pd.concat([results, parse_timetable(row["raw_html"])]) - -# ---------------- Parsing ---------------- -lessons = parse_name(results) -lessons, places, groups, teachers, subjects = parse_all(lessons) -lessons = multiple_lessons(lessons) -lessons = flatten(lessons) -lessons = all_to_array(lessons) +load_dotenv() +engine = sa.create_engine( + f"postgresql://{os.getenv('user')}:{os.getenv('password')}@{os.getenv('host')}/{os.getenv('database')}" +) +timetables.to_sql("raw_html", engine, schema="public", if_exists="replace", index=False) diff --git a/profcomff_parse_lib/timetable/core/parse_group.py b/profcomff_parse_lib/timetable/core/parse_group.py index f075e77..2b6a6bc 100644 --- a/profcomff_parse_lib/timetable/core/parse_group.py +++ b/profcomff_parse_lib/timetable/core/parse_group.py @@ -13,19 +13,13 @@ def _parse_group(group): """ name_group = r"[А-Яа-яёЁ \.,\-:]" number_group = r"\d{3}\w{0,2}" - # # заменяем 1xxmA на 5xxA - result = re.findall(r"([1]\d\d[м]([А-Яа-яёЁ:]|))", group) - if result is not None: - for string in result: - letter = string[-1] - group = group.replace(string[0], f'5{string[0][1:3]}{letter}') - # заменяем 6xxA на 2xxмA - result = re.findall(r"([6]\d\d([А-Яа-яёЁ:]|))", group) - if result is not None: - for string in result: - group = f'2{string[0][1:3]}м' + # магистратура: 1xxм -> 5xx, 2xxм -> 6xx + def _norm(m): + return ("5" if m.group(1) == "1" else "6") + m.group(2) + (m.group(3) or "") + + group = re.sub(r"([12])(\d\d)\s*[мМmM]([А-Яа-яёЁ]{0,2})", _norm, group) # '113 ...' - result = re.match(rf"(\d{3}\w{0,2}) *([А-Яа-яёЁ: ]*)", group) + result = re.match(r"(\d{3}\w{0,2}) *([А-Яа-яёЁ: ]*)", group) if not (result is None): if group == result[0]: return [(result[1], result[2])] diff --git a/profcomff_parse_lib/timetable/core/parse_name.py b/profcomff_parse_lib/timetable/core/parse_name.py index 894be22..f5638a5 100644 --- a/profcomff_parse_lib/timetable/core/parse_name.py +++ b/profcomff_parse_lib/timetable/core/parse_name.py @@ -96,7 +96,21 @@ def _parse_name(name): parsed_name["teacher"] = result[3] return parsed_name - + # Ручные исключения: строки, которые не берёт ни одна регулярка. + SPECIAL_CASES = { + "Мега-установки и 4D-физика материалов (Дисциплина по выбору)": { + "subject": "Мега-установки и 4D-физика материалов (Дисциплина по выбору)", + "teacher": "Орешко А. П.", + "place": "СФА", + }, + "422 - ЛАБОРАТОРИЯ СПЕЦИАЛИЗАЦИИ проф. Биленко И. А. ст. преп. Косых Т. Б.": { + "subject": "ЛАБОРАТОРИЯ СПЕЦИАЛИЗАЦИИ", + "teacher": "Биленко И. А.", + "place": None, + }, + } + if name.strip() in SPECIAL_CASES: + return dict(SPECIAL_CASES[name.strip()]) _logger.warning(f"Для '{name}' не найдено подходящее регулярное выражение.") return {"subject": name, "teacher": None, "place": None} diff --git a/profcomff_parse_lib/timetable/core/parse_subjects.py b/profcomff_parse_lib/timetable/core/parse_subjects.py index 474cbff..4c98f01 100644 --- a/profcomff_parse_lib/timetable/core/parse_subjects.py +++ b/profcomff_parse_lib/timetable/core/parse_subjects.py @@ -165,6 +165,9 @@ def _parse_subjects(group, subject, is_dash=False): if subject == result[0]: return result[2] + if subject.strip() == "Мега-установки и 4D-физика материалов (Дисциплина по выбору)": + return "Мега-установки и 4D-физика материалов (Дисциплина по выбору)" + _logger.warning(f"Для '{subject}' не найдено подходящее регулярное выражение.") return subject diff --git a/requirements.dev.txt b/requirements.dev.txt index 0b0eb5a..2763f1b 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -2,4 +2,4 @@ setuptools psycopg2-binary python-dotenv openpyxl -sqlalchemy +sqlalchemy \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e69de29..be90668 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,4 @@ +requests +pandas<3 +beautifulsoup4 +retrying \ No newline at end of file