Fix/timetable update

Makefile

@@ -1,5 +1,5 @@
 configure: venv
-	source ./venv/bin/activate && pip install -r requirements.dev.txt
+	source ./venv/bin/activate && pip install -r requirements.dev.txt -r requirements.txt
 
 venv:
 	python3.11 -m venv venv
@@ -14,3 +14,9 @@ format:
 	source ./venv/bin/activate && autoflake -r --in-place --remove-all-unused-imports ./migrations
 	source ./venv/bin/activate && isort ./migrations
 	source ./venv/bin/activate && black ./migrations
+
+run-main:
+	source ./venv/bin/activate && python3 main.py
+
+run-autoupdate:
+	source ./venv/bin/activate && python3 autoupdate.py

autoupdate.py

@@ -32,14 +32,14 @@
 
 lessons = parse_name(results)
 lessons, places, groups, teachers, subjects = parse_all(lessons)
-lessons = manual_edit(lessons)
+#lessons = manual_edit(lessons)
 lessons = multiple_lessons(lessons)
 lessons = flatten(lessons)
 lessons = all_to_array(lessons)
-completion(groups, places, teachers, headers, "prod")
+completion(groups, places, teachers, headers, "prod")#
 
 
-lessons = to_id(lessons, headers, "prod")
+lessons = to_id(lessons, headers, "prod")#
 conn = engine.connect()
 conn.execute(sa.text(f"""
 CREATE TABLE IF NOT EXISTS "{schema}".new(
@@ -110,15 +110,15 @@
 lessons_for_deleting = pd.read_sql_query(f"""select events_id from "{schema}".diff where action='delete'""", engine)
 lessons_for_creating = pd.read_sql_query(f"""select id, subject, "start", "end", "group", teacher, place, odd, even, weekday, num from "{schema}".diff where action='create'""", engine)
 
-begin = datetime.datetime(year=2024, month=9, day=1)
-end = datetime.datetime(year=2024, month=12, day=30)
+begin = datetime.datetime(year=2026, month=9, day=1)
+end = datetime.datetime(year=2026, month=12, day=30)
 begin = begin.strftime("%m/%d/%Y")
 end = end.strftime("%m/%d/%Y")
 for i, row in lessons_for_deleting.iterrows():
     for id in row["events_id"]:
-        if check_date(id, "prod", begin):
-            delete_lesson(headers, id, "prod")
-lessons_new = calc_date(lessons_for_creating, begin, end, "09/01/2024")
+        if check_date(id, "prod", begin):#
+            delete_lesson(headers, id, "prod")#
+lessons_new = calc_date(lessons_for_creating, begin, end, "08/25/2026")
 a = 1
 events = []
 for i, event in lessons_new.iterrows():
@@ -135,14 +135,17 @@
 batches = [events[i:i + 100] for i in range(0, len(events), 100)]
 batches_error = []
 for num, batch in enumerate(batches):
-    resp = requests.post("https://api.profcomff.com/timetable/event/bulk", json=batch, headers=headers)
+    resp = requests.post("https://api.profcomff.com/timetable/event/bulk", json=batch, headers=headers)#
     if int(resp.status_code) == 502:
         batches_error.append(batch)
     print(f"{num+1}/{len(batches)} -- {resp}")
     time.sleep(0.1)
 
 for num, batch in enumerate(batches_error):
-    resp = requests.post("https://api.profcomff.com/timetable/event/bulk", json=batch, headers=headers)
+    resp = requests.post("https://api.profcomff.com/timetable/event/bulk", json=batch, headers=headers)#
     print(f"error-bacthes: {num+1}/{len(batches)} -- {resp}")
     time.sleep(0.1)
 
+
+#DELETE FROM api_timetable.events_rooms WHERE event_id IN (SELECT id FROM api_timetable.event WHERE start_ts >= '2026-09-01' AND start_ts < '2026-12-31');
+#UPDATE api_timetable.event SET is_deleted = true WHERE start_ts >= '2026-09-01' AND start_ts < '2026-12-31';

main.py

@@ -2,15 +2,17 @@
 import pandas as pd
 import logging
 import requests as r
+import os, sqlalchemy as sa
+from dotenv import load_dotenv
 
 # [[курс, поток, количество групп], ...]
 SOURCES = [
-    [1, 1, 6], [1, 2, 6], [1, 3, 6],
-    [2, 1, 6], [2, 2, 6], [2, 3, 6],
+    [1, 1, 9], [1, 2, 9],
+    [2, 1, 9], [2, 2, 9],
     [3, 1, 10], [3, 2, 8],
-    [4, 1, 10], [4, 2, 10],
-    [5, 1, 13], [5, 2, 11],
-    [6, 1, 11], [6, 2, 10]
+    [4, 1, 10], [4, 2, 8],
+    [5, 1, 12], [5, 2, 11],
+    [6, 1, 13], [6, 2, 10]
 ]
 
 USER_AGENT = "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 " \
@@ -28,13 +30,8 @@
             'raw_html': response.text,
         })
 timetables = pd.DataFrame(data)
-results = pd.DataFrame()
-for i, row in timetables.iterrows():
-    results = pd.concat([results, parse_timetable(row["raw_html"])])
-
-# ---------------- Parsing ----------------
-lessons = parse_name(results)
-lessons, places, groups, teachers, subjects = parse_all(lessons)
-lessons = multiple_lessons(lessons)
-lessons = flatten(lessons)
-lessons = all_to_array(lessons)
+load_dotenv()
+engine = sa.create_engine(
+    f"postgresql://{os.getenv('user')}:{os.getenv('password')}@{os.getenv('host')}/{os.getenv('database')}"
+)
+timetables.to_sql("raw_html", engine, schema="public", if_exists="replace", index=False)

profcomff_parse_lib/timetable/core/parse_group.py

@@ -13,19 +13,13 @@ def _parse_group(group):
     """
     name_group = r"[А-Яа-яёЁ \.,\-:]"
     number_group = r"\d{3}\w{0,2}"
-    # # заменяем 1xxmA на 5xxA
-    result = re.findall(r"([1]\d\d[м]([А-Яа-яёЁ:]|))", group)
-    if result is not None:
-        for string in result:
-            letter = string[-1]
-            group = group.replace(string[0], f'5{string[0][1:3]}{letter}')
-    # заменяем 6xxA на 2xxмA
-    result = re.findall(r"([6]\d\d([А-Яа-яёЁ:]|))", group)
-    if result is not None:
-        for string in result:
-            group = f'2{string[0][1:3]}м'
+    # магистратура: 1xxм -> 5xx, 2xxм -> 6xx
+    def _norm(m):
+        return ("5" if m.group(1) == "1" else "6") + m.group(2) + (m.group(3) or "")
+
+    group = re.sub(r"([12])(\d\d)\s*[мМmM]([А-Яа-яёЁ]{0,2})", _norm, group)
     # '113 ...'
-    result = re.match(rf"(\d{3}\w{0,2}) *([А-Яа-яёЁ: ]*)", group)
+    result = re.match(r"(\d{3}\w{0,2}) *([А-Яа-яёЁ: ]*)", group)
     if not (result is None):
         if group == result[0]:
             return [(result[1], result[2])]

profcomff_parse_lib/timetable/core/parse_name.py

@@ -96,7 +96,21 @@ def _parse_name(name):
             parsed_name["teacher"] = result[3]
             return parsed_name
 
-
+    # Ручные исключения: строки, которые не берёт ни одна регулярка.
+    SPECIAL_CASES = {
+        "Мега-установки и 4D-физика материалов (Дисциплина по выбору)": {
+            "subject": "Мега-установки и 4D-физика материалов (Дисциплина по выбору)",
+            "teacher": "Орешко А. П.",
+            "place": "СФА",
+        },
+        "422 - ЛАБОРАТОРИЯ СПЕЦИАЛИЗАЦИИ проф. Биленко И. А. ст. преп. Косых Т. Б.": {
+            "subject": "ЛАБОРАТОРИЯ СПЕЦИАЛИЗАЦИИ",
+            "teacher": "Биленко И. А.",
+            "place": None,
+        },
+    }
+    if name.strip() in SPECIAL_CASES:
+        return dict(SPECIAL_CASES[name.strip()])
 
     _logger.warning(f"Для '{name}' не найдено подходящее регулярное выражение.")
     return {"subject": name, "teacher": None, "place": None}

profcomff_parse_lib/timetable/core/parse_subjects.py

@@ -165,6 +165,9 @@ def _parse_subjects(group, subject, is_dash=False):
         if subject == result[0]:
             return result[2]
 
+    if subject.strip() == "Мега-установки и 4D-физика материалов (Дисциплина по выбору)":
+        return "Мега-установки и 4D-физика материалов (Дисциплина по выбору)"
+
     _logger.warning(f"Для '{subject}' не найдено подходящее регулярное выражение.")
     return subject
 

requirements.dev.txt

@@ -2,4 +2,4 @@ setuptools
 psycopg2-binary
 python-dotenv
 openpyxl
-sqlalchemy
+sqlalchemy

requirements.txt

@@ -0,0 +1,4 @@
+requests
+pandas<3
+beautifulsoup4
+retrying