补齐通知日期写法并保留编号证据

crawler/src/crawler/parse/dates.py

@@ -10,28 +10,49 @@
 
 _CLOCK = r"(?:[0-9]{1,2}[:\uff1a][0-9]{2}|[0-9]{1,2}[时点](?:[0-9]{1,2}分?)?)"
 _WEEKDAY = r"[\uff08(]?(?:星期|周)[一二三四五六日天][\uff09)]?"
-_GAP = rf"(?:[ \t]*(?:{_WEEKDAY}|上午|中午|下午|晚上))*[ \t]*"
+_PERIOD = r"(?:上午|中午|下午|晚上)"
+_GAP = rf"(?:[ \t]*(?:{_WEEKDAY}|{_PERIOD}))*[ \t]*"
+_HAN_DIGITS = "零\u3007\u25cb一二三四五六七八九"
+_DIGITS = dict(zip(_HAN_DIGITS, "000123456789", strict=True))
+_CHINESE_NUMBER = r"(?:[一二三四五六七八九]?十[一二三四五六七八九]?|[一二三四五六七八九])"
+_NUMBER = rf"(?:[0-9]{{1,2}}|{_CHINESE_NUMBER})"
+_YEAR = rf"(?:[0-9]{{4}}|[{_HAN_DIGITS}]{{4}})"
+_MARK = r"(?:[ \t]*(?:前后|左右|之前|前))?"
+_TIME = rf"(?:{_GAP}{_CLOCK}|[ \t]*止[ \t]*[\uff08(]{_CLOCK}[\uff09)])?"
 # Spaces are allowed between components, but a line break ends an expression.
-_DAY = (
-    rf"[0-9]{{1,2}}[ \t]*日(?:{_GAP}{_CLOCK}|[ \t]*止[ \t]*[\uff08(]{_CLOCK}[\uff09)])?"
-    r"(?:[ \t]*(?:前后|左右|之前|前))?"
-)
+_DAY = rf"{_NUMBER}[ \t]*[日号]{_TIME}{_MARK}"
+_MONTH_START = rf"(?:{_YEAR}[ \t]*年[ \t]*)?{_NUMBER}[ \t]*月"
 _MONTH = (
-    rf"(?:[0-9]{{4}}[ \t]*年[ \t]*)?[0-9]{{1,2}}[ \t]*月"
+    rf"{_MONTH_START}"
     rf"(?:[ \t]*(?:{_DAY}|中下旬|上旬|中旬|下旬|初|底))?"
 )
-_ATOM = rf"(?:即日起|{_MONTH})"
+_NUMERIC_DATE = (
+    r"(?<![A-Za-z0-9/])[0-9]{4}(?:-[0-9]{1,2}-|/[0-9]{1,2}/|\.[0-9]{1,2}\.)[0-9]{1,2}"
+    r"(?![0-9年月]|[./][0-9])"
+)
+_NUMERIC = rf"{_NUMERIC_DATE}[日号]?{_TIME}{_MARK}"
+_DATED = rf"(?:{_MONTH_START}[ \t]*{_NUMBER}[ \t]*[日号]|{_NUMERIC_DATE}[日号]?)"
+_PAREN_DATE = (
+    rf"(?:\([ \t]*{_DATED}[ \t]*\)|\uff08[ \t]*{_DATED}[ \t]*\uff09)"
+    rf"{_GAP}{_CLOCK}{_MARK}"
+)
+_ATOM = rf"(?:即日起|{_PAREN_DATE}|{_NUMERIC}|{_MONTH})"
 _SEPARATOR = r"(?:至|到|—+|\u2013|\uff0d|\uff5e|~|-)"
+_HAN_BOUNDARY = (
+    rf"(?:(?<![{_HAN_DIGITS}十])|(?=[^{_HAN_DIGITS}十])"
+    r"|(?<=周[一二三四五六日天])|(?<=星期[一二三四五六日天]))"
+)
 _EXPRESSION = re.compile(
-    rf"(?<![0-9年月])(?P<first>(?:[0-9]{{4}}[ \t]*年[ \t]*)?"
-    rf"[0-9]{{1,2}}[ \t]*月[ \t]*[0-9]{{1,2}}(?=[ \t]*{_SEPARATOR}[ \t]*{_DAY})|{_ATOM})"
+    rf"(?<![0-9年月]){_HAN_BOUNDARY}"
+    rf"(?P<first>{_MONTH_START}[ \t]*{_NUMBER}"
+    rf"(?=[ \t]*{_SEPARATOR}[ \t]*(?:{_ATOM}|{_DAY}))|{_ATOM})"
     rf"(?:[ \t]*{_SEPARATOR}[ \t]*(?P<last>{_ATOM}|{_DAY}|{_GAP}{_CLOCK}(?:前后|左右|之前|前)?))?"
-    r"(?!(?<=[0-9])[0-9])(?!(?<=月)[ \t]*[0-9])"
+    r"(?!(?<=[0-9])[0-9])(?!(?<=月)[ \t]*[0-9一二三四五六七八九十])"
 )
 _POINT = re.compile(
     r"(?:(?P<year>[0-9]{4})年)?(?:(?P<month>[0-9]{1,2})月)?"
     r"(?:(?P<day>[0-9]{1,2})日)?"
-    r"(?P<period>上午|中午|下午|晚上)?"
+    rf"(?P<period>{_PERIOD})?"
     r"(?:(?P<hour>[0-9]{1,2})[:\uff1a](?P<minute>[0-9]{2})|"
     r"(?P<clock_hour>[0-9]{1,2})[时点](?:(?P<clock_minute>[0-9]{1,2})分?)?)?"
     r"(?P<part>上旬|中旬|下旬|初|底)?(?P<mark>前后|左右|之前|前)?"
@@ -56,6 +77,14 @@ class DateMatch:
     issue: Literal["missing_publication", "invalid_date"] | None = None
 
 
+def _chinese_number(match: re.Match[str]) -> str:
+    number = match[0]
+    if "十" not in number:
+        return "".join(_DIGITS[character] for character in number)
+    tens, _, units = number.partition("十")
+    return str(int(_DIGITS[tens] if tens else "1") * 10 + int(_DIGITS[units] if units else "0"))
+
+
 def _calendar(value: TimeValue) -> date:
     if isinstance(value, TimeToMonth):
         year, month = map(int, value.month.split("-"))
@@ -79,9 +108,27 @@ def _point(
     previous: TimeValue | None = None,
     inherited_period: str | None = None,
 ) -> TimeValue | None:
+    # Add calendar units before removing delimiters so a clock digit cannot become part of the day.
+    numeric = re.match(
+        r"(?P<opening>[\uff08(][ \t]*)?"
+        r"(?P<year>[0-9]{4})[-/.]"
+        r"(?P<month>[0-9]{1,2})[-/.]"
+        r"(?P<day>[0-9]{1,2})[日号]?",
+        text,
+    )
+    if numeric is not None:
+        text = (
+            f"{numeric['opening'] or ''}{numeric['year']}年{numeric['month']}月{numeric['day']}日"
+            f"{text[numeric.end() :]}"
+        )
     compact = re.sub(r"[ \t]", "", text)
     compact = re.sub(_WEEKDAY, "", compact)
     compact = re.sub(rf"止[\uff08(]({_CLOCK})[\uff09)]$", r"\1", compact)
+    if re.fullmatch(rf"{_MONTH_START}{_NUMBER}", compact):
+        compact += "日"
+    compact = re.sub(rf"[{_HAN_DIGITS}十]+(?=[年月日号])", _chinese_number, compact)
+    compact = re.sub(rf"^[\uff08(]([^\uff09)]+)[\uff09)](?={_PERIOD}|[0-9])", r"\1", compact)
+    compact = compact.replace("号", "日")
     if compact == "即日起":
         return TimeToDay(date=published_on) if published_on is not None else None
     match = _POINT.fullmatch(compact)
@@ -147,19 +194,20 @@ def find_dates(text: str, published_on: date | None) -> list[DateMatch]:
                 if last is None:
                     last = first.removesuffix("中下旬") + "下旬"
                 first = first.removesuffix("中下旬") + "中旬"
-            if first[-1].isdigit() and "月" in first and "日" not in first:
-                first += "日"
             value = _point(first, published_on)
             if value is None:
                 issue = "missing_publication"
             if last is not None:
-                period = re.search(r"上午|中午|下午|晚上", first)
+                period = re.search(_PERIOD, first)
                 end_value = _point(last, published_on, value, period[0] if period else None)
                 if end_value is None:
                     issue = "missing_publication"
                 elif value is not None and _earliest(end_value) < _earliest(value):
                     raise ValueError("Reversed date range")
-            if re.match(rf"[ \t]*{_SEPARATOR}[ \t]*[0-9]", text[match.end() :]):
+            if re.match(
+                rf"[ \t]*{_SEPARATOR}[ \t]*(?:[0-9]|[{_HAN_DIGITS}十百千万]+[ \t]*[年月日号])",
+                text[match.end() :],
+            ):
                 issue = "invalid_date"
             if re.match(r"[ \t]*[0-9]{1,2}[:\uff1a][0-9]", text[match.end() :]):
                 issue = "invalid_date"

crawler/src/crawler/parse/dates_test.py

@@ -23,14 +23,18 @@ def test_dates_can_be_followed_immediately_by_numbers(text: str, days: list[int]
 @pytest.mark.parametrize(
     "text",
     [
-        "6月1日-6月14号",
+        "6月1日-6月31号",
         "6月1日8:00-17:0",
         "6月1日8:00至17:001",
         "6月1日-6月14日17:0",
         "6月14日17:001",
-        "6月1日到14号",
-        "6月1日—14号",
-        "6月1日\uff5e14号",
+        "6月1日到31号",
+        "6月1日—31号",
+        "6月1日\uff5e31号",
+        "六月一日至零日",
+        "六月一日至一百日",
+        "六月一日至一百号",
+        "6月1日至6.30",
     ],
 )
 def test_unrecognized_numeric_range_end_cannot_leave_a_valid_start(text: str) -> None:
@@ -46,6 +50,13 @@ def test_dash_before_prose_still_allows_an_ordinary_date() -> None:
     assert found.issue is None
 
 
+def test_chinese_floor_after_a_date_is_not_an_invalid_range_endpoint() -> None:
+    [found] = find_dates("六月一日至一楼报到", date(2027, 5, 1))
+    assert found.text == "六月一日"
+    assert found.value == TimeToDay(date=date(2027, 6, 1))
+    assert found.issue is None
+
+
 def test_mid_to_late_month_does_not_replace_an_explicit_range_end() -> None:
     [found] = find_dates("7月中下旬至8月上旬", date(2027, 5, 1))
     assert found.text == "7月中下旬至8月上旬"
@@ -364,3 +375,215 @@ def test_before_a_clock_time_keeps_the_minute_value_and_original_qualifier(
     assert found.value.time == clock
     assert found.text == text
     assert found.issue is None
+
+
+@pytest.mark.parametrize(
+    ("expression", "expected"),
+    [
+        ("2027-06-30", TimeToDay(date=date(2027, 6, 30))),
+        ("2027/6/30", TimeToDay(date=date(2027, 6, 30))),
+        ("2027.6.30", TimeToDay(date=date(2027, 6, 30))),
+        ("2027-06-30下午5点前", TimeToMinute(date=date(2027, 6, 30), time="17:00")),
+        ("6月30号", TimeToDay(date=date(2027, 6, 30))),
+        ("6月30号24时", TimeToMinute(date=date(2027, 6, 30), time="24:00")),
+        ("二\u3007二七年八月五日", TimeToDay(date=date(2027, 8, 5))),
+        ("二零二七年十二月三十一号", TimeToDay(date=date(2027, 12, 31))),
+        ("二\u3007二八年二月二十九日", TimeToDay(date=date(2028, 2, 29))),
+        ("八月上旬", TimeToMonth(month="2027-08", part="early")),
+        ("九月中旬", TimeToMonth(month="2027-09", part="mid")),
+        ("十一月底", TimeToMonth(month="2027-11", part="late")),
+    ],
+)
+def test_additional_date_spellings_preserve_source_and_precision(
+    expression: str, expected: TimeValue
+) -> None:
+    text = f"报名时间为{expression}\uff0c请及时提交。"
+    [found] = find_dates(text, date(2027, 5, 1))
+    assert found.value == expected
+    assert found.end_value is None
+    assert found.issue is None
+    assert found.text == expression
+    assert text[found.start : found.end] == expression
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "6.30",
+        "报名截止6/30",
+        "\uff086-30\uff0917:00",
+        "6.1至6.30",
+    ],
+)
+def test_short_numeric_dates_are_not_recognized_from_context(text: str) -> None:
+    assert find_dates(text, date(2027, 5, 1)) == []
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "Python 3.14",
+        "一二三月",
+        "一二六月一日",
+    ],
+)
+def test_unrelated_numeric_expressions_are_not_promoted_to_dates(text: str) -> None:
+    assert find_dates(text, date(2027, 5, 1)) == []
+
+
+@pytest.mark.parametrize("prefix", ["1.", "2. ", "3.\t"])
+def test_numbered_list_items_do_not_hide_a_chinese_date(prefix: str) -> None:
+    text = prefix + "7月1日00:00至7月7日24:00报名。"
+    [found] = find_dates(text, date(2027, 5, 1))
+    assert found.text == "7月1日00:00至7月7日24:00"
+    assert found.value == TimeToMinute(date=date(2027, 7, 1), time="00:00")
+    assert found.end_value == TimeToMinute(date=date(2027, 7, 7), time="24:00")
+
+
+@pytest.mark.parametrize(
+    ("text", "start", "end"),
+    [
+        ("6月1日-6月14号", date(2027, 6, 1), date(2027, 6, 14)),
+        ("6月1号到14号", date(2027, 6, 1), date(2027, 6, 14)),
+        ("2027-06-01-2027-06-14", date(2027, 6, 1), date(2027, 6, 14)),
+        ("2027/6/1至6月14号", date(2027, 6, 1), date(2027, 6, 14)),
+        ("二\u3007二七年十二月二十八日至一月十日", date(2027, 12, 28), date(2028, 1, 10)),
+        ("六月一至三十号", date(2027, 6, 1), date(2027, 6, 30)),
+        ("二\u3007二七年六月一至三十日", date(2027, 6, 1), date(2027, 6, 30)),
+        ("十二月二十八至一月十日", date(2027, 12, 28), date(2028, 1, 10)),
+        ("6月一至三十日", date(2027, 6, 1), date(2027, 6, 30)),
+    ],
+)
+def test_added_date_formats_share_range_and_year_rules(text: str, start: date, end: date) -> None:
+    [found] = find_dates(text, date(2027, 5, 1))
+    assert found.value == TimeToDay(date=start)
+    assert found.end_value == TimeToDay(date=end)
+    assert found.issue is None
+
+
+@pytest.mark.parametrize(
+    ("text", "expected_day", "expected_time"),
+    [
+        ("2027-6-1 17:00", 1, "17:00"),
+        ("2027.6.9 17点30分", 9, "17:30"),
+        ("(2027/6/1)17:00", 1, "17:00"),
+    ],
+)
+def test_numeric_day_does_not_absorb_the_first_clock_digit(
+    text: str, expected_day: int, expected_time: str
+) -> None:
+    [found] = find_dates(text, date(2027, 5, 1))
+    assert found.value == TimeToMinute(date=date(2027, 6, expected_day), time=expected_time)
+    assert found.issue is None
+    assert text[found.start : found.end] == found.text
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "2027-6-1 08:30至2027-6-9 17:00",
+        "(2027/6/1)08:30至(2027/6/9)17:00",
+    ],
+)
+def test_numeric_range_endpoints_preserve_the_clock_boundary(text: str) -> None:
+    [found] = find_dates(text, date(2027, 5, 1))
+    assert found.value == TimeToMinute(date=date(2027, 6, 1), time="08:30")
+    assert found.end_value == TimeToMinute(date=date(2027, 6, 9), time="17:00")
+    assert found.issue is None
+
+
+@pytest.mark.parametrize("text", ["六月三十号", "八月上旬"])
+def test_new_yearless_formats_still_require_publication(text: str) -> None:
+    [found] = find_dates(text, None)
+    assert found.issue == "missing_publication"
+    assert found.value is None
+    assert found.end_value is None
+
+
+@pytest.mark.parametrize("text", ["2027-06-30", "2027/6/30", "2027.6.30"])
+def test_complete_numeric_dates_do_not_require_publication(text: str) -> None:
+    [found] = find_dates(text, None)
+    assert found.value == TimeToDay(date=date(2027, 6, 30))
+    assert found.issue is None
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "2027-02-29",
+        "2027/6/31",
+        "二\u3007二七年二月二十九日",
+        "6月31号",
+        "2027-06-30至2027-06-01",
+        "2027-06-30 24:01",
+    ],
+)
+def test_new_formats_preserve_invalid_date_and_range_failure(text: str) -> None:
+    [found] = find_dates(text, date(2027, 5, 1))
+    assert found.issue == "invalid_date"
+    assert found.value is None
+    assert found.end_value is None
+
+
+@pytest.mark.parametrize(
+    ("expression", "expected"),
+    [
+        ("\uff086月20日\uff0900:00", TimeToMinute(date=date(2027, 6, 20), time="00:00")),
+        ("(6月20日)08:30", TimeToMinute(date=date(2027, 6, 20), time="08:30")),
+        (
+            "\uff08二\u3007二七年六月二十日\uff09下午2点前",
+            TimeToMinute(date=date(2027, 6, 20), time="14:00"),
+        ),
+        ("(2027/6/20) 24时", TimeToMinute(date=date(2027, 6, 20), time="24:00")),
+        ("( 2027/6/20 ) 24时", TimeToMinute(date=date(2027, 6, 20), time="24:00")),
+    ],
+)
+def test_parenthesized_dates_keep_a_clock_directly_after_the_closing_parenthesis(
+    expression: str, expected: TimeToMinute
+) -> None:
+    text = f"报到安排为{expression}\uff0c请提前准备。"
+    [found] = find_dates(text, date(2027, 5, 1))
+    assert found.text == expression
+    assert text[found.start : found.end] == expression
+    assert found.value == expected
+    assert found.issue is None
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "\uff086月20日\uff09\n08:30报到",
+        "6月20日\uff08注意事项\uff0908:30报到",
+        "\uff08见6月20日\uff0908:30报到",
+    ],
+)
+def test_clocks_are_not_joined_across_prose_or_line_breaks(text: str) -> None:
+    [found] = find_dates(text, date(2027, 5, 1))
+    assert found.text == "6月20日"
+    assert found.value == TimeToDay(date=date(2027, 6, 20))
+
+
+@pytest.mark.parametrize("weekday", ["周一", "星期一", "星期六"])
+@pytest.mark.parametrize("expression", ["6月1日17:00", "六月一日17:00"])
+def test_a_weekday_immediately_before_a_date_does_not_hide_it(
+    weekday: str, expression: str
+) -> None:
+    text = f"{weekday}{expression}截止"
+    [found] = find_dates(text, date(2027, 5, 1))
+    assert found.text == expression
+    assert found.value == TimeToMinute(date=date(2027, 6, 1), time="17:00")
+
+
+@pytest.mark.parametrize(
+    ("text", "expression"),
+    [
+        ("1.2027.6.30截止", "2027.6.30"),
+        ("说明。1. 2027/6/30报名", "2027/6/30"),
+    ],
+)
+def test_numbered_lists_can_start_with_a_complete_numeric_date(text: str, expression: str) -> None:
+    [found] = find_dates(text, date(2027, 5, 1))
+    assert found.text == expression
+    assert text[found.start : found.end] == expression
+    assert found.value == TimeToDay(date=date(2027, 6, 30))
+    assert found.issue is None

crawler/src/crawler/parse/indexing_test.py

@@ -120,8 +120,11 @@ def test_date_ids_link_exact_source_with_values_and_attachment() -> None:
         )
 
 
-def test_dates_in_addresses_are_excluded_but_anchor_labels_are_kept() -> None:
-    text = "https://example.org/2027年6月1日。6月18日截止。"
+@pytest.mark.parametrize(
+    "url", ["https://example.org/2027年6月1日", "https://example.org/?date=2027-06-01"]
+)
+def test_dates_in_addresses_are_excluded_but_anchor_labels_are_kept(url: str) -> None:
+    text = f"{url}。6月18日截止。"
     start = text.index("6月18日")
     index = index_documents(
         [Document(text, hyperlinks=(Hyperlink(start, start + 5, "https://example.org/notice"),))],