diff --git a/crawler/src/crawler/parse/dates.py b/crawler/src/crawler/parse/dates.py index 18536ef..5fe1fb9 100644 --- a/crawler/src/crawler/parse/dates.py +++ b/crawler/src/crawler/parse/dates.py @@ -10,28 +10,49 @@ _CLOCK = r"(?:[0-9]{1,2}[:\uff1a][0-9]{2}|[0-9]{1,2}[时点](?:[0-9]{1,2}分?)?)" _WEEKDAY = r"[\uff08(]?(?:星期|周)[一二三四五六日天][\uff09)]?" -_GAP = rf"(?:[ \t]*(?:{_WEEKDAY}|上午|中午|下午|晚上))*[ \t]*" +_PERIOD = r"(?:上午|中午|下午|晚上)" +_GAP = rf"(?:[ \t]*(?:{_WEEKDAY}|{_PERIOD}))*[ \t]*" +_HAN_DIGITS = "零\u3007\u25cb一二三四五六七八九" +_DIGITS = dict(zip(_HAN_DIGITS, "000123456789", strict=True)) +_CHINESE_NUMBER = r"(?:[一二三四五六七八九]?十[一二三四五六七八九]?|[一二三四五六七八九])" +_NUMBER = rf"(?:[0-9]{{1,2}}|{_CHINESE_NUMBER})" +_YEAR = rf"(?:[0-9]{{4}}|[{_HAN_DIGITS}]{{4}})" +_MARK = r"(?:[ \t]*(?:前后|左右|之前|前))?" +_TIME = rf"(?:{_GAP}{_CLOCK}|[ \t]*止[ \t]*[\uff08(]{_CLOCK}[\uff09)])?" # Spaces are allowed between components, but a line break ends an expression. -_DAY = ( - rf"[0-9]{{1,2}}[ \t]*日(?:{_GAP}{_CLOCK}|[ \t]*止[ \t]*[\uff08(]{_CLOCK}[\uff09)])?" - r"(?:[ \t]*(?:前后|左右|之前|前))?" -) +_DAY = rf"{_NUMBER}[ \t]*[日号]{_TIME}{_MARK}" +_MONTH_START = rf"(?:{_YEAR}[ \t]*年[ \t]*)?{_NUMBER}[ \t]*月" _MONTH = ( - rf"(?:[0-9]{{4}}[ \t]*年[ \t]*)?[0-9]{{1,2}}[ \t]*月" + rf"{_MONTH_START}" rf"(?:[ \t]*(?:{_DAY}|中下旬|上旬|中旬|下旬|初|底))?" ) -_ATOM = rf"(?:即日起|{_MONTH})" +_NUMERIC_DATE = ( + r"(?(?:[0-9]{{4}}[ \t]*年[ \t]*)?" - rf"[0-9]{{1,2}}[ \t]*月[ \t]*[0-9]{{1,2}}(?=[ \t]*{_SEPARATOR}[ \t]*{_DAY})|{_ATOM})" + rf"(?{_MONTH_START}[ \t]*{_NUMBER}" + rf"(?=[ \t]*{_SEPARATOR}[ \t]*(?:{_ATOM}|{_DAY}))|{_ATOM})" rf"(?:[ \t]*{_SEPARATOR}[ \t]*(?P{_ATOM}|{_DAY}|{_GAP}{_CLOCK}(?:前后|左右|之前|前)?))?" - r"(?!(?<=[0-9])[0-9])(?!(?<=月)[ \t]*[0-9])" + r"(?!(?<=[0-9])[0-9])(?!(?<=月)[ \t]*[0-9一二三四五六七八九十])" ) _POINT = re.compile( r"(?:(?P[0-9]{4})年)?(?:(?P[0-9]{1,2})月)?" r"(?:(?P[0-9]{1,2})日)?" - r"(?P上午|中午|下午|晚上)?" + rf"(?P{_PERIOD})?" r"(?:(?P[0-9]{1,2})[:\uff1a](?P[0-9]{2})|" r"(?P[0-9]{1,2})[时点](?:(?P[0-9]{1,2})分?)?)?" r"(?P上旬|中旬|下旬|初|底)?(?P前后|左右|之前|前)?" @@ -56,6 +77,14 @@ class DateMatch: issue: Literal["missing_publication", "invalid_date"] | None = None +def _chinese_number(match: re.Match[str]) -> str: + number = match[0] + if "十" not in number: + return "".join(_DIGITS[character] for character in number) + tens, _, units = number.partition("十") + return str(int(_DIGITS[tens] if tens else "1") * 10 + int(_DIGITS[units] if units else "0")) + + def _calendar(value: TimeValue) -> date: if isinstance(value, TimeToMonth): year, month = map(int, value.month.split("-")) @@ -79,9 +108,27 @@ def _point( previous: TimeValue | None = None, inherited_period: str | None = None, ) -> TimeValue | None: + # Add calendar units before removing delimiters so a clock digit cannot become part of the day. + numeric = re.match( + r"(?P[\uff08(][ \t]*)?" + r"(?P[0-9]{4})[-/.]" + r"(?P[0-9]{1,2})[-/.]" + r"(?P[0-9]{1,2})[日号]?", + text, + ) + if numeric is not None: + text = ( + f"{numeric['opening'] or ''}{numeric['year']}年{numeric['month']}月{numeric['day']}日" + f"{text[numeric.end() :]}" + ) compact = re.sub(r"[ \t]", "", text) compact = re.sub(_WEEKDAY, "", compact) compact = re.sub(rf"止[\uff08(]({_CLOCK})[\uff09)]$", r"\1", compact) + if re.fullmatch(rf"{_MONTH_START}{_NUMBER}", compact): + compact += "日" + compact = re.sub(rf"[{_HAN_DIGITS}十]+(?=[年月日号])", _chinese_number, compact) + compact = re.sub(rf"^[\uff08(]([^\uff09)]+)[\uff09)](?={_PERIOD}|[0-9])", r"\1", compact) + compact = compact.replace("号", "日") if compact == "即日起": return TimeToDay(date=published_on) if published_on is not None else None match = _POINT.fullmatch(compact) @@ -147,19 +194,20 @@ def find_dates(text: str, published_on: date | None) -> list[DateMatch]: if last is None: last = first.removesuffix("中下旬") + "下旬" first = first.removesuffix("中下旬") + "中旬" - if first[-1].isdigit() and "月" in first and "日" not in first: - first += "日" value = _point(first, published_on) if value is None: issue = "missing_publication" if last is not None: - period = re.search(r"上午|中午|下午|晚上", first) + period = re.search(_PERIOD, first) end_value = _point(last, published_on, value, period[0] if period else None) if end_value is None: issue = "missing_publication" elif value is not None and _earliest(end_value) < _earliest(value): raise ValueError("Reversed date range") - if re.match(rf"[ \t]*{_SEPARATOR}[ \t]*[0-9]", text[match.end() :]): + if re.match( + rf"[ \t]*{_SEPARATOR}[ \t]*(?:[0-9]|[{_HAN_DIGITS}十百千万]+[ \t]*[年月日号])", + text[match.end() :], + ): issue = "invalid_date" if re.match(r"[ \t]*[0-9]{1,2}[:\uff1a][0-9]", text[match.end() :]): issue = "invalid_date" diff --git a/crawler/src/crawler/parse/dates_test.py b/crawler/src/crawler/parse/dates_test.py index 1ba5cd5..56509fd 100644 --- a/crawler/src/crawler/parse/dates_test.py +++ b/crawler/src/crawler/parse/dates_test.py @@ -23,14 +23,18 @@ def test_dates_can_be_followed_immediately_by_numbers(text: str, days: list[int] @pytest.mark.parametrize( "text", [ - "6月1日-6月14号", + "6月1日-6月31号", "6月1日8:00-17:0", "6月1日8:00至17:001", "6月1日-6月14日17:0", "6月14日17:001", - "6月1日到14号", - "6月1日—14号", - "6月1日\uff5e14号", + "6月1日到31号", + "6月1日—31号", + "6月1日\uff5e31号", + "六月一日至零日", + "六月一日至一百日", + "六月一日至一百号", + "6月1日至6.30", ], ) def test_unrecognized_numeric_range_end_cannot_leave_a_valid_start(text: str) -> None: @@ -46,6 +50,13 @@ def test_dash_before_prose_still_allows_an_ordinary_date() -> None: assert found.issue is None +def test_chinese_floor_after_a_date_is_not_an_invalid_range_endpoint() -> None: + [found] = find_dates("六月一日至一楼报到", date(2027, 5, 1)) + assert found.text == "六月一日" + assert found.value == TimeToDay(date=date(2027, 6, 1)) + assert found.issue is None + + def test_mid_to_late_month_does_not_replace_an_explicit_range_end() -> None: [found] = find_dates("7月中下旬至8月上旬", date(2027, 5, 1)) assert found.text == "7月中下旬至8月上旬" @@ -364,3 +375,215 @@ def test_before_a_clock_time_keeps_the_minute_value_and_original_qualifier( assert found.value.time == clock assert found.text == text assert found.issue is None + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + ("2027-06-30", TimeToDay(date=date(2027, 6, 30))), + ("2027/6/30", TimeToDay(date=date(2027, 6, 30))), + ("2027.6.30", TimeToDay(date=date(2027, 6, 30))), + ("2027-06-30下午5点前", TimeToMinute(date=date(2027, 6, 30), time="17:00")), + ("6月30号", TimeToDay(date=date(2027, 6, 30))), + ("6月30号24时", TimeToMinute(date=date(2027, 6, 30), time="24:00")), + ("二\u3007二七年八月五日", TimeToDay(date=date(2027, 8, 5))), + ("二零二七年十二月三十一号", TimeToDay(date=date(2027, 12, 31))), + ("二\u3007二八年二月二十九日", TimeToDay(date=date(2028, 2, 29))), + ("八月上旬", TimeToMonth(month="2027-08", part="early")), + ("九月中旬", TimeToMonth(month="2027-09", part="mid")), + ("十一月底", TimeToMonth(month="2027-11", part="late")), + ], +) +def test_additional_date_spellings_preserve_source_and_precision( + expression: str, expected: TimeValue +) -> None: + text = f"报名时间为{expression}\uff0c请及时提交。" + [found] = find_dates(text, date(2027, 5, 1)) + assert found.value == expected + assert found.end_value is None + assert found.issue is None + assert found.text == expression + assert text[found.start : found.end] == expression + + +@pytest.mark.parametrize( + "text", + [ + "6.30", + "报名截止6/30", + "\uff086-30\uff0917:00", + "6.1至6.30", + ], +) +def test_short_numeric_dates_are_not_recognized_from_context(text: str) -> None: + assert find_dates(text, date(2027, 5, 1)) == [] + + +@pytest.mark.parametrize( + "text", + [ + "Python 3.14", + "一二三月", + "一二六月一日", + ], +) +def test_unrelated_numeric_expressions_are_not_promoted_to_dates(text: str) -> None: + assert find_dates(text, date(2027, 5, 1)) == [] + + +@pytest.mark.parametrize("prefix", ["1.", "2. ", "3.\t"]) +def test_numbered_list_items_do_not_hide_a_chinese_date(prefix: str) -> None: + text = prefix + "7月1日00:00至7月7日24:00报名。" + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == "7月1日00:00至7月7日24:00" + assert found.value == TimeToMinute(date=date(2027, 7, 1), time="00:00") + assert found.end_value == TimeToMinute(date=date(2027, 7, 7), time="24:00") + + +@pytest.mark.parametrize( + ("text", "start", "end"), + [ + ("6月1日-6月14号", date(2027, 6, 1), date(2027, 6, 14)), + ("6月1号到14号", date(2027, 6, 1), date(2027, 6, 14)), + ("2027-06-01-2027-06-14", date(2027, 6, 1), date(2027, 6, 14)), + ("2027/6/1至6月14号", date(2027, 6, 1), date(2027, 6, 14)), + ("二\u3007二七年十二月二十八日至一月十日", date(2027, 12, 28), date(2028, 1, 10)), + ("六月一至三十号", date(2027, 6, 1), date(2027, 6, 30)), + ("二\u3007二七年六月一至三十日", date(2027, 6, 1), date(2027, 6, 30)), + ("十二月二十八至一月十日", date(2027, 12, 28), date(2028, 1, 10)), + ("6月一至三十日", date(2027, 6, 1), date(2027, 6, 30)), + ], +) +def test_added_date_formats_share_range_and_year_rules(text: str, start: date, end: date) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.value == TimeToDay(date=start) + assert found.end_value == TimeToDay(date=end) + assert found.issue is None + + +@pytest.mark.parametrize( + ("text", "expected_day", "expected_time"), + [ + ("2027-6-1 17:00", 1, "17:00"), + ("2027.6.9 17点30分", 9, "17:30"), + ("(2027/6/1)17:00", 1, "17:00"), + ], +) +def test_numeric_day_does_not_absorb_the_first_clock_digit( + text: str, expected_day: int, expected_time: str +) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.value == TimeToMinute(date=date(2027, 6, expected_day), time=expected_time) + assert found.issue is None + assert text[found.start : found.end] == found.text + + +@pytest.mark.parametrize( + "text", + [ + "2027-6-1 08:30至2027-6-9 17:00", + "(2027/6/1)08:30至(2027/6/9)17:00", + ], +) +def test_numeric_range_endpoints_preserve_the_clock_boundary(text: str) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.value == TimeToMinute(date=date(2027, 6, 1), time="08:30") + assert found.end_value == TimeToMinute(date=date(2027, 6, 9), time="17:00") + assert found.issue is None + + +@pytest.mark.parametrize("text", ["六月三十号", "八月上旬"]) +def test_new_yearless_formats_still_require_publication(text: str) -> None: + [found] = find_dates(text, None) + assert found.issue == "missing_publication" + assert found.value is None + assert found.end_value is None + + +@pytest.mark.parametrize("text", ["2027-06-30", "2027/6/30", "2027.6.30"]) +def test_complete_numeric_dates_do_not_require_publication(text: str) -> None: + [found] = find_dates(text, None) + assert found.value == TimeToDay(date=date(2027, 6, 30)) + assert found.issue is None + + +@pytest.mark.parametrize( + "text", + [ + "2027-02-29", + "2027/6/31", + "二\u3007二七年二月二十九日", + "6月31号", + "2027-06-30至2027-06-01", + "2027-06-30 24:01", + ], +) +def test_new_formats_preserve_invalid_date_and_range_failure(text: str) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.issue == "invalid_date" + assert found.value is None + assert found.end_value is None + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + ("\uff086月20日\uff0900:00", TimeToMinute(date=date(2027, 6, 20), time="00:00")), + ("(6月20日)08:30", TimeToMinute(date=date(2027, 6, 20), time="08:30")), + ( + "\uff08二\u3007二七年六月二十日\uff09下午2点前", + TimeToMinute(date=date(2027, 6, 20), time="14:00"), + ), + ("(2027/6/20) 24时", TimeToMinute(date=date(2027, 6, 20), time="24:00")), + ("( 2027/6/20 ) 24时", TimeToMinute(date=date(2027, 6, 20), time="24:00")), + ], +) +def test_parenthesized_dates_keep_a_clock_directly_after_the_closing_parenthesis( + expression: str, expected: TimeToMinute +) -> None: + text = f"报到安排为{expression}\uff0c请提前准备。" + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == expression + assert text[found.start : found.end] == expression + assert found.value == expected + assert found.issue is None + + +@pytest.mark.parametrize( + "text", + [ + "\uff086月20日\uff09\n08:30报到", + "6月20日\uff08注意事项\uff0908:30报到", + "\uff08见6月20日\uff0908:30报到", + ], +) +def test_clocks_are_not_joined_across_prose_or_line_breaks(text: str) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == "6月20日" + assert found.value == TimeToDay(date=date(2027, 6, 20)) + + +@pytest.mark.parametrize("weekday", ["周一", "星期一", "星期六"]) +@pytest.mark.parametrize("expression", ["6月1日17:00", "六月一日17:00"]) +def test_a_weekday_immediately_before_a_date_does_not_hide_it( + weekday: str, expression: str +) -> None: + text = f"{weekday}{expression}截止" + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == expression + assert found.value == TimeToMinute(date=date(2027, 6, 1), time="17:00") + + +@pytest.mark.parametrize( + ("text", "expression"), + [ + ("1.2027.6.30截止", "2027.6.30"), + ("说明。1. 2027/6/30报名", "2027/6/30"), + ], +) +def test_numbered_lists_can_start_with_a_complete_numeric_date(text: str, expression: str) -> None: + [found] = find_dates(text, date(2027, 5, 1)) + assert found.text == expression + assert text[found.start : found.end] == expression + assert found.value == TimeToDay(date=date(2027, 6, 30)) + assert found.issue is None diff --git a/crawler/src/crawler/parse/indexing_test.py b/crawler/src/crawler/parse/indexing_test.py index 9bd49aa..85b744c 100644 --- a/crawler/src/crawler/parse/indexing_test.py +++ b/crawler/src/crawler/parse/indexing_test.py @@ -120,8 +120,11 @@ def test_date_ids_link_exact_source_with_values_and_attachment() -> None: ) -def test_dates_in_addresses_are_excluded_but_anchor_labels_are_kept() -> None: - text = "https://example.org/2027年6月1日。6月18日截止。" +@pytest.mark.parametrize( + "url", ["https://example.org/2027年6月1日", "https://example.org/?date=2027-06-01"] +) +def test_dates_in_addresses_are_excluded_but_anchor_labels_are_kept(url: str) -> None: + text = f"{url}。6月18日截止。" start = text.index("6月18日") index = index_documents( [Document(text, hyperlinks=(Hyperlink(start, start + 5, "https://example.org/notice"),))], diff --git a/docs/design.md b/docs/design.md index 54430b8..77e7c00 100644 --- a/docs/design.md +++ b/docs/design.md @@ -262,7 +262,11 @@ CPU 不是问题，要省的是带宽，做法见第 14 节。 | 7 月上旬、中旬、下旬；月初算上旬，月底算下旬 | `{"month": "2026-07", "part": "early"}`，`mid`、`late` | | 7 月 | `{"month": "2026-07"}` | -- 没写年份的先按发布年份算，比发布日期早半年以上的算下一年：12 月发的通知写「1 月 10 日截止」算下一年，回顾 7 月的夏令营仍算当年。时间段右端没写的年月跟左端，跨年同样按这条。「即日起」是发布那天；发布日期为空时，要靠它推年份的不给时间值。 +- 日期的数字可以用阿拉伯数字或中文数字，比如「二〇二六年六月三十日」「九月中旬」。「号」和「日」等价。完整年月日也接受 `2026-06-30`、`2026/6/30`、`2026.6.30`，同一日期的数字分隔符一致，中间不带空格。数字日期也能带时刻，比如 `2026-6-1 17:00`。 +- 日期在括号内、时刻紧跟在括号外时一起识别，比如「（6 月 30 日）17:00」。 +- 数字分隔的年月日须包含完整年份。`6.30`、`6/30`、`6-30` 这样的短数字月日按普通文字保留，不论前后写了什么。原文明示的未识别时间走前述 `unresolved_dates` 交接流程。 +- 日期与时刻的识别以同一行内的表达式为界，表格跨行的日期和时刻分别保留，同日第二段时间保持原文。 +- 日期范围的首端可省略日/号，比如「六月一至三十日」「十二月二十八至一月十日」。没写年份的先按发布年份算，比发布日期早半年以上的算下一年：12 月发的通知写「1 月 10 日截止」算下一年，回顾 7 月的夏令营仍算当年。时间段右端没写的年月跟左端，跨年同样按这条。「即日起」是发布那天；发布日期为空时，要靠它推年份的不给时间值。 - 换算成时刻（排序、倒计时、状态和提醒用）时，拿不准的往早里算：只写到日的截止算到当天结束，带 `mark` 的算到前一天结束，只写到旬或月的算到这段时间开始；开始都从最早的可能算起。页面上照原文的写法显示，不显示换算出的时刻。 ### 8.3 校验