From 77f27ee4f07fa2053f9c8dd33922753198475914 Mon Sep 17 00:00:00 2001 From: renatodvc <40128530+renatodvc@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:19:02 -0300 Subject: [PATCH 1/4] =?UTF-8?q?Introduce=20ShortLawCitation=20class=20that?= =?UTF-8?q?=20wraps=20bare=20section=20references=20(e.g=20"=C2=A7=20484(a?= =?UTF-8?q?)")?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 4 +- eyecite/find.py | 9 +- eyecite/models.py | 33 +++++++ eyecite/regexes.py | 9 +- eyecite/resolve.py | 69 ++++++++++++- eyecite/test_factories.py | 9 ++ eyecite/tokenizers.py | 12 +++ tests/test_AnnotateTest.py | 7 ++ tests/test_FindTest.py | 86 +++++++++++++++- tests/test_ModelsTest.py | 38 ++++++++ tests/test_ResolveTest.py | 195 ++++++++++++++++++++++++++++++++++++- 11 files changed, 455 insertions(+), 16 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3732d1d8..3059c7b8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,7 +5,9 @@ The following changes are not yet released, but are code complete: Features: -- +- Add `ShortLawCitation`, emitted for bare section references like + `§ 484(a)`, with spans covering the marker through the section number + (#329) Changes: - diff --git a/eyecite/find.py b/eyecite/find.py index 381d281d..65b9b1f8 100644 --- a/eyecite/find.py +++ b/eyecite/find.py @@ -27,10 +27,10 @@ ResourceCitation, SectionToken, ShortCaseCitation, + ShortLawCitation, SupraCitation, SupraToken, Tokens, - UnknownCitation, ) from eyecite.regexes import SUPRA_ANTECEDENT_REGEX, reference_pin_cite_re from eyecite.tokenizers import Tokenizer, default_tokenizer @@ -118,12 +118,9 @@ def get_citations( citation = _extract_supra_citation(document.words, i) # CASE 4: Token is a section marker. - # In this case, it's likely that this is a reference to a citation, - # but we're not sure what it is if it doesn't match any of the above. - # So we record this marker in order to keep an accurate list of the - # possible antecedents for id citations. + # A bare section reference like "§ 484(a)" is a short-form law citation elif token_type is SectionToken: - citation = UnknownCitation(cast(SectionToken, token), i) + citation = ShortLawCitation(cast(SectionToken, token), i) # CASE 5: The token is not a citation. else: diff --git a/eyecite/models.py b/eyecite/models.py index 33a23219..618acd16 100644 --- a/eyecite/models.py +++ b/eyecite/models.py @@ -418,6 +418,39 @@ def corrected_citation_full(self): return "".join(parts) +@dataclass(eq=False, unsafe_hash=False, repr=False) +class ShortLawCitation(CitationBase): + """A bare section reference, e.g. "§ 484(a)", that inherits its reporter + and title from a preceding FullLawCitation. Detection captures only the + `section` group; `eyecite.resolve.resolve_citations` backfills the + inherited `reporter` and `title` into `metadata` when an antecedent is + found.""" + + def __hash__(self) -> int: + """Always unique: two identical section markers may refer to different + laws.""" + return id(self) + + @dataclass(eq=True, unsafe_hash=True) + class Metadata(CitationBase.Metadata): + """Define fields on self.metadata.""" + + reporter: str | None = None + title: str | None = None + + def corrected_citation_full(self): + """Return citation with the inherited identity, if resolved. The + backfilled metadata carries the antecedent's raw (uncorrected) + groups, so the reporter renders as it appeared in the source.""" + m = self.metadata + if not (m.reporter and m.title): + return self.matched_text() + if m.reporter.startswith("Pub"): + # Public Laws read "Pub. L. 116-136, § 3610" + return f"{m.reporter} {m.title}, {self.matched_text()}" + return f"{m.title} {m.reporter} {self.matched_text()}" + + @dataclass(eq=False, unsafe_hash=False, repr=False) class FullJournalCitation(FullCitation): """Citation to a source from `reporters_db/journals.json`.""" diff --git a/eyecite/regexes.py b/eyecite/regexes.py index df1cdddf..29b7a4f2 100644 --- a/eyecite/regexes.py +++ b/eyecite/regexes.py @@ -95,8 +95,13 @@ def reference_pin_cite_re(regexes): strip_punctuation_re(rf"(?P{'|'.join(STOP_WORDS)})") ) -# Regex for SectionToken -SECTION_REGEX = r"(\S*§\S*)" +# Regex for SectionToken. Not reporters_db's law_section: its alternation +# never reaches the parenthetical branch ("484(a)" captures "484"). The +# trailing guard is a consumed char because hyperscan can't compile lookaheads. +LAW_SECTION_REGEX = ( + r"(?P
\d+(?:[\-.:]\d+){0,3}(?:\((?:[a-zA-Z]|\d{1,2})\))*)" +) +SECTION_REGEX = rf"(§§?\s*{LAW_SECTION_REGEX})(?:[^a-zA-Z0-9]|$)" # Regex for ParagraphToken PARAGRAPH_REGEX = r"(\n)" diff --git a/eyecite/resolve.py b/eyecite/resolve.py index 11ee71bf..db82c5f0 100644 --- a/eyecite/resolve.py +++ b/eyecite/resolve.py @@ -5,13 +5,16 @@ from eyecite.models import ( CitationBase, + CitationToken, FullCaseCitation, FullCitation, + FullLawCitation, IdCitation, ReferenceCitation, Resource, ResourceType, ShortCaseCitation, + ShortLawCitation, SupraCitation, ) from eyecite.utils import strip_punct @@ -26,6 +29,11 @@ # such as "1 U.S. 1. Id. at 200.": MAX_OPINION_PAGE_COUNT = 150 +# Reporters whose full cites a bare section reference may inherit from +# (compared against corrected_reporter()). CFR and state codes are +# follow-up work; see #77. +SHORT_LAW_REPORTERS = ("U.S.C.", "Pub. L.") + def resolve_full_citation(full_citation: FullCitation) -> Resource: """By default, resolve `eyecite.models.FullCaseCitation` objects to a @@ -183,6 +191,52 @@ def _resolve_shortcase_citation( return None +def _resolve_shortlaw_citation( + short_citation: ShortLawCitation, + resolved_full_cites: ResolvedFullCites, +) -> ResourceType | None: + """ + Resolve a bare section reference like "§ 484(a)" by walking backward + for the nearest preceding FullLawCitation with a section group of its + own, skipping page-based law cites (48 Stat. 891, 65 Fed. Reg. 12905) + and non-law citations. A U.S.C. or Pub. L. antecedent is inherited; + any other section bearing law cite (e.g. 12 CFR § 7.4000) blocks + resolution rather than being leapfrogged. + """ + section = short_citation.groups.get("section") + if not section: + return None + for full_citation, _resource in reversed(resolved_full_cites): + if not isinstance(full_citation, FullLawCitation): + continue + if not full_citation.groups.get("section"): + # page-based law cites have no title/section structure to + # inherit; keep walking + continue + if full_citation.corrected_reporter() not in SHORT_LAW_REPORTERS: + return None + # Backfill the inherited identity into metadata. The raw groups + # are used (not corrected_reporter()) so the minted resource + # hashes identically to the antecedent's own resource. + short_citation.metadata.reporter = full_citation.groups.get("reporter") + short_citation.metadata.title = full_citation.groups.get("title") + span_start, span_end = short_citation.span() + synthetic = FullLawCitation( + CitationToken( + short_citation.matched_text(), + span_start, + span_end, + {**full_citation.groups, "section": section}, + ), + 0, + exact_editions=full_citation.exact_editions, + variation_editions=full_citation.variation_editions, + edition_guess=full_citation.edition_guess, + ) + return Resource(synthetic) + return None + + def _resolve_supra_citation( supra_citation: SupraCitation, resolved_full_cites: ResolvedFullCites, @@ -265,10 +319,15 @@ def resolve_citations( resolve_id_citation: Callable[ [IdCitation, ResourceType, Resolutions], ResourceType | None ] = _resolve_id_citation, + resolve_shortlaw_citation: Callable[ + [ShortLawCitation, ResolvedFullCites], + ResourceType | None, + ] = _resolve_shortlaw_citation, ) -> Resolutions: """Resolve a list of citations to their associated resources by matching each type of Citation object (FullCaseCitation, ShortCaseCitation, - SupraCitation, and IdCitation) to a "resource" object. A "resource" could + ShortLawCitation, ReferenceCitation, SupraCitation, and IdCitation) to + a "resource" object. A "resource" could be a document, a URL, a database entry, etc. -- anything that conforms to the (non-prescriptive) requirements of the `eyecite.models.ResourceType` type. By default, eyecite uses an extremely thin "resource" object that @@ -303,6 +362,8 @@ def resolve_citations( `eyecite.models.SupraCitation` objects to resources. resolve_id_citation: A function that resolves `eyecite.models.IdCitation` objects to resources. + resolve_shortlaw_citation: A function that resolves + `eyecite.models.ShortLawCitation` objects to resources. Returns: A dictionary mapping `eyecite.models.ResourceType` objects (the keys) @@ -330,6 +391,12 @@ def resolve_citations( citation, resolved_full_cites ) + # If the citation is a short law citation, try to resolve it + elif isinstance(citation, ShortLawCitation): + resolution = resolve_shortlaw_citation( + citation, resolved_full_cites + ) + # If the citation is a supra citation, try to resolve it elif isinstance(citation, SupraCitation): resolution = resolve_supra_citation(citation, resolved_full_cites) diff --git a/eyecite/test_factories.py b/eyecite/test_factories.py index f736a95a..19fd7b25 100644 --- a/eyecite/test_factories.py +++ b/eyecite/test_factories.py @@ -10,6 +10,7 @@ ReferenceCitation, SectionToken, ShortCaseCitation, + ShortLawCitation, SupraCitation, SupraToken, UnknownCitation, @@ -112,6 +113,14 @@ def unknown_citation(source_text=None, index=0, **kwargs): return UnknownCitation(SectionToken(source_text, 0, 99), index, **kwargs) +def short_law_citation(source_text=None, index=0, **kwargs): + """Convenience function for creating mock ShortLawCitation objects.""" + groups = kwargs.pop("groups", {}) + return ShortLawCitation( + SectionToken(source_text, 0, 99, groups=groups), index, **kwargs + ) + + def supra_citation(source_text=None, index=0, **kwargs): """Convenience function for creating mock SupraCitation objects.""" return SupraCitation(SupraToken(source_text, 0, 99), index, **kwargs) diff --git a/eyecite/tokenizers.py b/eyecite/tokenizers.py index 68805476..d8c404ef 100644 --- a/eyecite/tokenizers.py +++ b/eyecite/tokenizers.py @@ -407,6 +407,18 @@ def tokenize(self, text: str) -> tuple[Tokens, list[tuple[int, Token]]]: # other case citation. See #221 and #174 citation_tokens.pop(-1) all_tokens.pop(-1) + elif ( + last_token + and isinstance(token, CitationToken) + and isinstance(last_token, SectionToken) + ): + # a section token like "§ 550" can swallow the start of + # a full citation ("§ 550 U.S. 544"); prefer the citation + citation_tokens.pop(-1) + all_tokens.pop(-1) + self.append_text( + all_tokens, text[last_token.start : token.start] + ) else: # skip overlaps continue diff --git a/tests/test_AnnotateTest.py b/tests/test_AnnotateTest.py index b2e14f31..9e29870d 100644 --- a/tests/test_AnnotateTest.py +++ b/tests/test_AnnotateTest.py @@ -33,6 +33,13 @@ def lower_annotator(before, text, after): "foo. <0>Mass. Gen. Laws ch. 1, § 2. bar", [], ), + # short law cite: span ends at the section, keeping trailing + # punctuation outside the annotation + ( + "foo. See § 484(a); bar §484(a). baz", + "foo. See <0>§ 484(a); bar <1>§484(a). baz", + [], + ), # journal cite ( "foo. 1 Minn. L. Rev. 2. bar", diff --git a/tests/test_FindTest.py b/tests/test_FindTest.py index a2438228..d7dbda79 100644 --- a/tests/test_FindTest.py +++ b/tests/test_FindTest.py @@ -15,6 +15,7 @@ FullCaseCitation, ReferenceCitation, ResourceCitation, + ShortLawCitation, ) from eyecite.test_factories import ( case_citation, @@ -22,8 +23,8 @@ journal_citation, law_citation, reference_citation, + short_law_citation, supra_citation, - unknown_citation, ) from eyecite.tokenizers import ( EDITIONS_LOOKUP, @@ -608,9 +609,8 @@ def test_find_citations(self): [id_citation("Id.", metadata={'pin_cite': 'at 2', 'parenthetical': 'overruling ...'})]), - # Test unknown citation ('lorem ipsum see §99 of the U.S. code.', - [unknown_citation('§99')]), + [short_law_citation('§99', groups={'section': '99'})]), # Test address that's not a citation (#1338) ('lorem 111 S.W. 12th St.', [],), @@ -1019,6 +1019,82 @@ def test_find_law_citations(self): # fmt: on self.run_test_pairs(test_pairs, "Law citation extraction") + def test_find_short_law_citations(self): + """Do short law citation spans cover the section marker through the + last character of the section group?""" + test_triples = ( + ("See § 484(a);", "§ 484(a)", "484(a)"), + ('... law ...." §484(a).', "§484(a)", "484(a)"), + # span mirrors full cites: stops after the first section + ("See §§ 24, 93a, 371(a).", "§§ 24", "24"), + ) + for text, span_text, section in test_triples: + for tokenizer in tested_tokenizers: + with self.subTest( + "Short law spans", + q=text, + tokenizer=type(tokenizer).__name__, + ): + cites = get_citations(text, tokenizer=tokenizer) + self.assertEqual(len(cites), 1, f"got {cites}") + cite = cites[0] + self.assertIsInstance(cite, ShortLawCitation) + start, end = cite.span() + self.assertEqual(text[start:end], span_text) + self.assertEqual(cite.groups["section"], section) + + # Letter-suffixed sections like "§ 93a" are a clean miss, matching + # full cites, which don't detect "12 U.S.C. § 93a" either; a partial + # "§ 93" match would be wrong data. + for tokenizer in tested_tokenizers: + with self.subTest( + "Letter-suffixed miss", tokenizer=type(tokenizer).__name__ + ): + self.assertEqual( + get_citations("See § 93a.", tokenizer=tokenizer), [] + ) + with self.subTest( + "Multi-space marker", tokenizer=type(tokenizer).__name__ + ): + cites = get_citations("See § 484(a);", tokenizer=tokenizer) + self.assertEqual(len(cites), 1) + self.assertEqual(cites[0].groups["section"], "484(a)") + + # A multi-byte char after the section number (e.g. curly quote ”) must + # not affect detection. HyperscanTokenizer fails this due to matching + # bytes and needs the pending multibyte-offsets fix (PR #XXX) for + # parity, so it is excluded here; + for tokenizer in tested_tokenizers[:2]: + with self.subTest( + "Multi-byte follower", tokenizer=type(tokenizer).__name__ + ): + text = "“Nothing in § 484(a)” said" + cites = get_citations(text, tokenizer=tokenizer) + self.assertEqual(len(cites), 1) + start, end = cites[0].span() + self.assertEqual(text[start:end], "§ 484(a)") + + # A section token must not swallow the start of a following full + # citation whose volume/title abuts the section number. + overlap_pairs = ( + ("As held in § 550 U.S. 544 the rule applies.", "550 U.S. 544"), + ("see § 18 U.S.C. § 921 for details.", "18 U.S.C. § 921"), + ) + for text, expected in overlap_pairs: + for tokenizer in tested_tokenizers: + with self.subTest( + "Citation shadowing", + q=text, + tokenizer=type(tokenizer).__name__, + ): + cites = get_citations(text, tokenizer=tokenizer) + spans = [text[c.span()[0] : c.span()[1]] for c in cites] + self.assertIn(expected, spans) + self.assertFalse( + any(isinstance(c, ShortLawCitation) for c in cites), + f"section token should yield to full cite: {cites}", + ) + def test_find_journal_citations(self): """Can we find citations from journals.json?""" # fmt: off @@ -1747,7 +1823,7 @@ def test_markup_plaintiff_and_antecedent_guesses(self) -> None: ( """§ 3.1 (2d ed. 1977), Strawberry Hill, 725 S.W.2d at 176 (Gonzalez, J., dissenting);""", [ - unknown_citation("§"), + short_law_citation("§ 3.1", groups={"section": "3.1"}), case_citation( page="176", reporter="S.W.2d", @@ -1766,7 +1842,7 @@ def test_markup_plaintiff_and_antecedent_guesses(self) -> None: ( """§ 3.1 (2d ed. 1977), (See Hill, 725 S.W.2d at 176 (Gonzalez, J., dissenting));""", [ - unknown_citation("§"), + short_law_citation("§ 3.1", groups={"section": "3.1"}), case_citation( page="176", reporter="S.W.2d", diff --git a/tests/test_ModelsTest.py b/tests/test_ModelsTest.py index b195530b..40f7e336 100644 --- a/tests/test_ModelsTest.py +++ b/tests/test_ModelsTest.py @@ -7,6 +7,7 @@ id_citation, journal_citation, law_citation, + short_law_citation, unknown_citation, ) @@ -158,6 +159,43 @@ def test_unknown_citation_comparison(self): self.assertNotEqual(hash(citations[0]), hash(citations[1])) print("✓") + def test_short_law_citation_comparison(self): + """Are two ShortLawCitation objects always different?""" + citations = [ + short_law_citation("§ 484(a)", groups={"section": "484(a)"}), + short_law_citation("§ 484(a)", groups={"section": "484(a)"}), + ] + print("Testing short law citation comparison...", end=" ") + self.assertNotEqual(citations[0], citations[1]) + self.assertNotEqual(hash(citations[0]), hash(citations[1])) + print("✓") + + def test_short_law_citation_corrected_citation_full(self): + """Does corrected_citation_full render the inherited identity?""" + print("Testing short law corrected_citation_full...", end=" ") + # unresolved: falls back to matched text + cite = short_law_citation("§ 484(a)", groups={"section": "484(a)"}) + self.assertEqual(cite.corrected_citation_full(), "§ 484(a)") + # U.S.C. - title before reporter + cite = short_law_citation( + "§ 484(a)", + groups={"section": "484(a)"}, + metadata={"reporter": "U. S. C.", "title": "12"}, + ) + self.assertEqual( + cite.corrected_citation_full(), "12 U. S. C. § 484(a)" + ) + # Pub. L. - reporter before title + cite = short_law_citation( + "§ 3610", + groups={"section": "3610"}, + metadata={"reporter": "Pub. L.", "title": "116-136"}, + ) + self.assertEqual( + cite.corrected_citation_full(), "Pub. L. 116-136, § 3610" + ) + print("✓") + def test_missing_page_cite_conversion(self): """Do citations with missing page numbers get their groups['page'] attribute set to None?""" diff --git a/tests/test_ResolveTest.py b/tests/test_ResolveTest.py index c6ad9003..744569d3 100644 --- a/tests/test_ResolveTest.py +++ b/tests/test_ResolveTest.py @@ -4,7 +4,12 @@ from eyecite import get_citations from eyecite.find import extract_reference_citations from eyecite.helpers import filter_citations -from eyecite.models import Document, FullCitation, Resource +from eyecite.models import ( + Document, + FullCitation, + Resource, + ShortLawCitation, +) from eyecite.resolve import resolve_citations @@ -259,6 +264,194 @@ def test_ambigous_short_cite(self): (None, "Foo, 1 U.S., at 2."), ) + watters_6_7 = ( # From issue #329 + "Business activities of national banks are controlled by the " + "National Bank Act (NBA or Act), 12 U. S. C. § 1 et seq., and " + "regulations promulgated thereunder by the Office of the " + "Comptroller of the Currency (OCC). See §§ 24, 93a, 371(a). As " + "the agency charged by Congress with supervision of the NBA, OCC " + "oversees the operations of national banks and their interactions " + "with customers. See NationsBank of N. C., N. A. v. Variable " + "Annuity Life Ins. Co., 513 U. S. 251, 254, 256 (1995). The " + "agency exercises visitorial powers, including the authority to " + "audit the bank's books and records, largely to the exclusion of " + "other governmental entities, state or federal. See § 484(a); " + "12 CFR § 7.4000 (2006)." + ) + + def resolve_text(self, text): + """Return (citations, resolutions dict, formatted resolution).""" + citations = get_citations(text) + resolutions = resolve_citations(citations) + return citations, resolutions, format_resolution(resolutions) + + def shortlaw_cites(self, citations): + return [c for c in citations if isinstance(c, ShortLawCitation)] + + def test_shortlaw_resolution(self): + """Do short law citations inherit from the nearest U.S.C. cite, + without intervening case citations breaking the chain, and without + clustering with the antecedent's own section?""" + citations, _, formatted = self.resolve_text(self.watters_6_7) + self.assertEqual( + formatted, + { + "12 U. S. C. § 1": ["12 U. S. C. § 1"], + "§§ 24": ["§§ 24"], + "513 U. S. 251": ["513 U. S. 251"], + "§ 484(a)": ["§ 484(a)"], + "12 CFR § 7.4000": ["12 CFR § 7.4000"], + }, + ) + shorts = self.shortlaw_cites(citations) + self.assertEqual(len(shorts), 2) + for short in shorts: + self.assertEqual(short.metadata.title, "12") + self.assertEqual(short.metadata.reporter, "U. S. C.") + + def test_shortlaw_no_leapfrog(self): + """Does a section bearing CFR cite block resolution rather than + being leapfrogged? And is a page-based Fed. Reg. cite never an + antecedent?""" + # The leading U.S.C. cite would wrongly resolve the short cite if + # the CFR cite were skipped instead of blocking. + text = ( + "12 U. S. C. § 1 authorizes national bank activities. " + '... the OCC may "direct the bank or operating subsidiary to ' + 'take appropriate remedial action ...." 12 CFR § 5.34(e)(3) ' + "(2006). OCC subsequently revised its regulations to track " + "the statute. See § 5.34(e)(1), (3); Financial Subsidiaries " + "and Operating Subsidiaries, 65 Fed. Reg. 12905, 12911 (2000)." + ) + citations, _, formatted = self.resolve_text(text) + self.assertEqual( + formatted, + { + "12 U. S. C. § 1": ["12 U. S. C. § 1"], + "12 CFR § 5.34": ["12 CFR § 5.34"], + "65 Fed. Reg. 12905": ["65 Fed. Reg. 12905"], + }, + ) + (short,) = self.shortlaw_cites(citations) + self.assertEqual(short.matched_text(), "§ 5.34(e)(1)") + self.assertIsNone(short.metadata.reporter) + self.assertIsNone(short.metadata.title) + + def test_shortlaw_popular_name_unresolved(self): + """Does a section reference with no preceding law cite stay unresolved, + while a later one inherits and clusters with the full cite naming the + same section? (See issue #324)""" + text = ( # From issue #329 + "Liability under § 1 of the Sherman Act, 15 U. S. C. § 1, " + 'requires a "contract, combination ..., or conspiracy, in ' + 'restraint of trade or commerce." The question in this ' + "putative class action is whether a § 1 complaint can survive " + "a motion to dismiss ..." + ) + citations, resolutions, formatted = self.resolve_text(text) + self.assertEqual( + formatted, + {"15 U. S. C. § 1": ["15 U. S. C. § 1", "§ 1"]}, + ) + shorts = self.shortlaw_cites(citations) + self.assertEqual(len(shorts), 2) + resolved_cites = [c for v in resolutions.values() for c in v] + self.assertNotIn(shorts[0], resolved_cites) + self.assertIsNone(shorts[0].metadata.reporter) + self.assertIn(shorts[1], resolved_cites) + self.assertEqual(shorts[1].metadata.title, "15") + self.assertEqual(shorts[1].metadata.reporter, "U. S. C.") + + def test_shortlaw_clustering(self): + """Do short cites naming the same section resolve to one minted + resource, distinct from other sections, and does that resource + merge with a later full cite of the same section?""" + watters_11 = ( # From issue #329 + "The Act vested in nationally chartered banks enumerated " + 'powers and "all such incidental powers as shall be necessary ' + 'to carry on the business of banking." 12 U. S. C. §24 ' + "Seventh. To prevent inconsistent or intrusive state " + "regulation from impairing the national system, Congress " + 'provided: "No national bank shall be subject to any ' + 'visitorial powers except as authorized by Federal law ...." ' + "§484(a)." + ) + _, _, formatted = self.resolve_text( + self.watters_6_7 + " " + watters_11 + ) + self.assertEqual( + formatted, + { + "12 U. S. C. § 1": ["12 U. S. C. § 1"], + "§§ 24": ["§§ 24", "12 U. S. C. §24"], + "513 U. S. 251": ["513 U. S. 251"], + "§ 484(a)": ["§ 484(a)", "§484(a)"], + "12 CFR § 7.4000": ["12 CFR § 7.4000"], + }, + ) + + def test_shortlaw_pub_l_resolution(self): + """Does a short cite inherit from a Pub. L. antecedent (uncodified + statute), skipping the page-based Stat. cite in between?""" + text = ( # From issue #329 + "Coronavirus Aid, Relief, and Economic Security (CARES) Act, " + "Pub. L. No. 116-136, § 3610, 134 Stat. 281, 414 (2020). ... " + "Costs claimed under § 3610 must be supported by evidence of " + "paid leave actually provided." + ) + citations, _, formatted = self.resolve_text(text) + self.assertEqual( + formatted, + { + "Pub. L. No. 116-136, § 3610": [ + "Pub. L. No. 116-136, § 3610", + "§ 3610", + ], + "134 Stat. 281": ["134 Stat. 281"], + }, + ) + (short,) = self.shortlaw_cites(citations) + self.assertEqual(short.metadata.reporter, "Pub. L.") + self.assertEqual(short.metadata.title, "116-136") + self.assertEqual( + short.corrected_citation_full(), "Pub. L. 116-136, § 3610" + ) + + def test_shortlaw_id_resolution(self): + """Does an Id. following a minted short law resource attach to + it?""" + # "Id. at 5." exercises the pin cite path in _has_invalid_pin_cite, + # where the minted resource's first citation is a ShortLawCitation, + # not the FullCitation the code casts to. + for id_text in ("Id.", "Id. at 5."): + text = ( + f"12 U. S. C. § 1 et seq. was cited. See § 484(a). {id_text}" + ) + _, _, formatted = self.resolve_text(text) + self.assertEqual( + formatted, + { + "12 U. S. C. § 1": ["12 U. S. C. § 1"], + "§ 484(a)": ["§ 484(a)", "Id."], + }, + f"failed for {id_text!r}", + ) + + def test_shortlaw_known_failure_quoted_material(self): + """Known limitation: quoted material breaks nearest antecedent + proximity, inheriting U.S.C. where CFR is correct. This documents + the wrong behavior; if it starts failing, the antecedent rule got + smarter and the assertions should flip to CFR.""" + text = ( + "12 CFR § 5.34(e)(1) (2001). The brief argued that " + '"12 USC §24" controls. See § 5.34(e)(3).' + ) + citations, _, _ = self.resolve_text(text) + (short,) = self.shortlaw_cites(citations) + # Wrong on purpose: inherited from the quoted U.S.C. cite + self.assertEqual(short.metadata.reporter, "USC") + self.assertEqual(short.metadata.title, "12") + def test_id_resolution(self): # Test resolving an Id. citation self.checkResolution( From bcb66d1aaad3fef9491b00d5a1ed0ec796219116 Mon Sep 17 00:00:00 2001 From: renatodvc <40128530+renatodvc@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:34:49 -0300 Subject: [PATCH 2/4] Improve regexes for parsing SectionToken and polish comments --- CHANGES.md | 12 ++++++++---- eyecite/find.py | 1 - eyecite/regexes.py | 14 +++++++++----- eyecite/resolve.py | 29 +++++++++++++++++------------ eyecite/tokenizers.py | 2 ++ tests/test_FindTest.py | 37 ++++++++++++++++++++++++++----------- tests/test_ResolveTest.py | 3 +-- 7 files changed, 63 insertions(+), 35 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3059c7b8..2350fc26 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,12 +5,16 @@ The following changes are not yet released, but are code complete: Features: -- Add `ShortLawCitation`, emitted for bare section references like - `§ 484(a)`, with spans covering the marker through the section number - (#329) +- Add `ShortLawCitation`, emitted for bare section references like `§ 484(a)`, + with spans covering the marker through the section number (#329) Changes: -- +- `UnknownCitation` is no longer emitted for section markers; section tokens + now produce `ShortLawCitation`. Section numbers with a letter glued to the + digits (`§ 93a`, `§ 78j(b)`, `§ 2000e-2`) are captured, which + `reporters_db`'s `law_section` does not yet allow for full cites. A marker + whose number can't be parsed at all still yields a `ShortLawCitation` + spanning the bare marker, with `groups["section"]` set to `None` (#329) Fixes: - diff --git a/eyecite/find.py b/eyecite/find.py index 65b9b1f8..b8043521 100644 --- a/eyecite/find.py +++ b/eyecite/find.py @@ -118,7 +118,6 @@ def get_citations( citation = _extract_supra_citation(document.words, i) # CASE 4: Token is a section marker. - # A bare section reference like "§ 484(a)" is a short-form law citation elif token_type is SectionToken: citation = ShortLawCitation(cast(SectionToken, token), i) diff --git a/eyecite/regexes.py b/eyecite/regexes.py index 29b7a4f2..7f9d3cd1 100644 --- a/eyecite/regexes.py +++ b/eyecite/regexes.py @@ -95,13 +95,17 @@ def reference_pin_cite_re(regexes): strip_punctuation_re(rf"(?P{'|'.join(STOP_WORDS)})") ) -# Regex for SectionToken. Not reporters_db's law_section: its alternation -# never reaches the parenthetical branch ("484(a)" captures "484"). The -# trailing guard is a consumed char because hyperscan can't compile lookaheads. +# Regex for SectionToken. Not reporters_db's law_section: its alternation never +# reaches the parenthetical branch ("484(a)" captures "484") and it rejects a +# letter glued to the digits ("93a"), which full cites need fixed upstream but +# short cites can capture today. The trailing guard keeps a partially consumed +# number out of the group ("§ 5th Cir." would otherwise capture "5t"); it is a +# consumed char, not a lookahead, because hyperscan rejects zero-width +# assertions. Section is optional so an unparseable marker is still found. LAW_SECTION_REGEX = ( - r"(?P
\d+(?:[\-.:]\d+){0,3}(?:\((?:[a-zA-Z]|\d{1,2})\))*)" + r"(?P
\d+[a-z]?(?:[\-.:]\d+){0,3}(?:\((?:[a-zA-Z]|\d{1,2})\))*)" ) -SECTION_REGEX = rf"(§§?\s*{LAW_SECTION_REGEX})(?:[^a-zA-Z0-9]|$)" +SECTION_REGEX = rf"(§§?(?:\s*{LAW_SECTION_REGEX})?)(?:[^a-zA-Z0-9]|$)" # Regex for ParagraphToken PARAGRAPH_REGEX = r"(\n)" diff --git a/eyecite/resolve.py b/eyecite/resolve.py index db82c5f0..a630564b 100644 --- a/eyecite/resolve.py +++ b/eyecite/resolve.py @@ -29,8 +29,8 @@ # such as "1 U.S. 1. Id. at 200.": MAX_OPINION_PAGE_COUNT = 150 -# Reporters whose full cites a bare section reference may inherit from -# (compared against corrected_reporter()). CFR and state codes are +# Matched against corrected_reporter(), so an addition here needs the +# normalized spelling ("C.F.R.", not "CFR"). CFR and state codes are # follow-up work; see #77. SHORT_LAW_REPORTERS = ("U.S.C.", "Pub. L.") @@ -202,6 +202,10 @@ def _resolve_shortlaw_citation( and non-law citations. A U.S.C. or Pub. L. antecedent is inherited; any other section bearing law cite (e.g. 12 CFR § 7.4000) blocks resolution rather than being leapfrogged. + + On a match, backfills the inherited reporter and title into + short_citation.metadata. Unlike shortcase resolution, this can mint a + resource that no full citation produced. """ section = short_citation.groups.get("section") if not section: @@ -215,12 +219,12 @@ def _resolve_shortlaw_citation( continue if full_citation.corrected_reporter() not in SHORT_LAW_REPORTERS: return None - # Backfill the inherited identity into metadata. The raw groups - # are used (not corrected_reporter()) so the minted resource - # hashes identically to the antecedent's own resource. short_citation.metadata.reporter = full_citation.groups.get("reporter") short_citation.metadata.title = full_citation.groups.get("title") span_start, span_end = short_citation.span() + # Resource hashes raw groups, so copying them verbatim (not + # corrected_reporter()) is what lets a short cite cluster with an + # antecedent naming the same section. synthetic = FullLawCitation( CitationToken( short_citation.matched_text(), @@ -326,13 +330,12 @@ def resolve_citations( ) -> Resolutions: """Resolve a list of citations to their associated resources by matching each type of Citation object (FullCaseCitation, ShortCaseCitation, - ShortLawCitation, ReferenceCitation, SupraCitation, and IdCitation) to - a "resource" object. A "resource" could - be a document, a URL, a database entry, etc. -- anything that conforms to - the (non-prescriptive) requirements of the `eyecite.models.ResourceType` - type. By default, eyecite uses an extremely thin "resource" object that - simply serves as a conceptual way to group citations with the same - references together. + ShortLawCitation, ReferenceCitation, SupraCitation, and IdCitation) to a + "resource" object. A "resource" could be a document, a URL, a database + entry, etc. -- anything that conforms to the (non-prescriptive) + requirements of the `eyecite.models.ResourceType` type. By default, + eyecite uses an extremely thin "resource" object that simply serves as a + conceptual way to group citations with the same references together. This function assumes that the given list of citations is ordered in the order that they were extracted from the text (i.e., assumes that supra @@ -360,6 +363,8 @@ def resolve_citations( `eyecite.models.ShortCaseCitation` objects to resources. resolve_supra_citation: A function that resolves `eyecite.models.SupraCitation` objects to resources. + resolve_reference_citation: A function that resolves + `eyecite.models.ReferenceCitation` objects to resources. resolve_id_citation: A function that resolves `eyecite.models.IdCitation` objects to resources. resolve_shortlaw_citation: A function that resolves diff --git a/eyecite/tokenizers.py b/eyecite/tokenizers.py index d8c404ef..ca5e9e76 100644 --- a/eyecite/tokenizers.py +++ b/eyecite/tokenizers.py @@ -416,6 +416,8 @@ def tokenize(self, text: str) -> tuple[Tokens, list[tuple[int, Token]]]: # a full citation ("§ 550 U.S. 544"); prefer the citation citation_tokens.pop(-1) all_tokens.pop(-1) + # the section token started earlier, so its marker ("§ ") + # would vanish from the token stream without this self.append_text( all_tokens, text[last_token.start : token.start] ) diff --git a/tests/test_FindTest.py b/tests/test_FindTest.py index d7dbda79..cd568371 100644 --- a/tests/test_FindTest.py +++ b/tests/test_FindTest.py @@ -1020,13 +1020,18 @@ def test_find_law_citations(self): self.run_test_pairs(test_pairs, "Law citation extraction") def test_find_short_law_citations(self): - """Do short law citation spans cover the section marker through the - last character of the section group?""" + """Do bare section references produce ShortLawCitations, spanning the + marker through the last character of the section group?""" test_triples = ( ("See § 484(a);", "§ 484(a)", "484(a)"), ('... law ...." §484(a).', "§484(a)", "484(a)"), # span mirrors full cites: stops after the first section ("See §§ 24, 93a, 371(a).", "§§ 24", "24"), + # letter-suffixed forms, which full cites miss entirely until + # reporters_db's law_section is fixed upstream + ("See § 93a.", "§ 93a", "93a"), + ("See § 78j(b).", "§ 78j(b)", "78j(b)"), + ("under § 2000e-2 the", "§ 2000e-2", "2000e-2"), ) for text, span_text, section in test_triples: for tokenizer in tested_tokenizers: @@ -1043,16 +1048,26 @@ def test_find_short_law_citations(self): self.assertEqual(text[start:end], span_text) self.assertEqual(cite.groups["section"], section) - # Letter-suffixed sections like "§ 93a" are a clean miss, matching - # full cites, which don't detect "12 U.S.C. § 93a" either; a partial - # "§ 93" match would be wrong data. + # Shape alone identifies a short cite, so an unparseable number still + # yields a marker-only cite, which resolution then drops. A partial + # "§ 5" match would be wrong data. + unparsed_sections = ("A bare § here.", "See § 5th Cir.", "§ ibid") + for text in unparsed_sections: + for tokenizer in tested_tokenizers: + with self.subTest( + "Unparsed section", + q=text, + tokenizer=type(tokenizer).__name__, + ): + cites = get_citations(text, tokenizer=tokenizer) + self.assertEqual(len(cites), 1, f"got {cites}") + cite = cites[0] + self.assertIsInstance(cite, ShortLawCitation) + self.assertIsNone(cite.groups["section"]) + start, end = cite.span() + self.assertEqual(text[start:end], "§") + for tokenizer in tested_tokenizers: - with self.subTest( - "Letter-suffixed miss", tokenizer=type(tokenizer).__name__ - ): - self.assertEqual( - get_citations("See § 93a.", tokenizer=tokenizer), [] - ) with self.subTest( "Multi-space marker", tokenizer=type(tokenizer).__name__ ): diff --git a/tests/test_ResolveTest.py b/tests/test_ResolveTest.py index 744569d3..4d411e4e 100644 --- a/tests/test_ResolveTest.py +++ b/tests/test_ResolveTest.py @@ -311,8 +311,7 @@ def test_shortlaw_resolution(self): def test_shortlaw_no_leapfrog(self): """Does a section bearing CFR cite block resolution rather than - being leapfrogged? And is a page-based Fed. Reg. cite never an - antecedent?""" + being leapfrogged?""" # The leading U.S.C. cite would wrongly resolve the short cite if # the CFR cite were skipped instead of blocking. text = ( From b28b87303daefb1575bdeeafbfdf4f6fb346d91e Mon Sep 17 00:00:00 2001 From: renatodvc <40128530+renatodvc@users.noreply.github.com> Date: Fri, 7 Aug 2026 19:52:57 -0300 Subject: [PATCH 3/4] Replace placeholder text to reference correct PR with Hyperscan fix --- tests/test_FindTest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_FindTest.py b/tests/test_FindTest.py index cd568371..20f871e5 100644 --- a/tests/test_FindTest.py +++ b/tests/test_FindTest.py @@ -1077,7 +1077,7 @@ def test_find_short_law_citations(self): # A multi-byte char after the section number (e.g. curly quote ”) must # not affect detection. HyperscanTokenizer fails this due to matching - # bytes and needs the pending multibyte-offsets fix (PR #XXX) for + # bytes and needs the pending multibyte-offsets fix (PR #334) for # parity, so it is excluded here; for tokenizer in tested_tokenizers[:2]: with self.subTest( From 3104ce04ecb182e0498f40e7d336a9269c2c1545 Mon Sep 17 00:00:00 2001 From: renatodvc <40128530+renatodvc@users.noreply.github.com> Date: Wed, 12 Aug 2026 01:43:29 -0300 Subject: [PATCH 4/4] Fix findings 1 and 3 from code review --- eyecite/models.py | 15 +++++++++++++++ eyecite/regexes.py | 28 +++++++++++----------------- tests/test_FindTest.py | 19 +++++++++++++++++-- tests/test_ResolveTest.py | 20 ++++++++++++++++++++ 4 files changed, 63 insertions(+), 19 deletions(-) diff --git a/eyecite/models.py b/eyecite/models.py index 618acd16..cc40b92c 100644 --- a/eyecite/models.py +++ b/eyecite/models.py @@ -837,6 +837,21 @@ def merge(self, other: "Token") -> Optional["Token"]: class SectionToken(Token): """Word containing a section symbol.""" + @classmethod + def from_match(cls, m, extra, offset=0) -> "Token": + """Group 1 of SECTION_REGEX is the bare marker; the section number + sits outside it so its trailing guard only applies when a number + matched. Extend the span through the section when one is present.""" + start = m.start(1) + end = m.end("section") if m["section"] else m.end(1) + return cls( + m.string[start:end], + start + offset, + end + offset, + groups=m.groupdict(), + **extra, + ) + @dataclass(eq=True, unsafe_hash=True) class SupraToken(Token): diff --git a/eyecite/regexes.py b/eyecite/regexes.py index 7f9d3cd1..7577543e 100644 --- a/eyecite/regexes.py +++ b/eyecite/regexes.py @@ -95,17 +95,19 @@ def reference_pin_cite_re(regexes): strip_punctuation_re(rf"(?P{'|'.join(STOP_WORDS)})") ) -# Regex for SectionToken. Not reporters_db's law_section: its alternation never -# reaches the parenthetical branch ("484(a)" captures "484") and it rejects a -# letter glued to the digits ("93a"), which full cites need fixed upstream but -# short cites can capture today. The trailing guard keeps a partially consumed -# number out of the group ("§ 5th Cir." would otherwise capture "5t"); it is a -# consumed char, not a lookahead, because hyperscan rejects zero-width -# assertions. Section is optional so an unparseable marker is still found. +# Law subsection, capture a single subsection like "(a)" or "(viii)": +LAW_SUBSECTION = r"(?:\([0-9a-zA-Z]{1,4}\))" + +# Regex for SectionToken. Not reporters_db's law_section, which never reaches +# its parenthetical branch and rejects letter suffixes ("93a"). The trailing +# guard stops partial number capture ("§ 5th Cir." would capture "5t"); it is +# a consumed char, not a lookahead, which hyperscan rejects. Section and guard +# are optional as a unit so a bare or glued marker ("§Analysis") still counts. +# Group 1 is the marker alone; SectionToken.from_match extends the span. LAW_SECTION_REGEX = ( - r"(?P
\d+[a-z]?(?:[\-.:]\d+){0,3}(?:\((?:[a-zA-Z]|\d{1,2})\))*)" + rf"(?P
\d+[a-z]?(?:[\-.:]\d+){{0,3}}{LAW_SUBSECTION}*)" ) -SECTION_REGEX = rf"(§§?(?:\s*{LAW_SECTION_REGEX})?)(?:[^a-zA-Z0-9]|$)" +SECTION_REGEX = rf"(§§?)(?:\s*{LAW_SECTION_REGEX}(?:[^a-zA-Z0-9]|$))?" # Regex for ParagraphToken PARAGRAPH_REGEX = r"(\n)" @@ -243,14 +245,6 @@ def reference_pin_cite_re(regexes): ) """ -# Law subsection regex: -# Capture a single subsection like "(a)", "(1)", or "(viii)": -LAW_SUBSECTION = r""" - (?: - \([0-9a-zA-Z]{1,4}\) - ) -""" - # Law pin cite regex: # Capture pin cite immediately after a law section number. # Examples: diff --git a/tests/test_FindTest.py b/tests/test_FindTest.py index 20f871e5..412ab72c 100644 --- a/tests/test_FindTest.py +++ b/tests/test_FindTest.py @@ -1032,6 +1032,14 @@ def test_find_short_law_citations(self): ("See § 93a.", "§ 93a", "93a"), ("See § 78j(b).", "§ 78j(b)", "78j(b)"), ("under § 2000e-2 the", "§ 2000e-2", "2000e-2"), + # roman-numeral and 3-digit subsections, wider than a single + # letter or two digits + ( + "under § 1158(b)(2)(A)(ii) an alien", + "§ 1158(b)(2)(A)(ii)", + "1158(b)(2)(A)(ii)", + ), + ("See § 42(100).", "§ 42(100)", "42(100)"), ) for text, span_text, section in test_triples: for tokenizer in tested_tokenizers: @@ -1050,8 +1058,15 @@ def test_find_short_law_citations(self): # Shape alone identifies a short cite, so an unparseable number still # yields a marker-only cite, which resolution then drops. A partial - # "§ 5" match would be wrong data. - unparsed_sections = ("A bare § here.", "See § 5th Cir.", "§ ibid") + # "§ 5" match would be wrong data. Markers glued to a word must be + # found too, since the guard only applies when a number matched. + unparsed_sections = ( + "A bare § here.", + "See § 5th Cir.", + "§ ibid", + "Notwithstanding §Analysis of the code applies.", + "mid-word cross§reference too", + ) for text in unparsed_sections: for tokenizer in tested_tokenizers: with self.subTest( diff --git a/tests/test_ResolveTest.py b/tests/test_ResolveTest.py index 4d411e4e..f07d9f58 100644 --- a/tests/test_ResolveTest.py +++ b/tests/test_ResolveTest.py @@ -389,6 +389,26 @@ def test_shortlaw_clustering(self): }, ) + def test_shortlaw_subsection_distinct_resources(self): + """Do short cites naming different subsections of the same section + mint distinct resources? Roman-numeral subsections used to truncate + ("§ 1158(b)(2)(A)(ii)" captured "1158(b)(2)(A)"), merging them.""" + text = ( + "Asylum eligibility is governed by 8 U. S. C. § 1158. The " + "persecutor bar appears at § 1158(b)(2)(A)(i), while " + "§ 1158(b)(2)(A)(ii) covers conviction of a particularly " + "serious crime." + ) + _, _, formatted = self.resolve_text(text) + self.assertEqual( + formatted, + { + "8 U. S. C. § 1158": ["8 U. S. C. § 1158"], + "§ 1158(b)(2)(A)(i)": ["§ 1158(b)(2)(A)(i)"], + "§ 1158(b)(2)(A)(ii)": ["§ 1158(b)(2)(A)(ii)"], + }, + ) + def test_shortlaw_pub_l_resolution(self): """Does a short cite inherit from a Pub. L. antecedent (uncodified statute), skipping the page-based Stat. cite in between?"""