Implement short-form federal statute citations

CHANGES.md

@@ -5,9 +5,16 @@
 The following changes are not yet released, but are code complete:
 
 Features:
--
+- Add `ShortLawCitation`, emitted for bare section references like `§ 484(a)`,
+  with spans covering the marker through the section number (#329)
 
 Changes:
+- `UnknownCitation` is no longer emitted for section markers; section tokens
+  now produce `ShortLawCitation`. Section numbers with a letter glued to the
+  digits (`§ 93a`, `§ 78j(b)`, `§ 2000e-2`) are captured, which
+  `reporters_db`'s `law_section` does not yet allow for full cites. A marker
+  whose number can't be parsed at all still yields a `ShortLawCitation`
+  spanning the bare marker, with `groups["section"]` set to `None` (#329)
 - Add a PR template with an AI Disclosure section.
 - CI: the benchmark now works on fork and Dependabot PRs, with commenting and artifact pushes split into a separate privileged workflow, and fork PRs gated behind a `run-benchmark` label. #332
 - CI: benchmark runs are superseded when a PR is updated, time out after 15 minutes, and pin their third-party action to a commit SHA. #332

eyecite/find.py

@@ -27,10 +27,10 @@
     ResourceCitation,
     SectionToken,
     ShortCaseCitation,
+    ShortLawCitation,
     SupraCitation,
     SupraToken,
     Tokens,
-    UnknownCitation,
 )
 from eyecite.regexes import SUPRA_ANTECEDENT_REGEX, reference_pin_cite_re
 from eyecite.tokenizers import Tokenizer, default_tokenizer
@@ -118,12 +118,8 @@ def get_citations(
             citation = _extract_supra_citation(document.words, i)
 
         # CASE 4: Token is a section marker.
-        # In this case, it's likely that this is a reference to a citation,
-        # but we're not sure what it is if it doesn't match any of the above.
-        # So we record this marker in order to keep an accurate list of the
-        # possible antecedents for id citations.
         elif token_type is SectionToken:
-            citation = UnknownCitation(cast(SectionToken, token), i)
+            citation = ShortLawCitation(cast(SectionToken, token), i)
 
         # CASE 5: The token is not a citation.
         else:

eyecite/models.py

@@ -418,6 +418,39 @@ def corrected_citation_full(self):
         return "".join(parts)
 
 
+@dataclass(eq=False, unsafe_hash=False, repr=False)
+class ShortLawCitation(CitationBase):
+    """A bare section reference, e.g. "§ 484(a)", that inherits its reporter
+    and title from a preceding FullLawCitation. Detection captures only the
+    `section` group; `eyecite.resolve.resolve_citations` backfills the
+    inherited `reporter` and `title` into `metadata` when an antecedent is
+    found."""
+
+    def __hash__(self) -> int:
+        """Always unique: two identical section markers may refer to different
+        laws."""
+        return id(self)
+
+    @dataclass(eq=True, unsafe_hash=True)
+    class Metadata(CitationBase.Metadata):
+        """Define fields on self.metadata."""
+
+        reporter: str | None = None
+        title: str | None = None
+
+    def corrected_citation_full(self):
+        """Return citation with the inherited identity, if resolved. The
+        backfilled metadata carries the antecedent's raw (uncorrected)
+        groups, so the reporter renders as it appeared in the source."""
+        m = self.metadata
+        if not (m.reporter and m.title):
+            return self.matched_text()
+        if m.reporter.startswith("Pub"):
+            # Public Laws read "Pub. L. 116-136, § 3610"
+            return f"{m.reporter} {m.title}, {self.matched_text()}"
+        return f"{m.title} {m.reporter} {self.matched_text()}"
+
+
 @dataclass(eq=False, unsafe_hash=False, repr=False)
 class FullJournalCitation(FullCitation):
     """Citation to a source from `reporters_db/journals.json`."""
@@ -804,6 +837,21 @@ def merge(self, other: "Token") -> Optional["Token"]:
 class SectionToken(Token):
     """Word containing a section symbol."""
 
+    @classmethod
+    def from_match(cls, m, extra, offset=0) -> "Token":
+        """Group 1 of SECTION_REGEX is the bare marker; the section number
+        sits outside it so its trailing guard only applies when a number
+        matched. Extend the span through the section when one is present."""
+        start = m.start(1)
+        end = m.end("section") if m["section"] else m.end(1)
+        return cls(
+            m.string[start:end],
+            start + offset,
+            end + offset,
+            groups=m.groupdict(),
+            **extra,
+        )
+
 
 @dataclass(eq=True, unsafe_hash=True)
 class SupraToken(Token):

eyecite/regexes.py

@@ -95,8 +95,19 @@ def reference_pin_cite_re(regexes):
     strip_punctuation_re(rf"(?P<stop_word>{'|'.join(STOP_WORDS)})")
 )
 
-# Regex for SectionToken
-SECTION_REGEX = r"(\S*§\S*)"
+# Law subsection, capture a single subsection like "(a)" or "(viii)":
+LAW_SUBSECTION = r"(?:\([0-9a-zA-Z]{1,4}\))"
+
+# Regex for SectionToken. Not reporters_db's law_section, which never reaches
+# its parenthetical branch and rejects letter suffixes ("93a"). The trailing
+# guard stops partial number capture ("§ 5th Cir." would capture "5t"); it is
+# a consumed char, not a lookahead, which hyperscan rejects. Section and guard
+# are optional as a unit so a bare or glued marker ("§Analysis") still counts.
+# Group 1 is the marker alone; SectionToken.from_match extends the span.
+LAW_SECTION_REGEX = (
+    rf"(?P<section>\d+[a-z]?(?:[\-.:]\d+){{0,3}}{LAW_SUBSECTION}*)"
+)
+SECTION_REGEX = rf"(§§?)(?:\s*{LAW_SECTION_REGEX}(?:[^a-zA-Z0-9]|$))?"
 
 # Regex for ParagraphToken
 PARAGRAPH_REGEX = r"(\n)"
@@ -234,14 +245,6 @@ def reference_pin_cite_re(regexes):
     )
 """
 
-# Law subsection regex:
-# Capture a single subsection like "(a)", "(1)", or "(viii)":
-LAW_SUBSECTION = r"""
-    (?:
-        \([0-9a-zA-Z]{1,4}\)
-    )
-"""
-
 # Law pin cite regex:
 # Capture pin cite immediately after a law section number.
 # Examples:

eyecite/resolve.py

@@ -5,13 +5,16 @@
 
 from eyecite.models import (
     CitationBase,
+    CitationToken,
     FullCaseCitation,
     FullCitation,
+    FullLawCitation,
     IdCitation,
     ReferenceCitation,
     Resource,
     ResourceType,
     ShortCaseCitation,
+    ShortLawCitation,
     SupraCitation,
 )
 from eyecite.utils import strip_punct
@@ -26,6 +29,11 @@
 # such as "1 U.S. 1. Id. at 200.":
 MAX_OPINION_PAGE_COUNT = 150
 
+# Matched against corrected_reporter(), so an addition here needs the
+# normalized spelling ("C.F.R.", not "CFR"). CFR and state codes are
+# follow-up work; see #77.
+SHORT_LAW_REPORTERS = ("U.S.C.", "Pub. L.")
+
 
 def resolve_full_citation(full_citation: FullCitation) -> Resource:
     """By default, resolve `eyecite.models.FullCaseCitation` objects to a
@@ -183,6 +191,56 @@ def _resolve_shortcase_citation(
         return None
 
 
+def _resolve_shortlaw_citation(
+    short_citation: ShortLawCitation,
+    resolved_full_cites: ResolvedFullCites,
+) -> ResourceType | None:
+    """
+    Resolve a bare section reference like "§ 484(a)" by walking backward
+    for the nearest preceding FullLawCitation with a section group of its
+    own, skipping page-based law cites (48 Stat. 891, 65 Fed. Reg. 12905)
+    and non-law citations. A U.S.C. or Pub. L. antecedent is inherited;
+    any other section bearing law cite (e.g. 12 CFR § 7.4000) blocks
+    resolution rather than being leapfrogged.
+
+    On a match, backfills the inherited reporter and title into
+    short_citation.metadata. Unlike shortcase resolution, this can mint a
+    resource that no full citation produced.
+    """
+    section = short_citation.groups.get("section")
+    if not section:
+        return None
+    for full_citation, _resource in reversed(resolved_full_cites):
+        if not isinstance(full_citation, FullLawCitation):
+            continue
+        if not full_citation.groups.get("section"):
+            # page-based law cites have no title/section structure to
+            # inherit; keep walking
+            continue
+        if full_citation.corrected_reporter() not in SHORT_LAW_REPORTERS:
+            return None
+        short_citation.metadata.reporter = full_citation.groups.get("reporter")
+        short_citation.metadata.title = full_citation.groups.get("title")
+        span_start, span_end = short_citation.span()
+        # Resource hashes raw groups, so copying them verbatim (not
+        # corrected_reporter()) is what lets a short cite cluster with an
+        # antecedent naming the same section.
+        synthetic = FullLawCitation(
+            CitationToken(
+                short_citation.matched_text(),
+                span_start,
+                span_end,
+                {**full_citation.groups, "section": section},
+            ),
+            0,
+            exact_editions=full_citation.exact_editions,
+            variation_editions=full_citation.variation_editions,
+            edition_guess=full_citation.edition_guess,
+        )
+        return Resource(synthetic)
+    return None
+
+
 def _resolve_supra_citation(
     supra_citation: SupraCitation,
     resolved_full_cites: ResolvedFullCites,
@@ -265,15 +323,19 @@ def resolve_citations(
     resolve_id_citation: Callable[
         [IdCitation, ResourceType, Resolutions], ResourceType | None
     ] = _resolve_id_citation,
+    resolve_shortlaw_citation: Callable[
+        [ShortLawCitation, ResolvedFullCites],
+        ResourceType | None,
+    ] = _resolve_shortlaw_citation,
 ) -> Resolutions:
     """Resolve a list of citations to their associated resources by matching
     each type of Citation object (FullCaseCitation, ShortCaseCitation,
-    SupraCitation, and IdCitation) to a "resource" object. A "resource" could
-    be a document, a URL, a database entry, etc. -- anything that conforms to
-    the (non-prescriptive) requirements of the `eyecite.models.ResourceType`
-    type. By default, eyecite uses an extremely thin "resource" object that
-    simply serves as a conceptual way to group citations with the same
-    references together.
+    ShortLawCitation, ReferenceCitation, SupraCitation, and IdCitation) to a
+    "resource" object. A "resource" could be a document, a URL, a database
+    entry, etc. -- anything that conforms to the (non-prescriptive)
+    requirements of the `eyecite.models.ResourceType` type. By default,
+    eyecite uses an extremely thin "resource" object that simply serves as a
+    conceptual way to group citations with the same references together.
 
     This function assumes that the given list of citations is ordered in the
     order that they were extracted from the text (i.e., assumes that supra
@@ -301,8 +363,12 @@ def resolve_citations(
             `eyecite.models.ShortCaseCitation` objects to resources.
         resolve_supra_citation: A function that resolves
             `eyecite.models.SupraCitation` objects to resources.
+        resolve_reference_citation: A function that resolves
+            `eyecite.models.ReferenceCitation` objects to resources.
         resolve_id_citation: A function that resolves
             `eyecite.models.IdCitation` objects to resources.
+        resolve_shortlaw_citation: A function that resolves
+            `eyecite.models.ShortLawCitation` objects to resources.
 
     Returns:
         A dictionary mapping `eyecite.models.ResourceType` objects (the keys)
@@ -330,6 +396,12 @@ def resolve_citations(
                 citation, resolved_full_cites
             )
 
+        # If the citation is a short law citation, try to resolve it
+        elif isinstance(citation, ShortLawCitation):
+            resolution = resolve_shortlaw_citation(
+                citation, resolved_full_cites
+            )
+
         # If the citation is a supra citation, try to resolve it
         elif isinstance(citation, SupraCitation):
             resolution = resolve_supra_citation(citation, resolved_full_cites)

eyecite/test_factories.py

@@ -10,6 +10,7 @@
     ReferenceCitation,
     SectionToken,
     ShortCaseCitation,
+    ShortLawCitation,
     SupraCitation,
     SupraToken,
     UnknownCitation,
@@ -112,6 +113,14 @@ def unknown_citation(source_text=None, index=0, **kwargs):
     return UnknownCitation(SectionToken(source_text, 0, 99), index, **kwargs)
 
 
+def short_law_citation(source_text=None, index=0, **kwargs):
+    """Convenience function for creating mock ShortLawCitation objects."""
+    groups = kwargs.pop("groups", {})
+    return ShortLawCitation(
+        SectionToken(source_text, 0, 99, groups=groups), index, **kwargs
+    )
+
+
 def supra_citation(source_text=None, index=0, **kwargs):
     """Convenience function for creating mock SupraCitation objects."""
     return SupraCitation(SupraToken(source_text, 0, 99), index, **kwargs)

eyecite/tokenizers.py

@@ -407,6 +407,20 @@ def tokenize(self, text: str) -> tuple[Tokens, list[tuple[int, Token]]]:
                     # other case citation. See #221 and #174
                     citation_tokens.pop(-1)
                     all_tokens.pop(-1)
+                elif (
+                    last_token
+                    and isinstance(token, CitationToken)
+                    and isinstance(last_token, SectionToken)
+                ):
+                    # a section token like "§ 550" can swallow the start of
+                    # a full citation ("§ 550 U.S. 544"); prefer the citation
+                    citation_tokens.pop(-1)
+                    all_tokens.pop(-1)
+                    # the section token started earlier, so its marker ("§ ")
+                    # would vanish from the token stream without this
+                    self.append_text(
+                        all_tokens, text[last_token.start : token.start]
+                    )
                 else:
                     # skip overlaps
                     continue

tests/test_AnnotateTest.py

@@ -33,6 +33,13 @@ def lower_annotator(before, text, after):
                 "foo. <0>Mass. Gen. Laws ch. 1, § 2</0>. bar",
                 [],
             ),
+            # short law cite: span ends at the section, keeping trailing
+            # punctuation outside the annotation
+            (
+                "foo. See § 484(a); bar §484(a). baz",
+                "foo. See <0>§ 484(a)</0>; bar <1>§484(a)</1>. baz",
+                [],
+            ),
             # journal cite
             (
                 "foo. 1 Minn. L. Rev. 2. bar",