From f5220d994197cf672e015e6ca1a2d0e08ea278c2 Mon Sep 17 00:00:00 2001 From: Alajmah <55954113+Alajmah@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:23:34 +0300 Subject: [PATCH 1/5] Harden corpus contracts from Codex review --- .github/workflows/validate.yml | 4 +- data/README.md | 11 +- .../2023-12-08-mixtral-8x7b-weights.yaml | 9 ++ .../2023-12-11-mixtral-8x7b-announced.yaml | 9 ++ .../2023-12-11-mixtral-8x7b-weights.yaml | 8 -- data/sources/mixtral-8x7b-magnet-post.yaml | 9 ++ docs/ontology.md | 37 ++++++- schemas/claim.schema.json | 10 +- schemas/entity.schema.json | 25 +++++ schemas/event.schema.json | 8 +- schemas/relationship.schema.json | 10 +- schemas/year-view.schema.json | 20 +++- tests/test_review_hardening.py | 104 ++++++++++++++++++ tools/build_timeline.py | 11 +- tools/build_year.py | 25 +++-- tools/records.py | 48 ++++++++ tools/validate.py | 24 +--- 17 files changed, 311 insertions(+), 61 deletions(-) create mode 100644 data/events/2023-12-08-mixtral-8x7b-weights.yaml create mode 100644 data/events/2023-12-11-mixtral-8x7b-announced.yaml delete mode 100644 data/events/2023-12-11-mixtral-8x7b-weights.yaml create mode 100644 data/sources/mixtral-8x7b-magnet-post.yaml create mode 100644 tests/test_review_hardening.py mode change 100644 => 100755 tools/build_year.py create mode 100644 tools/records.py diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 723448a..24cb988 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -20,9 +20,11 @@ jobs: cache-dependency-path: requirements-dev.txt - name: Install validation dependencies run: python -m pip install -r requirements-dev.txt + - name: Run review-hardening regression tests + run: python -m unittest discover -s tests -v - name: Validate schemas, records, and references run: python tools/validate.py - name: Build chronological timeline run: python tools/build_timeline.py --format json > /tmp/modelmap-timeline.json - name: Build 2023 year view - run: python tools/build_year.py 2023 --format markdown > /tmp/modelmap-2023.md + run: python tools/build_year.py 2023 --format json > /tmp/modelmap-2023.json diff --git a/data/README.md b/data/README.md index 8a0c586..7ac81f7 100644 --- a/data/README.md +++ b/data/README.md @@ -35,15 +35,16 @@ Directories are introduced as canonical records are added; empty directories do Initial canonical records should be YAML where hand-editability is valuable. JSON Schema files in `/schemas` define the contracts. Validation tooling may normalize YAML to JSON internally. -One canonical entity per file is preferred for reviewable diffs: +Exactly one canonical record is stored per YAML/JSON file. This is a repository invariant, not merely a preference: it keeps Git diffs, provenance, review state, and correction history scoped to one stable ID. ```text data/techniques/grouped-query-attention.yaml data/releases/deepseek-v3.yaml data/sources/deepseek-v3-technical-report.yaml +data/claims/deepseek-v3-uses-mla.yaml ``` -Atomic claims may be grouped by subject when that improves review ergonomics, provided every claim retains a globally unique canonical ID. +Do not store arrays of claims or other multi-record containers in canonical files. Related atomic claims may share a naming prefix or directory grouping, but each retains its own file and globally unique canonical ID. ## Identifier conventions @@ -60,7 +61,7 @@ claim:llama-3.1-uses-gqa relationship:llama-3.1-uses-gqa ``` -IDs must remain stable if display names change. +IDs must remain stable if display names change. Entity types are bound to canonical namespaces as documented in `docs/ontology.md`. ## Null, unknown, and not applicable @@ -74,6 +75,10 @@ Do not overload values: Evidence references should identify a source plus the smallest useful locator available: section, table, page, config key, file/line range, release-note heading, or commit. +## Dates and timestamps + +Quote ISO dates and timestamps in hand-authored YAML when practical. The shared record loader normalizes YAML-native `date` and `datetime` scalars to ISO strings before schema validation and generation, so equivalent unquoted YAML remains valid. + ## Review policy A record may move through: diff --git a/data/events/2023-12-08-mixtral-8x7b-weights.yaml b/data/events/2023-12-08-mixtral-8x7b-weights.yaml new file mode 100644 index 0000000..5cf8aff --- /dev/null +++ b/data/events/2023-12-08-mixtral-8x7b-weights.yaml @@ -0,0 +1,9 @@ +id: event:2023-12-08-mixtral-8x7b-weights +type: event +name: Mixtral 8x7B weights released via magnet link +event_type: weights_released +subject: release:mixtral-8x7b +occurred_at: '2023-12-08' +evidence: + - source:mixtral-8x7b-magnet-post +notes: The formal Mistral product announcement followed on 2023-12-11 and is represented as a separate event. diff --git a/data/events/2023-12-11-mixtral-8x7b-announced.yaml b/data/events/2023-12-11-mixtral-8x7b-announced.yaml new file mode 100644 index 0000000..423b77a --- /dev/null +++ b/data/events/2023-12-11-mixtral-8x7b-announced.yaml @@ -0,0 +1,9 @@ +id: event:2023-12-11-mixtral-8x7b-announced +type: event +name: Mixtral 8x7B formally announced by Mistral AI +event_type: model_announced +subject: release:mixtral-8x7b +occurred_at: '2023-12-11' +evidence: + - source:mixtral-8x7b-announcement +notes: Formal product announcement; the weights had already been distributed via magnet link on 2023-12-08. diff --git a/data/events/2023-12-11-mixtral-8x7b-weights.yaml b/data/events/2023-12-11-mixtral-8x7b-weights.yaml deleted file mode 100644 index 52c0716..0000000 --- a/data/events/2023-12-11-mixtral-8x7b-weights.yaml +++ /dev/null @@ -1,8 +0,0 @@ -id: event:2023-12-11-mixtral-8x7b-weights -type: event -name: Mixtral 8x7B open weights released -event_type: weights_released -subject: release:mixtral-8x7b -occurred_at: '2023-12-11' -evidence: - - source:mixtral-8x7b-announcement diff --git a/data/sources/mixtral-8x7b-magnet-post.yaml b/data/sources/mixtral-8x7b-magnet-post.yaml new file mode 100644 index 0000000..7d9cdf8 --- /dev/null +++ b/data/sources/mixtral-8x7b-magnet-post.yaml @@ -0,0 +1,9 @@ +id: source:mixtral-8x7b-magnet-post +type: source +title: Mistral AI Mixtral 8x7B magnet-link post +source_type: official_announcement +url: https://x.com/MistralAI/status/1733150512395038967 +publisher: Mistral AI +published_at: '2023-12-08' +retrieved_at: '2026-09-29T07:20:00Z' +notes: Original Mistral AI post distributing the Mixtral 8x7B weights via magnet link. diff --git a/docs/ontology.md b/docs/ontology.md index 3d4ea2a..08f83ec 100644 --- a/docs/ontology.md +++ b/docs/ontology.md @@ -78,6 +78,34 @@ paper:arxiv-2412-19437 benchmark:mmlu ``` +Namespaces are bound to entity types. The v0.1 mapping is: + +| Entity type | Namespace | +|---|---| +| model_family | `family:` | +| model_release | `release:` | +| model_variant | `variant:` | +| architecture_spec | `arch:` | +| technique | `technique:` | +| training_run | `training:` | +| implementation | `implementation:` | +| paper | `paper:` | +| source | `source:` | +| claim | `claim:` | +| relationship | `relationship:` | +| event | `event:` | +| dataset | `dataset:` | +| dataset_version | `dataset-version:` | +| benchmark | `benchmark:` | +| benchmark_version | `benchmark-version:` | +| evaluation_protocol | `eval-protocol:` | +| evaluation_run | `eval-run:` | +| organization | `org:` | +| person | `person:` | +| license | `license:` | +| hardware_system | `hardware:` | +| product_system | `product:` | + Display names and aliases are mutable metadata, not primary keys. ## 3. Claim model @@ -86,16 +114,17 @@ A claim is the smallest independently supportable assertion. ```yaml id: claim:example-model-uses-gqa +type: claim subject: release:example-model predicate: uses_technique object: technique:grouped-query-attention status: confirmed_primary evidence: - - source:example-technical-report -valid_from: 2025-01-01 + - source: source:example-technical-report +valid_from: '2025-01-01' ``` -Claims may point to entities (`object`) or literal values (`value`), but not both simultaneously. +Claims may point to entities (`object`) or literal values (`value`), but not both simultaneously. Claim and relationship endpoints must use canonical IDs, never display names. ## 4. Evidence states @@ -184,6 +213,8 @@ The UI should warn when two displayed results use materially different protocols `occurred_at` belongs to an Event. `published_at` belongs to a Source. `retrieved_at` records when ModelMap observed a source. These dates are intentionally separate. +Quote dates in hand-authored YAML for readability and portability. The validator also normalizes YAML date/timestamp scalars to ISO strings before applying JSON Schema. + ## 10. Source hierarchy Preferred evidence order for technical claims: diff --git a/schemas/claim.schema.json b/schemas/claim.schema.json index ea5bbd2..2587778 100644 --- a/schemas/claim.schema.json +++ b/schemas/claim.schema.json @@ -3,13 +3,19 @@ "$id": "https://modelmap.dev/schemas/claim.schema.json", "title": "ModelMap Claim", "type": "object", + "$defs": { + "canonicalId": { + "type": "string", + "pattern": "^[a-z][a-z0-9_-]*:[a-z0-9][a-z0-9._-]*$" + } + }, "required": ["id", "type", "subject", "predicate", "status", "evidence"], "properties": { "id": {"type": "string", "pattern": "^claim:[a-z0-9][a-z0-9._-]*$"}, "type": {"const": "claim"}, - "subject": {"type": "string", "minLength": 3}, + "subject": {"$ref": "#/$defs/canonicalId"}, "predicate": {"type": "string", "pattern": "^[a-z][a-z0-9_]*$"}, - "object": {"type": "string", "minLength": 3}, + "object": {"$ref": "#/$defs/canonicalId"}, "value": {}, "status": { "type": "string", diff --git a/schemas/entity.schema.json b/schemas/entity.schema.json index 692c873..7e4c1d4 100644 --- a/schemas/entity.schema.json +++ b/schemas/entity.schema.json @@ -54,5 +54,30 @@ "updated_at": {"type": ["string", "null"], "format": "date-time"}, "metadata": {"type": "object", "additionalProperties": true} }, + "allOf": [ + {"if": {"properties": {"type": {"const": "model_family"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^family:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "model_release"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^release:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "model_variant"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^variant:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "architecture_spec"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^arch:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "technique"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^technique:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "training_run"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^training:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "implementation"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^implementation:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "paper"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^paper:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "source"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^source:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "claim"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^claim:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "relationship"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^relationship:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "event"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^event:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "dataset"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^dataset:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "dataset_version"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^dataset-version:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "benchmark"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^benchmark:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "benchmark_version"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^benchmark-version:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "evaluation_protocol"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^eval-protocol:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "evaluation_run"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^eval-run:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "organization"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^org:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "person"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^person:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "license"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^license:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "hardware_system"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^hardware:[a-z0-9][a-z0-9._-]*$"}}}}, + {"if": {"properties": {"type": {"const": "product_system"}}, "required": ["type"]}, "then": {"properties": {"id": {"pattern": "^product:[a-z0-9][a-z0-9._-]*$"}}}} + ], "additionalProperties": false } diff --git a/schemas/event.schema.json b/schemas/event.schema.json index ddd4e36..148b48a 100644 --- a/schemas/event.schema.json +++ b/schemas/event.schema.json @@ -3,6 +3,12 @@ "$id": "https://modelmap.dev/schemas/event.schema.json", "title": "ModelMap Event", "type": "object", + "$defs": { + "canonicalId": { + "type": "string", + "pattern": "^[a-z][a-z0-9_-]*:[a-z0-9][a-z0-9._-]*$" + } + }, "required": ["id", "type", "name", "event_type", "subject", "occurred_at", "evidence"], "properties": { "id": {"type": "string", "pattern": "^event:[a-z0-9][a-z0-9._-]*$"}, @@ -28,7 +34,7 @@ "organization_founded" ] }, - "subject": {"type": "string", "minLength": 3}, + "subject": {"$ref": "#/$defs/canonicalId"}, "occurred_at": {"type": "string", "format": "date"}, "evidence": { "type": "array", diff --git a/schemas/relationship.schema.json b/schemas/relationship.schema.json index 1d940be..fd97814 100644 --- a/schemas/relationship.schema.json +++ b/schemas/relationship.schema.json @@ -3,11 +3,17 @@ "$id": "https://modelmap.dev/schemas/relationship.schema.json", "title": "ModelMap Relationship", "type": "object", + "$defs": { + "canonicalId": { + "type": "string", + "pattern": "^[a-z][a-z0-9_-]*:[a-z0-9][a-z0-9._-]*$" + } + }, "required": ["id", "type", "subject", "predicate", "object"], "properties": { "id": {"type": "string", "pattern": "^relationship:[a-z0-9][a-z0-9._-]*$"}, "type": {"const": "relationship"}, - "subject": {"type": "string", "minLength": 3}, + "subject": {"$ref": "#/$defs/canonicalId"}, "predicate": { "type": "string", "enum": [ @@ -32,7 +38,7 @@ "replaces" ] }, - "object": {"type": "string", "minLength": 3}, + "object": {"$ref": "#/$defs/canonicalId"}, "claim_refs": { "type": "array", "items": {"type": "string", "pattern": "^claim:[a-z0-9][a-z0-9._-]*$"}, diff --git a/schemas/year-view.schema.json b/schemas/year-view.schema.json index aea5a79..740857c 100644 --- a/schemas/year-view.schema.json +++ b/schemas/year-view.schema.json @@ -1,20 +1,30 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://modelmap.dev/schemas/year-view.schema.json", - "title": "ModelMap Year View", + "title": "ModelMap Generated Year View", "type": "object", - "required": ["year", "generated_from", "sections"], + "required": ["year", "generated_from", "events"], "properties": { "year": {"type": "integer", "minimum": 2017}, "generated_from": {"const": "canonical-events"}, - "sections": { + "events": { "type": "array", "items": { "type": "object", - "required": ["name", "event_types"], + "required": ["id", "type", "name", "event_type", "subject", "occurred_at", "evidence"], "properties": { + "id": {"type": "string", "pattern": "^event:[a-z0-9][a-z0-9._-]*$"}, + "type": {"const": "event"}, "name": {"type": "string", "minLength": 1}, - "event_types": {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + "event_type": {"type": "string", "minLength": 1}, + "subject": {"type": "string", "pattern": "^[a-z][a-z0-9_-]*:[a-z0-9][a-z0-9._-]*$"}, + "occurred_at": {"type": "string", "format": "date"}, + "evidence": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "pattern": "^source:[a-z0-9][a-z0-9._-]*$"} + }, + "notes": {"type": ["string", "null"]} }, "additionalProperties": false } diff --git a/tests/test_review_hardening.py b/tests/test_review_hardening.py new file mode 100644 index 0000000..9b01d8f --- /dev/null +++ b/tests/test_review_hardening.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +from pathlib import Path + +from jsonschema import Draft202012Validator, FormatChecker + +ROOT = Path(__file__).resolve().parents[1] +TOOLS = ROOT / "tools" +if str(TOOLS) not in sys.path: + sys.path.insert(0, str(TOOLS)) + +import build_year # noqa: E402 +import records # noqa: E402 + + +class ReviewHardeningTests(unittest.TestCase): + def load_schema(self, name: str) -> dict: + return json.loads((ROOT / "schemas" / name).read_text(encoding="utf-8")) + + def assert_invalid(self, schema_name: str, instance: dict) -> None: + validator = Draft202012Validator( + self.load_schema(schema_name), format_checker=FormatChecker() + ) + self.assertTrue(list(validator.iter_errors(instance))) + + def test_yaml_dates_are_normalized_to_strings(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "record.yaml" + path.write_text("occurred_at: 2025-01-01\n", encoding="utf-8") + loaded = records.load_document(path) + self.assertEqual(loaded["occurred_at"], "2025-01-01") + self.assertIsInstance(loaded["occurred_at"], str) + + def test_all_supported_extensions_are_discovered(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "a.yaml").write_text("id: event:a\n", encoding="utf-8") + (root / "b.yml").write_text("id: event:b\n", encoding="utf-8") + (root / "c.json").write_text('{"id":"event:c"}', encoding="utf-8") + suffixes = {path.suffix for path in records.iter_record_paths(root)} + self.assertEqual(suffixes, {".yaml", ".yml", ".json"}) + + def test_claim_endpoint_requires_canonical_id(self) -> None: + self.assert_invalid( + "claim.schema.json", + { + "id": "claim:bad-endpoint", + "type": "claim", + "subject": "display name", + "predicate": "uses_technique", + "object": "technique:gqa", + "status": "inferred", + "evidence": [], + }, + ) + + def test_relationship_endpoint_requires_canonical_id(self) -> None: + self.assert_invalid( + "relationship.schema.json", + { + "id": "relationship:bad-endpoint", + "type": "relationship", + "subject": "release:model", + "predicate": "release_of", + "object": "Family Name", + }, + ) + + def test_entity_type_is_bound_to_namespace(self) -> None: + self.assert_invalid( + "entity.schema.json", + {"id": "claim:not-a-family", "type": "model_family", "name": "Example"}, + ) + + def test_generated_year_payload_conforms_to_schema(self) -> None: + event = { + "id": "event:example", + "type": "event", + "name": "Example event", + "event_type": "model_announced", + "subject": "release:example", + "occurred_at": "2023-01-01", + "evidence": ["source:example"], + } + payload = build_year.build_payload(2023, [event]) + validator = Draft202012Validator( + self.load_schema("year-view.schema.json"), format_checker=FormatChecker() + ) + self.assertEqual(list(validator.iter_errors(payload)), []) + + def test_canonical_file_contains_one_record(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "claims.yaml").write_text("- id: claim:a\n- id: claim:b\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "exactly one mapping/object record"): + list(records.iter_records(root)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/build_timeline.py b/tools/build_timeline.py index f27c00b..313c209 100755 --- a/tools/build_timeline.py +++ b/tools/build_timeline.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any -import yaml +from records import iter_records ROOT = Path(__file__).resolve().parents[1] EVENTS = ROOT / "data" / "events" @@ -18,12 +18,9 @@ def load_events() -> list[dict[str, Any]]: events: list[dict[str, Any]] = [] if not EVENTS.exists(): return events - for path in sorted(EVENTS.glob("*.yaml")): - with path.open("r", encoding="utf-8") as fh: - record = yaml.safe_load(fh) - if not isinstance(record, dict) or record.get("type") != "event": - continue - events.append(record) + for _, record in iter_records(EVENTS): + if record.get("type") == "event": + events.append(record) return sorted(events, key=lambda item: (item["occurred_at"], item["id"])) diff --git a/tools/build_year.py b/tools/build_year.py old mode 100644 new mode 100755 index b47d2b5..7100c2c --- a/tools/build_year.py +++ b/tools/build_year.py @@ -6,24 +6,33 @@ import argparse import json from pathlib import Path +from typing import Any -import yaml +from records import iter_records ROOT = Path(__file__).resolve().parents[1] EVENTS = ROOT / "data" / "events" -def load_events(year: int) -> list[dict]: - out = [] - for path in sorted(EVENTS.glob("*.yaml")): - with path.open("r", encoding="utf-8") as fh: - event = yaml.safe_load(fh) +def load_events(year: int) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for _, event in iter_records(EVENTS): + if event.get("type") != "event": + continue if str(event.get("occurred_at", "")).startswith(f"{year:04d}-"): out.append(event) return sorted(out, key=lambda e: (e["occurred_at"], e["id"])) -def render_markdown(year: int, events: list[dict]) -> str: +def build_payload(year: int, events: list[dict[str, Any]]) -> dict[str, Any]: + return { + "year": year, + "generated_from": "canonical-events", + "events": events, + } + + +def render_markdown(year: int, events: list[dict[str, Any]]) -> str: lines = [f"# {year}", "", "Generated from canonical ModelMap event records.", ""] for event in events: title = event.get("name") or event["id"].split(":", 1)[-1].replace("-", " ").title() @@ -38,7 +47,7 @@ def main() -> int: args = parser.parse_args() events = load_events(args.year) if args.format == "json": - print(json.dumps({"year": args.year, "events": events}, indent=2)) + print(json.dumps(build_payload(args.year, events), indent=2, sort_keys=True)) else: print(render_markdown(args.year, events), end="") return 0 diff --git a/tools/records.py b/tools/records.py new file mode 100644 index 0000000..4722ae9 --- /dev/null +++ b/tools/records.py @@ -0,0 +1,48 @@ +"""Shared canonical-record loading utilities for ModelMap tools.""" + +from __future__ import annotations + +import json +from datetime import date, datetime +from pathlib import Path +from typing import Any, Iterator + +import yaml + +SUPPORTED_EXTENSIONS = {".yaml", ".yml", ".json"} + + +def normalize_scalars(value: Any) -> Any: + """Convert YAML-native date/time objects to JSON-compatible ISO strings.""" + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, date): + return value.isoformat() + if isinstance(value, list): + return [normalize_scalars(item) for item in value] + if isinstance(value, dict): + return {key: normalize_scalars(item) for key, item in value.items()} + return value + + +def load_document(path: Path) -> Any: + with path.open("r", encoding="utf-8") as fh: + if path.suffix == ".json": + document = json.load(fh) + else: + document = yaml.safe_load(fh) + return normalize_scalars(document) + + +def iter_record_paths(root: Path) -> Iterator[Path]: + for path in sorted(root.rglob("*")): + if path.is_file() and path.suffix in SUPPORTED_EXTENSIONS: + yield path + + +def iter_records(root: Path) -> Iterator[tuple[Path, dict[str, Any]]]: + for path in iter_record_paths(root): + document = load_document(path) + if not isinstance(document, dict): + raise ValueError(f"{path} must contain exactly one mapping/object record") + yield path, document diff --git a/tools/validate.py b/tools/validate.py index b905a29..7693448 100755 --- a/tools/validate.py +++ b/tools/validate.py @@ -8,9 +8,10 @@ from pathlib import Path from typing import Any -import yaml from jsonschema import Draft202012Validator, FormatChecker +from records import iter_records + ROOT = Path(__file__).resolve().parents[1] DATA = ROOT / "data" SCHEMAS = ROOT / "schemas" @@ -24,34 +25,15 @@ } -def load_document(path: Path) -> Any: - with path.open("r", encoding="utf-8") as fh: - if path.suffix == ".json": - return json.load(fh) - return yaml.safe_load(fh) - - def load_schema(name: str) -> dict[str, Any]: with (SCHEMAS / name).open("r", encoding="utf-8") as fh: return json.load(fh) -def iter_records() -> list[tuple[Path, dict[str, Any]]]: - records: list[tuple[Path, dict[str, Any]]] = [] - for path in sorted(DATA.rglob("*")): - if path.suffix not in {".yaml", ".yml", ".json"}: - continue - doc = load_document(path) - if not isinstance(doc, dict): - raise ValueError(f"{path.relative_to(ROOT)} must contain one mapping/object") - records.append((path, doc)) - return records - - def main() -> int: errors: list[str] = [] try: - records = iter_records() + records = list(iter_records(DATA)) except Exception as exc: print(f"ERROR: failed to load corpus: {exc}") return 1 From 99fd1bbf08aa4d1f592fe94f8953b3a460c59237 Mon Sep 17 00:00:00 2001 From: Alajmah <55954113+Alajmah@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:29:05 +0300 Subject: [PATCH 2/5] Restrict canonical endpoint namespaces --- schemas/claim.schema.json | 2 +- schemas/event.schema.json | 2 +- schemas/relationship.schema.json | 2 +- tests/test_review_hardening.py | 14 ++++++++++++++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/schemas/claim.schema.json b/schemas/claim.schema.json index 2587778..491caab 100644 --- a/schemas/claim.schema.json +++ b/schemas/claim.schema.json @@ -6,7 +6,7 @@ "$defs": { "canonicalId": { "type": "string", - "pattern": "^[a-z][a-z0-9_-]*:[a-z0-9][a-z0-9._-]*$" + "pattern": "^(?:family|release|variant|arch|technique|training|implementation|paper|source|claim|relationship|event|dataset|dataset-version|benchmark|benchmark-version|eval-protocol|eval-run|org|person|license|hardware|product):[a-z0-9][a-z0-9._-]*$" } }, "required": ["id", "type", "subject", "predicate", "status", "evidence"], diff --git a/schemas/event.schema.json b/schemas/event.schema.json index 148b48a..4fc4fb3 100644 --- a/schemas/event.schema.json +++ b/schemas/event.schema.json @@ -6,7 +6,7 @@ "$defs": { "canonicalId": { "type": "string", - "pattern": "^[a-z][a-z0-9_-]*:[a-z0-9][a-z0-9._-]*$" + "pattern": "^(?:family|release|variant|arch|technique|training|implementation|paper|source|claim|relationship|event|dataset|dataset-version|benchmark|benchmark-version|eval-protocol|eval-run|org|person|license|hardware|product):[a-z0-9][a-z0-9._-]*$" } }, "required": ["id", "type", "name", "event_type", "subject", "occurred_at", "evidence"], diff --git a/schemas/relationship.schema.json b/schemas/relationship.schema.json index fd97814..f4d7131 100644 --- a/schemas/relationship.schema.json +++ b/schemas/relationship.schema.json @@ -6,7 +6,7 @@ "$defs": { "canonicalId": { "type": "string", - "pattern": "^[a-z][a-z0-9_-]*:[a-z0-9][a-z0-9._-]*$" + "pattern": "^(?:family|release|variant|arch|technique|training|implementation|paper|source|claim|relationship|event|dataset|dataset-version|benchmark|benchmark-version|eval-protocol|eval-run|org|person|license|hardware|product):[a-z0-9][a-z0-9._-]*$" } }, "required": ["id", "type", "subject", "predicate", "object"], diff --git a/tests/test_review_hardening.py b/tests/test_review_hardening.py index 9b01d8f..1849593 100644 --- a/tests/test_review_hardening.py +++ b/tests/test_review_hardening.py @@ -58,6 +58,20 @@ def test_claim_endpoint_requires_canonical_id(self) -> None: }, ) + def test_claim_endpoint_rejects_unsupported_namespace(self) -> None: + self.assert_invalid( + "claim.schema.json", + { + "id": "claim:unsupported-namespace", + "type": "claim", + "subject": "unsupported:anything", + "predicate": "uses_technique", + "object": "technique:gqa", + "status": "inferred", + "evidence": [], + }, + ) + def test_relationship_endpoint_requires_canonical_id(self) -> None: self.assert_invalid( "relationship.schema.json", From 049220bee6911a0dc426b7ed0405b51a211237a1 Mon Sep 17 00:00:00 2001 From: Alajmah <55954113+Alajmah@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:56:13 +0300 Subject: [PATCH 3/5] Tighten year events and relationship evidence validation --- schemas/year-view.schema.json | 65 ++++++++++++++++++++++++---------- tests/test_review_hardening.py | 53 +++++++++++++++++++++++++++ tools/validate.py | 37 ++++++++++++++++--- 3 files changed, 132 insertions(+), 23 deletions(-) mode change 100755 => 100644 tools/validate.py diff --git a/schemas/year-view.schema.json b/schemas/year-view.schema.json index 740857c..09e0fe9 100644 --- a/schemas/year-view.schema.json +++ b/schemas/year-view.schema.json @@ -3,31 +3,58 @@ "$id": "https://modelmap.dev/schemas/year-view.schema.json", "title": "ModelMap Generated Year View", "type": "object", + "$defs": { + "canonicalId": { + "type": "string", + "pattern": "^(?:family|release|variant|arch|technique|training|implementation|paper|source|claim|relationship|event|dataset|dataset-version|benchmark|benchmark-version|eval-protocol|eval-run|org|person|license|hardware|product):[a-z0-9][a-z0-9._-]*$" + }, + "event": { + "type": "object", + "required": ["id", "type", "name", "event_type", "subject", "occurred_at", "evidence"], + "properties": { + "id": {"type": "string", "pattern": "^event:[a-z0-9][a-z0-9._-]*$"}, + "type": {"const": "event"}, + "name": {"type": "string", "minLength": 1}, + "event_type": { + "type": "string", + "enum": [ + "technique_introduced", + "paper_published", + "report_published", + "model_announced", + "weights_released", + "api_released", + "code_released", + "dataset_released", + "license_changed", + "checkpoint_updated", + "context_extended", + "benchmark_published", + "model_deprecated", + "training_completed", + "organization_founded" + ] + }, + "subject": {"$ref": "#/$defs/canonicalId"}, + "occurred_at": {"type": "string", "format": "date"}, + "evidence": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"type": "string", "pattern": "^source:[a-z0-9][a-z0-9._-]*$"} + }, + "notes": {"type": ["string", "null"]} + }, + "additionalProperties": false + } + }, "required": ["year", "generated_from", "events"], "properties": { "year": {"type": "integer", "minimum": 2017}, "generated_from": {"const": "canonical-events"}, "events": { "type": "array", - "items": { - "type": "object", - "required": ["id", "type", "name", "event_type", "subject", "occurred_at", "evidence"], - "properties": { - "id": {"type": "string", "pattern": "^event:[a-z0-9][a-z0-9._-]*$"}, - "type": {"const": "event"}, - "name": {"type": "string", "minLength": 1}, - "event_type": {"type": "string", "minLength": 1}, - "subject": {"type": "string", "pattern": "^[a-z][a-z0-9_-]*:[a-z0-9][a-z0-9._-]*$"}, - "occurred_at": {"type": "string", "format": "date"}, - "evidence": { - "type": "array", - "minItems": 1, - "items": {"type": "string", "pattern": "^source:[a-z0-9][a-z0-9._-]*$"} - }, - "notes": {"type": ["string", "null"]} - }, - "additionalProperties": false - } + "items": {"$ref": "#/$defs/event"} } }, "additionalProperties": false diff --git a/tests/test_review_hardening.py b/tests/test_review_hardening.py index 1849593..608d2c5 100644 --- a/tests/test_review_hardening.py +++ b/tests/test_review_hardening.py @@ -15,6 +15,7 @@ import build_year # noqa: E402 import records # noqa: E402 +import validate as corpus_validate # noqa: E402 class ReviewHardeningTests(unittest.TestCase): @@ -106,6 +107,58 @@ def test_generated_year_payload_conforms_to_schema(self) -> None: ) self.assertEqual(list(validator.iter_errors(payload)), []) + def test_year_view_rejects_noncanonical_embedded_event(self) -> None: + invalid_event = { + "id": "event:example", + "type": "event", + "name": "Example event", + "event_type": "typo", + "subject": "unsupported:anything", + "occurred_at": "2023-01-01", + "evidence": ["source:example"], + } + self.assert_invalid( + "year-view.schema.json", + build_year.build_payload(2023, [invalid_event]), + ) + + def test_relationship_claim_endpoints_must_match(self) -> None: + relationship = { + "subject": "release:example", + "predicate": "uses_technique", + "object": "technique:gqa", + } + matching_claim = { + "subject": "release:example", + "object": "technique:gqa", + } + unrelated_claim = { + "subject": "release:other", + "object": "technique:gqa", + } + self.assertTrue( + corpus_validate.claim_supports_relationship(relationship, matching_claim) + ) + self.assertFalse( + corpus_validate.claim_supports_relationship(relationship, unrelated_claim) + ) + + def test_conceptual_predecessor_claim_uses_reverse_endpoint_order(self) -> None: + relationship = { + "subject": "technique:multi-head-attention", + "predicate": "conceptual_predecessor_of", + "object": "technique:multi-query-attention", + } + successor_perspective_claim = { + "subject": "technique:multi-query-attention", + "object": "technique:multi-head-attention", + } + self.assertTrue( + corpus_validate.claim_supports_relationship( + relationship, successor_perspective_claim + ) + ) + def test_canonical_file_contains_one_record(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/tools/validate.py b/tools/validate.py old mode 100755 new mode 100644 index 7693448..f717b77 --- a/tools/validate.py +++ b/tools/validate.py @@ -30,6 +30,22 @@ def load_schema(name: str) -> dict[str, Any]: return json.load(fh) +def claim_supports_relationship(relationship: dict[str, Any], claim: dict[str, Any]) -> bool: + """Return whether a cited claim supports the relationship's materialized endpoints. + + Conceptual predecessor relationships are intentionally oriented predecessor -> successor, + while their source claims are written from the successor's perspective (for example, + MQA variant_of MHA). All other relationship types require identical endpoint order. + """ + relationship_subject = relationship.get("subject") + relationship_object = relationship.get("object") + if relationship.get("predicate") == "conceptual_predecessor_of": + expected_subject, expected_object = relationship_object, relationship_subject + else: + expected_subject, expected_object = relationship_subject, relationship_object + return claim.get("subject") == expected_subject and claim.get("object") == expected_object + + def main() -> int: errors: list[str] = [] try: @@ -59,20 +75,24 @@ def main() -> int: else: by_id[record_id] = (path, record) - def require_id(owner: Path, ref: Any, expected_type: str | None = None) -> None: + def require_id( + owner: Path, ref: Any, expected_type: str | None = None + ) -> tuple[Path, dict[str, Any]] | None: rel = owner.relative_to(ROOT) if not isinstance(ref, str): errors.append(f"{rel}: reference must be a string, got {type(ref).__name__}") - return + return None target = by_id.get(ref) if target is None: errors.append(f"{rel}: unresolved reference {ref!r}") - return + return None if expected_type is not None and target[1].get("type") != expected_type: errors.append( f"{rel}: {ref!r} must reference type {expected_type!r}, " f"found {target[1].get('type')!r}" ) + return None + return target for path, record in records: record_type = record.get("type") @@ -86,7 +106,16 @@ def require_id(owner: Path, ref: Any, expected_type: str | None = None) -> None: require_id(path, record.get("subject")) require_id(path, record.get("object")) for claim_ref in record.get("claim_refs", []): - require_id(path, claim_ref, "claim") + claim_target = require_id(path, claim_ref, "claim") + if claim_target is None: + continue + claim = claim_target[1] + if not claim_supports_relationship(record, claim): + rel = path.relative_to(ROOT) + errors.append( + f"{rel}: claim_ref {claim_ref!r} does not support relationship " + f"endpoints {record.get('subject')!r} -> {record.get('object')!r}" + ) elif record_type == "event": require_id(path, record.get("subject")) for source_ref in record.get("evidence", []): From 8f275deda2e3b98b6a0c635be31102694ac51923 Mon Sep 17 00:00:00 2001 From: Alajmah <55954113+Alajmah@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:02:55 +0300 Subject: [PATCH 4/5] Validate relationship claim predicates --- tests/test_review_hardening.py | 23 ++++++++++++++++++-- tools/validate.py | 38 +++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/tests/test_review_hardening.py b/tests/test_review_hardening.py index 608d2c5..8a4ec26 100644 --- a/tests/test_review_hardening.py +++ b/tests/test_review_hardening.py @@ -122,7 +122,7 @@ def test_year_view_rejects_noncanonical_embedded_event(self) -> None: build_year.build_payload(2023, [invalid_event]), ) - def test_relationship_claim_endpoints_must_match(self) -> None: + def test_relationship_claim_predicate_and_endpoints_must_match(self) -> None: relationship = { "subject": "release:example", "predicate": "uses_technique", @@ -130,10 +130,17 @@ def test_relationship_claim_endpoints_must_match(self) -> None: } matching_claim = { "subject": "release:example", + "predicate": "uses_technique", "object": "technique:gqa", } unrelated_claim = { "subject": "release:other", + "predicate": "uses_technique", + "object": "technique:gqa", + } + contradictory_claim = { + "subject": "release:example", + "predicate": "does_not_use_technique", "object": "technique:gqa", } self.assertTrue( @@ -142,8 +149,11 @@ def test_relationship_claim_endpoints_must_match(self) -> None: self.assertFalse( corpus_validate.claim_supports_relationship(relationship, unrelated_claim) ) + self.assertFalse( + corpus_validate.claim_supports_relationship(relationship, contradictory_claim) + ) - def test_conceptual_predecessor_claim_uses_reverse_endpoint_order(self) -> None: + def test_conceptual_predecessor_claim_requires_reverse_order_and_mapped_predicate(self) -> None: relationship = { "subject": "technique:multi-head-attention", "predicate": "conceptual_predecessor_of", @@ -151,6 +161,12 @@ def test_conceptual_predecessor_claim_uses_reverse_endpoint_order(self) -> None: } successor_perspective_claim = { "subject": "technique:multi-query-attention", + "predicate": "variant_of_technique", + "object": "technique:multi-head-attention", + } + wrong_predicate_claim = { + "subject": "technique:multi-query-attention", + "predicate": "unrelated", "object": "technique:multi-head-attention", } self.assertTrue( @@ -158,6 +174,9 @@ def test_conceptual_predecessor_claim_uses_reverse_endpoint_order(self) -> None: relationship, successor_perspective_claim ) ) + self.assertFalse( + corpus_validate.claim_supports_relationship(relationship, wrong_predicate_claim) + ) def test_canonical_file_contains_one_record(self) -> None: with tempfile.TemporaryDirectory() as tmp: diff --git a/tools/validate.py b/tools/validate.py index f717b77..c9ecfb1 100644 --- a/tools/validate.py +++ b/tools/validate.py @@ -24,6 +24,12 @@ "event": "event.schema.json", } +CONCEPTUAL_PREDECESSOR_CLAIM_PREDICATES = { + "variant_of_technique", + "generalizes", + "extends", +} + def load_schema(name: str) -> dict[str, Any]: with (SCHEMAS / name).open("r", encoding="utf-8") as fh: @@ -31,19 +37,30 @@ def load_schema(name: str) -> dict[str, Any]: def claim_supports_relationship(relationship: dict[str, Any], claim: dict[str, Any]) -> bool: - """Return whether a cited claim supports the relationship's materialized endpoints. + """Return whether a cited claim supports the relationship's predicate and endpoints. - Conceptual predecessor relationships are intentionally oriented predecessor -> successor, - while their source claims are written from the successor's perspective (for example, - MQA variant_of MHA). All other relationship types require identical endpoint order. + Most materialized relationships require the supporting claim to use the same + subject, predicate, and object. Conceptual-predecessor relationships are the + deliberate exception: the edge is oriented predecessor -> successor while the + source claim is written from the successor's perspective using a small explicit + vocabulary such as `variant_of_technique`, `generalizes`, or `extends`. """ relationship_subject = relationship.get("subject") relationship_object = relationship.get("object") - if relationship.get("predicate") == "conceptual_predecessor_of": - expected_subject, expected_object = relationship_object, relationship_subject - else: - expected_subject, expected_object = relationship_subject, relationship_object - return claim.get("subject") == expected_subject and claim.get("object") == expected_object + relationship_predicate = relationship.get("predicate") + + if relationship_predicate == "conceptual_predecessor_of": + return ( + claim.get("subject") == relationship_object + and claim.get("object") == relationship_subject + and claim.get("predicate") in CONCEPTUAL_PREDECESSOR_CLAIM_PREDICATES + ) + + return ( + claim.get("subject") == relationship_subject + and claim.get("object") == relationship_object + and claim.get("predicate") == relationship_predicate + ) def main() -> int: @@ -114,7 +131,8 @@ def require_id( rel = path.relative_to(ROOT) errors.append( f"{rel}: claim_ref {claim_ref!r} does not support relationship " - f"endpoints {record.get('subject')!r} -> {record.get('object')!r}" + f"predicate/endpoints {record.get('subject')!r} " + f"-{record.get('predicate')}-> {record.get('object')!r}" ) elif record_type == "event": require_id(path, record.get("subject")) From 886f909da14f6a5e533fc07a4b556919990b0480 Mon Sep 17 00:00:00 2001 From: Alajmah <55954113+Alajmah@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:22:19 +0300 Subject: [PATCH 5/5] Restore validator executable mode --- tools/validate.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/validate.py diff --git a/tools/validate.py b/tools/validate.py old mode 100644 new mode 100755