diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index b1d2c0a..ed124b2 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -22,3 +22,5 @@ jobs: run: python -m pip install -r requirements-dev.txt - name: Validate schemas, records, and references run: python tools/validate.py + - name: Build chronological timeline + run: python tools/build_timeline.py --format json > /tmp/modelmap-timeline.json diff --git a/data/claims/deepseek-v3-uses-mtp.yaml b/data/claims/deepseek-v3-uses-mtp.yaml new file mode 100644 index 0000000..0faa868 --- /dev/null +++ b/data/claims/deepseek-v3-uses-mtp.yaml @@ -0,0 +1,9 @@ +id: claim:deepseek-v3-uses-mtp +type: claim +subject: release:deepseek-v3 +predicate: uses_technique +object: technique:multi-token-prediction +status: confirmed_primary +evidence: + - source: source:deepseek-v3-report + locator: Abstract diff --git a/data/claims/mistral-7b-uses-gqa.yaml b/data/claims/mistral-7b-uses-gqa.yaml new file mode 100644 index 0000000..271772b --- /dev/null +++ b/data/claims/mistral-7b-uses-gqa.yaml @@ -0,0 +1,9 @@ +id: claim:mistral-7b-uses-gqa +type: claim +subject: release:mistral-7b +predicate: uses_technique +object: technique:grouped-query-attention +status: confirmed_primary +evidence: + - source: source:mistral-7b-paper + locator: Abstract diff --git a/data/claims/mistral-7b-uses-swa.yaml b/data/claims/mistral-7b-uses-swa.yaml new file mode 100644 index 0000000..651dd55 --- /dev/null +++ b/data/claims/mistral-7b-uses-swa.yaml @@ -0,0 +1,9 @@ +id: claim:mistral-7b-uses-swa +type: claim +subject: release:mistral-7b +predicate: uses_technique +object: technique:sliding-window-attention +status: confirmed_primary +evidence: + - source: source:mistral-7b-paper + locator: Abstract diff --git a/data/claims/mixtral-8x7b-uses-sparse-moe.yaml b/data/claims/mixtral-8x7b-uses-sparse-moe.yaml new file mode 100644 index 0000000..064b20b --- /dev/null +++ b/data/claims/mixtral-8x7b-uses-sparse-moe.yaml @@ -0,0 +1,9 @@ +id: claim:mixtral-8x7b-uses-sparse-moe +type: claim +subject: release:mixtral-8x7b +predicate: uses_technique +object: technique:sparse-mixture-of-experts +status: confirmed_primary +evidence: + - source: source:mixtral-paper + locator: Abstract diff --git a/data/events/2017-06-12-transformer-introduced.yaml b/data/events/2017-06-12-transformer-introduced.yaml new file mode 100644 index 0000000..c4145ed --- /dev/null +++ b/data/events/2017-06-12-transformer-introduced.yaml @@ -0,0 +1,8 @@ +id: event:2017-06-12-transformer-introduced +type: event +name: Transformer architecture introduced +event_type: technique_introduced +subject: technique:transformer +occurred_at: '2017-06-12' +evidence: + - source:attention-is-all-you-need diff --git a/data/events/2019-11-06-mqa-introduced.yaml b/data/events/2019-11-06-mqa-introduced.yaml new file mode 100644 index 0000000..6fc6215 --- /dev/null +++ b/data/events/2019-11-06-mqa-introduced.yaml @@ -0,0 +1,8 @@ +id: event:2019-11-06-mqa-introduced +type: event +name: Multi-query attention introduced +event_type: technique_introduced +subject: technique:multi-query-attention +occurred_at: '2019-11-06' +evidence: + - source:multi-query-attention diff --git a/data/events/2021-04-20-rope-introduced.yaml b/data/events/2021-04-20-rope-introduced.yaml new file mode 100644 index 0000000..502b039 --- /dev/null +++ b/data/events/2021-04-20-rope-introduced.yaml @@ -0,0 +1,8 @@ +id: event:2021-04-20-rope-introduced +type: event +name: Rotary Position Embedding introduced +event_type: technique_introduced +subject: technique:rotary-position-embedding +occurred_at: '2021-04-20' +evidence: + - source:roformer-paper diff --git a/data/events/2023-05-22-gqa-introduced.yaml b/data/events/2023-05-22-gqa-introduced.yaml new file mode 100644 index 0000000..8203b84 --- /dev/null +++ b/data/events/2023-05-22-gqa-introduced.yaml @@ -0,0 +1,8 @@ +id: event:2023-05-22-gqa-introduced +type: event +name: Grouped-query attention introduced +event_type: technique_introduced +subject: technique:grouped-query-attention +occurred_at: '2023-05-22' +evidence: + - source:gqa-paper diff --git a/data/events/2023-07-18-llama-2-paper.yaml b/data/events/2023-07-18-llama-2-paper.yaml new file mode 100644 index 0000000..1489a75 --- /dev/null +++ b/data/events/2023-07-18-llama-2-paper.yaml @@ -0,0 +1,8 @@ +id: event:2023-07-18-llama-2-paper +type: event +name: Llama 2 technical report published +event_type: report_published +subject: release:llama-2 +occurred_at: '2023-07-18' +evidence: + - source:llama-2-paper diff --git a/data/events/2023-10-10-mistral-7b-paper.yaml b/data/events/2023-10-10-mistral-7b-paper.yaml new file mode 100644 index 0000000..159df31 --- /dev/null +++ b/data/events/2023-10-10-mistral-7b-paper.yaml @@ -0,0 +1,8 @@ +id: event:2023-10-10-mistral-7b-paper +type: event +name: Mistral 7B paper published +event_type: paper_published +subject: release:mistral-7b +occurred_at: '2023-10-10' +evidence: + - source:mistral-7b-paper diff --git a/data/events/2024-01-08-mixtral-paper.yaml b/data/events/2024-01-08-mixtral-paper.yaml new file mode 100644 index 0000000..e476e15 --- /dev/null +++ b/data/events/2024-01-08-mixtral-paper.yaml @@ -0,0 +1,8 @@ +id: event:2024-01-08-mixtral-paper +type: event +name: Mixtral of Experts paper published +event_type: paper_published +subject: release:mixtral-8x7b +occurred_at: '2024-01-08' +evidence: + - source:mixtral-paper diff --git a/data/events/2024-01-11-deepseekmoe-introduced.yaml b/data/events/2024-01-11-deepseekmoe-introduced.yaml new file mode 100644 index 0000000..dd39a8e --- /dev/null +++ b/data/events/2024-01-11-deepseekmoe-introduced.yaml @@ -0,0 +1,8 @@ +id: event:2024-01-11-deepseekmoe-introduced +type: event +name: DeepSeekMoE introduced +event_type: technique_introduced +subject: technique:deepseek-moe +occurred_at: '2024-01-11' +evidence: + - source:deepseekmoe-paper diff --git a/data/events/2024-04-30-mtp-introduced.yaml b/data/events/2024-04-30-mtp-introduced.yaml new file mode 100644 index 0000000..14127fa --- /dev/null +++ b/data/events/2024-04-30-mtp-introduced.yaml @@ -0,0 +1,8 @@ +id: event:2024-04-30-mtp-introduced +type: event +name: Multi-token prediction proposed for large language models +event_type: technique_introduced +subject: technique:multi-token-prediction +occurred_at: '2024-04-30' +evidence: + - source:mtp-paper diff --git a/data/events/2024-05-07-deepseek-v2-paper.yaml b/data/events/2024-05-07-deepseek-v2-paper.yaml new file mode 100644 index 0000000..fd0a178 --- /dev/null +++ b/data/events/2024-05-07-deepseek-v2-paper.yaml @@ -0,0 +1,8 @@ +id: event:2024-05-07-deepseek-v2-paper +type: event +name: DeepSeek-V2 paper published +event_type: paper_published +subject: release:deepseek-v2 +occurred_at: '2024-05-07' +evidence: + - source:deepseek-v2-paper diff --git a/data/events/2024-07-31-llama-3-report.yaml b/data/events/2024-07-31-llama-3-report.yaml new file mode 100644 index 0000000..b55ac8b --- /dev/null +++ b/data/events/2024-07-31-llama-3-report.yaml @@ -0,0 +1,8 @@ +id: event:2024-07-31-llama-3-report +type: event +name: Llama 3 technical report published +event_type: report_published +subject: release:llama-3 +occurred_at: '2024-07-31' +evidence: + - source:llama-3-paper diff --git a/data/events/2024-12-27-deepseek-v3-report.yaml b/data/events/2024-12-27-deepseek-v3-report.yaml new file mode 100644 index 0000000..d6867ff --- /dev/null +++ b/data/events/2024-12-27-deepseek-v3-report.yaml @@ -0,0 +1,8 @@ +id: event:2024-12-27-deepseek-v3-report +type: event +name: DeepSeek-V3 technical report published +event_type: report_published +subject: release:deepseek-v3 +occurred_at: '2024-12-27' +evidence: + - source:deepseek-v3-report diff --git a/data/families/mistral.yaml b/data/families/mistral.yaml new file mode 100644 index 0000000..967b5cb --- /dev/null +++ b/data/families/mistral.yaml @@ -0,0 +1,7 @@ +id: family:mistral +type: model_family +name: Mistral +aliases: + - Mistral AI models +description: Model family developed by Mistral AI, including dense Mistral models and sparse Mixtral models. +status: reviewed diff --git a/data/relationships/deepseek-v3-uses-mtp.yaml b/data/relationships/deepseek-v3-uses-mtp.yaml new file mode 100644 index 0000000..72e463f --- /dev/null +++ b/data/relationships/deepseek-v3-uses-mtp.yaml @@ -0,0 +1,7 @@ +id: relationship:deepseek-v3-uses-mtp +type: relationship +subject: release:deepseek-v3 +predicate: uses_technique +object: technique:multi-token-prediction +claim_refs: + - claim:deepseek-v3-uses-mtp diff --git a/data/relationships/mistral-7b-release-of-mistral.yaml b/data/relationships/mistral-7b-release-of-mistral.yaml new file mode 100644 index 0000000..1726aa4 --- /dev/null +++ b/data/relationships/mistral-7b-release-of-mistral.yaml @@ -0,0 +1,5 @@ +id: relationship:mistral-7b-release-of-mistral +type: relationship +subject: release:mistral-7b +predicate: release_of +object: family:mistral diff --git a/data/relationships/mistral-7b-uses-gqa.yaml b/data/relationships/mistral-7b-uses-gqa.yaml new file mode 100644 index 0000000..771142d --- /dev/null +++ b/data/relationships/mistral-7b-uses-gqa.yaml @@ -0,0 +1,7 @@ +id: relationship:mistral-7b-uses-gqa +type: relationship +subject: release:mistral-7b +predicate: uses_technique +object: technique:grouped-query-attention +claim_refs: + - claim:mistral-7b-uses-gqa diff --git a/data/relationships/mistral-7b-uses-swa.yaml b/data/relationships/mistral-7b-uses-swa.yaml new file mode 100644 index 0000000..a5ac8ff --- /dev/null +++ b/data/relationships/mistral-7b-uses-swa.yaml @@ -0,0 +1,7 @@ +id: relationship:mistral-7b-uses-swa +type: relationship +subject: release:mistral-7b +predicate: uses_technique +object: technique:sliding-window-attention +claim_refs: + - claim:mistral-7b-uses-swa diff --git a/data/relationships/mixtral-8x7b-release-of-mistral.yaml b/data/relationships/mixtral-8x7b-release-of-mistral.yaml new file mode 100644 index 0000000..1637172 --- /dev/null +++ b/data/relationships/mixtral-8x7b-release-of-mistral.yaml @@ -0,0 +1,5 @@ +id: relationship:mixtral-8x7b-release-of-mistral +type: relationship +subject: release:mixtral-8x7b +predicate: release_of +object: family:mistral diff --git a/data/relationships/mixtral-8x7b-uses-sparse-moe.yaml b/data/relationships/mixtral-8x7b-uses-sparse-moe.yaml new file mode 100644 index 0000000..a9b8e0a --- /dev/null +++ b/data/relationships/mixtral-8x7b-uses-sparse-moe.yaml @@ -0,0 +1,7 @@ +id: relationship:mixtral-8x7b-uses-sparse-moe +type: relationship +subject: release:mixtral-8x7b +predicate: uses_technique +object: technique:sparse-mixture-of-experts +claim_refs: + - claim:mixtral-8x7b-uses-sparse-moe diff --git a/data/releases/mistral-7b.yaml b/data/releases/mistral-7b.yaml new file mode 100644 index 0000000..281e83c --- /dev/null +++ b/data/releases/mistral-7b.yaml @@ -0,0 +1,9 @@ +id: release:mistral-7b +type: model_release +name: Mistral 7B +aliases: + - Mistral 7B v0.1 +status: reviewed +metadata: + family: family:mistral + first_publication: '2023-10-10' diff --git a/data/releases/mixtral-8x7b.yaml b/data/releases/mixtral-8x7b.yaml new file mode 100644 index 0000000..77958b7 --- /dev/null +++ b/data/releases/mixtral-8x7b.yaml @@ -0,0 +1,7 @@ +id: release:mixtral-8x7b +type: model_release +name: Mixtral 8x7B +status: reviewed +metadata: + family: family:mistral + first_publication: '2024-01-08' diff --git a/data/sources/attention-is-all-you-need.yaml b/data/sources/attention-is-all-you-need.yaml new file mode 100644 index 0000000..bebc9f9 --- /dev/null +++ b/data/sources/attention-is-all-you-need.yaml @@ -0,0 +1,8 @@ +id: source:attention-is-all-you-need +type: source +title: "Attention Is All You Need" +source_type: paper +url: https://arxiv.org/abs/1706.03762 +publisher: arXiv +published_at: '2017-06-12' +version: v1 diff --git a/data/sources/mistral-7b-paper.yaml b/data/sources/mistral-7b-paper.yaml new file mode 100644 index 0000000..9f66644 --- /dev/null +++ b/data/sources/mistral-7b-paper.yaml @@ -0,0 +1,8 @@ +id: source:mistral-7b-paper +type: source +title: "Mistral 7B" +source_type: paper +url: https://arxiv.org/abs/2310.06825 +publisher: arXiv +published_at: '2023-10-10' +version: v1 diff --git a/data/sources/mixtral-paper.yaml b/data/sources/mixtral-paper.yaml new file mode 100644 index 0000000..8aa6a8b --- /dev/null +++ b/data/sources/mixtral-paper.yaml @@ -0,0 +1,8 @@ +id: source:mixtral-paper +type: source +title: "Mixtral of Experts" +source_type: paper +url: https://arxiv.org/abs/2401.04088 +publisher: arXiv +published_at: '2024-01-08' +version: v1 diff --git a/data/sources/mtp-paper.yaml b/data/sources/mtp-paper.yaml new file mode 100644 index 0000000..c215d17 --- /dev/null +++ b/data/sources/mtp-paper.yaml @@ -0,0 +1,8 @@ +id: source:mtp-paper +type: source +title: "Better & Faster Large Language Models via Multi-token Prediction" +source_type: paper +url: https://arxiv.org/abs/2404.19737 +publisher: arXiv +published_at: '2024-04-30' +version: v1 diff --git a/data/sources/roformer-paper.yaml b/data/sources/roformer-paper.yaml new file mode 100644 index 0000000..025674b --- /dev/null +++ b/data/sources/roformer-paper.yaml @@ -0,0 +1,8 @@ +id: source:roformer-paper +type: source +title: "RoFormer: Enhanced Transformer with Rotary Position Embedding" +source_type: paper +url: https://arxiv.org/abs/2104.09864 +publisher: arXiv +published_at: '2021-04-20' +version: v1 diff --git a/data/techniques/multi-token-prediction.yaml b/data/techniques/multi-token-prediction.yaml new file mode 100644 index 0000000..7e18510 --- /dev/null +++ b/data/techniques/multi-token-prediction.yaml @@ -0,0 +1,9 @@ +id: technique:multi-token-prediction +type: technique +name: Multi-token Prediction +aliases: + - MTP +description: Auxiliary training objective that predicts multiple future tokens from each position rather than only the immediate next token. +status: reviewed +metadata: + domain: training_objective diff --git a/data/techniques/rotary-position-embedding.yaml b/data/techniques/rotary-position-embedding.yaml new file mode 100644 index 0000000..54eac89 --- /dev/null +++ b/data/techniques/rotary-position-embedding.yaml @@ -0,0 +1,9 @@ +id: technique:rotary-position-embedding +type: technique +name: Rotary Position Embedding +aliases: + - RoPE +description: Positional encoding method using rotations to incorporate absolute position and relative-position dependence in attention. +status: reviewed +metadata: + domain: positional_encoding diff --git a/data/techniques/sliding-window-attention.yaml b/data/techniques/sliding-window-attention.yaml new file mode 100644 index 0000000..658099f --- /dev/null +++ b/data/techniques/sliding-window-attention.yaml @@ -0,0 +1,9 @@ +id: technique:sliding-window-attention +type: technique +name: Sliding Window Attention +aliases: + - SWA +description: Attention restricted to a moving local window to reduce inference cost and bound attention span per layer. +status: reviewed +metadata: + domain: attention diff --git a/data/techniques/transformer.yaml b/data/techniques/transformer.yaml new file mode 100644 index 0000000..b9b51c3 --- /dev/null +++ b/data/techniques/transformer.yaml @@ -0,0 +1,9 @@ +id: technique:transformer +type: technique +name: Transformer +aliases: + - Transformer architecture +description: Attention-based sequence architecture introduced in Attention Is All You Need. +status: reviewed +metadata: + domain: architecture diff --git a/schemas/event.schema.json b/schemas/event.schema.json new file mode 100644 index 0000000..ddd4e36 --- /dev/null +++ b/schemas/event.schema.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://modelmap.dev/schemas/event.schema.json", + "title": "ModelMap Event", + "type": "object", + "required": ["id", "type", "name", "event_type", "subject", "occurred_at", "evidence"], + "properties": { + "id": {"type": "string", "pattern": "^event:[a-z0-9][a-z0-9._-]*$"}, + "type": {"const": "event"}, + "name": {"type": "string", "minLength": 1}, + "event_type": { + "type": "string", + "enum": [ + "technique_introduced", + "paper_published", + "report_published", + "model_announced", + "weights_released", + "api_released", + "code_released", + "dataset_released", + "license_changed", + "checkpoint_updated", + "context_extended", + "benchmark_published", + "model_deprecated", + "training_completed", + "organization_founded" + ] + }, + "subject": {"type": "string", "minLength": 3}, + "occurred_at": {"type": "string", "format": "date"}, + "evidence": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"type": "string", "pattern": "^source:[a-z0-9][a-z0-9._-]*$"} + }, + "notes": {"type": ["string", "null"]} + }, + "additionalProperties": false +} diff --git a/tools/build_timeline.py b/tools/build_timeline.py new file mode 100755 index 0000000..f27c00b --- /dev/null +++ b/tools/build_timeline.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Render ModelMap event records as a chronologically sorted timeline.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +EVENTS = ROOT / "data" / "events" + + +def load_events() -> list[dict[str, Any]]: + events: list[dict[str, Any]] = [] + if not EVENTS.exists(): + return events + for path in sorted(EVENTS.glob("*.yaml")): + with path.open("r", encoding="utf-8") as fh: + record = yaml.safe_load(fh) + if not isinstance(record, dict) or record.get("type") != "event": + continue + events.append(record) + return sorted(events, key=lambda item: (item["occurred_at"], item["id"])) + + +def render_markdown(events: list[dict[str, Any]]) -> str: + lines = ["# ModelMap timeline", "", "| Date | Event | Type | Subject |", "|---|---|---|---|"] + for event in events: + lines.append( + f"| {event['occurred_at']} | {event['name']} | {event['event_type']} | `{event['subject']}` |" + ) + return "\n".join(lines) + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--format", choices=["json", "markdown"], default="json") + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + events = load_events() + if args.format == "markdown": + rendered = render_markdown(events) + else: + rendered = json.dumps(events, indent=2, sort_keys=True) + "\n" + + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered, encoding="utf-8") + else: + print(rendered, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/validate.py b/tools/validate.py old mode 100644 new mode 100755 index cddfbd6..b905a29 --- a/tools/validate.py +++ b/tools/validate.py @@ -20,6 +20,7 @@ "claim": "claim.schema.json", "relationship": "relationship.schema.json", "architecture_spec": "architecture.schema.json", + "event": "event.schema.json", } @@ -86,20 +87,28 @@ def require_id(owner: Path, ref: Any, expected_type: str | None = None) -> None: errors.append(f"{rel}: unresolved reference {ref!r}") return if expected_type is not None and target[1].get("type") != expected_type: - errors.append(f"{rel}: {ref!r} must reference type {expected_type!r}, found {target[1].get('type')!r}") + errors.append( + f"{rel}: {ref!r} must reference type {expected_type!r}, " + f"found {target[1].get('type')!r}" + ) for path, record in records: - if record.get("type") == "claim": + record_type = record.get("type") + if record_type == "claim": require_id(path, record.get("subject")) if "object" in record: require_id(path, record.get("object")) for evidence in record.get("evidence", []): require_id(path, evidence.get("source"), "source") - elif record.get("type") == "relationship": + elif record_type == "relationship": require_id(path, record.get("subject")) require_id(path, record.get("object")) for claim_ref in record.get("claim_refs", []): require_id(path, claim_ref, "claim") + elif record_type == "event": + require_id(path, record.get("subject")) + for source_ref in record.get("evidence", []): + require_id(path, source_ref, "source") if errors: print(f"ModelMap validation failed with {len(errors)} error(s):")