From 08aad66ab6184a32fbd3086e78651b65a4d1272a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 08:34:35 +0000 Subject: [PATCH] feat: add a nightly topic map to the left rail --- .gitignore | 3 + backend/pipeline/topic_map.py | 404 ++++++++++++++++++ backend/tests/pipeline/test_topic_map.py | 304 +++++++++++++ deploy/Caddyfile | 9 + deploy/README.md | 21 +- deploy/agentique-topicmap.service | 18 + deploy/agentique-topicmap.timer | 13 + docs/glossary.md | 2 + docs/product.md | 4 + frontend/src/components/Rails/RailPanel.tsx | 8 +- frontend/src/components/Rails/panels.tsx | 7 + .../src/components/Rails/panels/TopicMap.tsx | 247 +++++++++++ frontend/src/index.css | 3 + frontend/tests/topic-map.spec.ts | 103 +++++ 14 files changed, 1144 insertions(+), 2 deletions(-) create mode 100644 backend/pipeline/topic_map.py create mode 100644 backend/tests/pipeline/test_topic_map.py create mode 100644 deploy/agentique-topicmap.service create mode 100644 deploy/agentique-topicmap.timer create mode 100644 frontend/src/components/Rails/panels/TopicMap.tsx create mode 100644 frontend/tests/topic-map.spec.ts diff --git a/.gitignore b/.gitignore index 3fae04d4..497fea49 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ node_modules/ .ignored/ .DS_Store .env + +# Written by `python -m pipeline.topic_map` in dev; built on the box in prod. +/frontend/public/data/topic-map.json diff --git a/backend/pipeline/topic_map.py b/backend/pipeline/topic_map.py new file mode 100644 index 00000000..f2f34fdf --- /dev/null +++ b/backend/pipeline/topic_map.py @@ -0,0 +1,404 @@ +"""The topic map's data file. Run with: python -m pipeline.topic_map + +Every recent article as a point, grouped by meaning into labelled clusters, for +the left rail. Built nightly from the stored embeddings and written as a static +JSON file the frontend reads directly: no API call, no LLM. + +The window is a week when the week has enough articles to make a map, and a +month otherwise. Deciding that costs one COUNT. The same articles always give +the same picture: input order is fixed, and k-means runs from a fixed seed. +""" + +from __future__ import annotations + +import json +import math +import os +import sys +import tempfile +from collections import Counter +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import TypedDict + +import numpy as np +from sqlmodel import Session, col, func, select + +from app.catalog.models import Article, ArticleTag, Publisher, Tag +from app.newsletter.digest import _vectors +from app.platform.logging import log +from pipeline.db import get_engine + +SCHEMA_VERSION = 1 +# Under this many articles in the last week, a week is too sparse to cluster, +# so the map falls back to a month. +MIN_POINTS = 60 +# The highest-scored this many. More dots than this do not fit the panel. +MAX_POINTS = 300 +WEEK_DAYS = 7 +MONTH_DAYS = 30 +MIN_K = 4 +MAX_K = 8 +# A cluster smaller than this is noise, not a topic. +MIN_CLUSTER = 3 +SEED = 0 +KMEANS_ITERATIONS = 100 +# Cluster centres sit in this band, so a cluster's dots around its centre stay +# inside 0..1. +CENTRE_LOW, CENTRE_HIGH = 0.12, 0.88 +# Centres closer than this get pushed apart, so clusters and their labels do +# not sit on each other. The PCA alone often stacks two of them. +MIN_GAP = 0.3 +SPREAD_ITERATIONS = 100 +# The radius of the widest cluster; a tighter one draws smaller. +CLUSTER_RADIUS = 0.1 +# A tag must label at least this many articles in a cluster to name it. +MIN_TAG_COUNT = 2 +# A second tag joins the label only when the label stays this short. +LABEL_CHARS = 18 +FALLBACK_LABEL = "other" + +DEFAULT_PATH = ( + Path(__file__).resolve().parents[2] / "frontend/public/data/topic-map.json" +) + + +class MapCluster(TypedDict): + id: int + label: str + # The centre, 0..1. + x: float + y: float + size: int + + +class MapPoint(TypedDict): + id: int + x: float + y: float + cluster: int + score: int + title: str + url: str + publisher: str + + +class TopicMap(TypedDict): + """The file, schema v1. The frontend checks ``v`` before reading the rest.""" + + v: int + generated_at: str + window_days: int + since: str + count: int + clusters: list[MapCluster] + points: list[MapPoint] + + +@dataclass(frozen=True) +class Point: + id: int + title: str + url: str + publisher: str + score: int + vector: list[float] + tags: list[str] + + +def window_days(week_count: int) -> int: + return WEEK_DAYS if week_count >= MIN_POINTS else MONTH_DAYS + + +def cluster_count(n: int) -> int: + k = min(max(round(math.sqrt(n / 2)), MIN_K), MAX_K) + return min(k, n) + + +def read_points( + session: Session, now: datetime | None = None +) -> tuple[int, datetime, list[Point]]: + """``(window_days, since, points)``: the window's highest-scored articles.""" + now = now or datetime.now(UTC) + live = col(Article.marked_for_deletion_at).is_(None) + week_count = session.exec( + select(func.count()) + .select_from(Article) + .where(col(Article.created_at) >= now - timedelta(days=WEEK_DAYS), live) + ).one() + days = window_days(week_count) + since = now - timedelta(days=days) + + rows = session.exec( + select(Article, Publisher) + .join(Publisher, col(Publisher.id) == col(Article.publisher_id)) + .where(col(Article.created_at) >= since, live) + .order_by(col(Article.score).desc(), col(Article.id)) + .limit(MAX_POINTS) + ).all() + if not rows: + return days, since, [] + + articles = [a for a, _ in rows] + tags: dict[int, list[str]] = {} + for article_id, name in session.exec( + select(ArticleTag.article_id, Tag.name) + .join(Tag, col(Tag.id) == col(ArticleTag.tag_id)) + .where(col(ArticleTag.article_id).in_([a.id for a in articles])) + .order_by(col(Tag.name)) + ).all(): + tags.setdefault(article_id, []).append(name) + + points = [ + Point( + id=a.id or 0, + title=a.title, + url=a.url, + publisher=p.name, + score=a.score, + vector=v, + tags=tags.get(a.id or 0, []), + ) + for (a, p), v in zip(rows, _vectors(articles), strict=True) + ] + return days, since, points + + +def kmeans(x: np.ndarray, k: int) -> np.ndarray: + """Cluster labels for each row of ``x``: k-means++ init from ``SEED``.""" + rng = np.random.default_rng(SEED) + n = len(x) + centres = [x[rng.integers(n)]] + for _ in range(1, k): + d2 = np.min(((x[:, None, :] - np.array(centres)[None]) ** 2).sum(-1), axis=1) + total = d2.sum() + # Every point already sits on a centre: the rest are duplicates. + pick = rng.integers(n) if total == 0 else rng.choice(n, p=d2 / total) + centres.append(x[pick]) + c = np.array(centres) + + labels = np.full(n, -1) + for _ in range(KMEANS_ITERATIONS): + new = ((x[:, None, :] - c[None]) ** 2).sum(-1).argmin(axis=1) + if np.array_equal(new, labels): + break + labels = new + for j in range(k): + members = x[labels == j] + if len(members): + c[j] = members.mean(axis=0) + return labels + + +def fold_small(x: np.ndarray, labels: np.ndarray) -> np.ndarray: + """Move the points of each cluster under ``MIN_CLUSTER`` to the nearest + other centroid, smallest first, then renumber clusters largest first.""" + labels = labels.copy() + while True: + sizes = Counter(labels.tolist()) + if len(sizes) <= 1: + break + small = min(sizes, key=lambda j: (sizes[j], j)) + if sizes[small] >= MIN_CLUSTER: + break + others = sorted(j for j in sizes if j != small) + centres = np.array([x[labels == j].mean(axis=0) for j in others]) + for i in np.flatnonzero(labels == small): + labels[i] = others[int(((centres - x[i]) ** 2).sum(-1).argmin())] + + sizes = Counter(labels.tolist()) + order = sorted(sizes, key=lambda j: (-sizes[j], j)) + renumber = {old: new for new, old in enumerate(order)} + return np.array([renumber[j] for j in labels.tolist()]) + + +def _pca2(x: np.ndarray) -> np.ndarray: + """Rows of ``x`` projected on their top two principal axes, with each + axis's sign fixed so the result does not depend on the SVD's choice.""" + if len(x) < 2: + return np.zeros((len(x), 2)) + centred = x - x.mean(axis=0) + _, _, vt = np.linalg.svd(centred, full_matrices=False) + axes = vt[:2] + for i, axis in enumerate(axes): + if axis[np.abs(axis).argmax()] < 0: + axes[i] = -axis + return centred @ axes.T + + +def _scale(values: np.ndarray, low: float, high: float) -> np.ndarray: + span = values.max() - values.min() + if span < 1e-12: + return np.full_like(values, (low + high) / 2) + return low + (values - values.min()) / span * (high - low) + + +def spread_apart(centres: np.ndarray) -> np.ndarray: + """Push centres closer than ``MIN_GAP`` apart, inside the centre band.""" + c = centres.copy() + m = len(c) + for _ in range(SPREAD_ITERATIONS): + moved = False + for i in range(m): + for j in range(i + 1, m): + d = c[j] - c[i] + dist = float(np.linalg.norm(d)) + if dist >= MIN_GAP: + continue + # Two centres on one spot: split them along a fixed direction. + unit = d / dist if dist > 1e-9 else np.array([1.0, 0.0]) + step = (MIN_GAP - dist) / 2 * unit + c[i] -= step + c[j] += step + moved = True + c = np.clip(c, CENTRE_LOW, CENTRE_HIGH) + if not moved: + break + return c + + +def layout(x: np.ndarray, labels: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """``(centres, points)``, both as 0..1 coordinates. Centres come from a PCA + of the centroids; each point sits around its centre by its cluster's own + PCA, drawn larger for a looser cluster.""" + m = int(labels.max()) + 1 + centroids = np.array([x[labels == j].mean(axis=0) for j in range(m)]) + flat = _pca2(centroids) + centres = spread_apart( + np.column_stack([_scale(flat[:, i], CENTRE_LOW, CENTRE_HIGH) for i in range(2)]) + ) + + spread = np.array( + [np.linalg.norm(x[labels == j] - centroids[j], axis=1).mean() for j in range(m)] + ) + widest = spread.max() or 1.0 + points = np.zeros((len(x), 2)) + for j in range(m): + idx = np.flatnonzero(labels == j) + local = _pca2(x[idx]) + reach = np.linalg.norm(local, axis=1).max() + if reach > 1e-12: + local = local / reach + radius = CLUSTER_RADIUS * (0.5 + 0.5 * spread[j] / widest) + points[idx] = centres[j] + local * radius + return centres, np.clip(points, 0.0, 1.0) + + +def cluster_labels(tags: list[list[str]], labels: np.ndarray) -> list[str]: + """One label per cluster: its tags by lift (share in the cluster over share + overall), so a tag on every article never names one. A tag already naming + a larger cluster is skipped.""" + n = len(tags) + overall = Counter(t for ts in tags for t in set(ts)) + used: set[str] = set() + out: list[str] = [] + for j in range(int(labels.max()) + 1): + members = [tags[i] for i in np.flatnonzero(labels == j)] + inside = Counter(t for ts in members for t in set(ts)) + lift = { + t: (c / len(members)) / (overall[t] / n) + for t, c in inside.items() + if c >= MIN_TAG_COUNT + } + ranked = [ + t + for t in sorted(lift, key=lambda t: (-lift[t], -inside[t], t)) + if lift[t] > 1 and t not in used + ] + if not ranked: + out.append(FALLBACK_LABEL) + continue + label = ranked[0] + if len(ranked) > 1 and len(f"{label} · {ranked[1]}") <= LABEL_CHARS: + label = f"{label} · {ranked[1]}" + used.add(ranked[0]) + out.append(label) + return out + + +def build(points: list[Point], days: int, since: datetime, now: datetime) -> TopicMap: + """The file's contents. Pure.""" + data: TopicMap = { + "v": SCHEMA_VERSION, + "generated_at": now.isoformat(), + "window_days": days, + "since": since.isoformat(), + "count": len(points), + "clusters": [], + "points": [], + } + if not points: + return data + + x = np.array([p.vector for p in points], dtype=np.float64) + norms = np.linalg.norm(x, axis=1, keepdims=True) + x = x / np.where(norms == 0, 1.0, norms) + + labels = fold_small(x, kmeans(x, cluster_count(len(points)))) + centres, xy = layout(x, labels) + names = cluster_labels([p.tags for p in points], labels) + sizes = Counter(labels.tolist()) + + data["clusters"] = [ + { + "id": j, + "label": names[j], + "x": round(float(centres[j, 0]), 4), + "y": round(float(centres[j, 1]), 4), + "size": sizes[j], + } + for j in range(len(names)) + ] + data["points"] = [ + { + "id": p.id, + "x": round(float(xy[i, 0]), 4), + "y": round(float(xy[i, 1]), 4), + "cluster": int(labels[i]), + "score": p.score, + "title": p.title, + "url": p.url, + "publisher": p.publisher, + } + for i, p in enumerate(points) + ] + return data + + +def write_atomic(path: Path, data: TopicMap) -> None: + """Write to a temp file beside ``path``, then rename over it, so a reader + never sees half a file and a failed run leaves the last good one.""" + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, separators=(",", ":")) + f.flush() + os.fsync(f.fileno()) + # mkstemp makes the file 0600; Caddy reads it as another user. + os.chmod(tmp, 0o644) + os.replace(tmp, path) + except BaseException: + Path(tmp).unlink(missing_ok=True) + raise + + +def output_path() -> Path: + return Path(os.environ.get("TOPIC_MAP_PATH") or DEFAULT_PATH) + + +if __name__ == "__main__": + try: + now = datetime.now(UTC) + with Session(get_engine()) as session: + days, since, points = read_points(session, now) + data = build(points, days, since, now) + write_atomic(output_path(), data) + log( + f"Topic map: {days}-day window, {data['count']} articles, " + f"k={len(data['clusters'])}" + ) + except Exception as e: + print(f"Topic map failed: {e}", file=sys.stderr) + sys.exit(1) diff --git a/backend/tests/pipeline/test_topic_map.py b/backend/tests/pipeline/test_topic_map.py new file mode 100644 index 00000000..807f0ce2 --- /dev/null +++ b/backend/tests/pipeline/test_topic_map.py @@ -0,0 +1,304 @@ +"""The topic map file: its window, its clusters, its labels, and how it lands. + +The clustering and layout are pure, so they run on made-up vectors: tight +groups along separate axes, which any sane k-means splits the same way. The +window and the read are pinned against the real schema, dated far in the +future so nothing else in the test database falls inside it. +""" + +from __future__ import annotations + +import json +from collections import Counter +from collections.abc import Generator +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import numpy as np +import pytest +from sqlmodel import Session, col, delete, select + +from app.catalog.models import ( + Article, + ArticleTag, + Publisher, + PublisherKind, + PublisherType, + Tag, +) +from pipeline import topic_map +from pipeline.topic_map import ( + MIN_POINTS, + MONTH_DAYS, + WEEK_DAYS, + Point, + TopicMap, + build, + fold_small, + window_days, + write_atomic, +) +from tests.random_data import random_lower_string + +DIM = 256 +NOW = datetime(2100, 1, 15, 6, 10, tzinfo=UTC) + + +def _group(axis: int, size: int, tags: list[str], seed: int) -> list[Point]: + rng = np.random.default_rng(seed) + out = [] + for i in range(size): + v = rng.normal(0, 0.05, DIM) + v[axis] += 1.0 + out.append( + Point( + id=axis * 1000 + i, + title=f"Article {axis}-{i}", + url=f"https://example.com/{axis}/{i}", + publisher=f"Publisher {axis}", + score=50 + (i * 7) % 50, + vector=v.tolist(), + tags=list(tags), + ) + ) + return out + + +def _world() -> list[Point]: + # Four topics. "agents" is on every article, so it names none of them. + return ( + _group(0, 14, ["agents", "retrieval"], 1) + + _group(1, 12, ["agents", "open weights"], 2) + + _group(2, 10, ["agents", "evals"], 3) + + _group(3, 9, ["agents", "inference"], 4) + ) + + +def _build(points: list[Point]) -> TopicMap: + return build(points, WEEK_DAYS, NOW - timedelta(days=WEEK_DAYS), NOW) + + +# ─── window ────────────────────────────────────────────────────────────────── + + +def test_the_window_is_a_week_from_min_points(): + assert window_days(MIN_POINTS) == WEEK_DAYS + assert window_days(MIN_POINTS + 40) == WEEK_DAYS + + +def test_the_window_is_a_month_below_min_points(): + assert window_days(MIN_POINTS - 1) == MONTH_DAYS + assert window_days(0) == MONTH_DAYS + + +# ─── clusters ──────────────────────────────────────────────────────────────── + + +def test_the_same_input_gives_the_same_file(): + assert json.dumps(_build(_world())) == json.dumps(_build(_world())) + + +def test_each_topic_is_one_cluster(): + data = _build(_world()) + by_publisher: dict[str, set[int]] = {} + for p in data["points"]: + by_publisher.setdefault(p["publisher"], set()).add(p["cluster"]) + assert all(len(c) == 1 for c in by_publisher.values()) + assert len({next(iter(c)) for c in by_publisher.values()}) == 4 + + +def test_small_clusters_fold_into_the_nearest(): + x = np.array([[0.0, 0.0], [0.1, 0.0], [0.0, 0.1], [5.0, 5.0], [5.1, 5.0]]) + # Two points near (5, 5) on their own, and a cluster beside them. + x = np.vstack([x, [[4.9, 4.9], [5.0, 5.2], [5.2, 5.1]]]) + labels = np.array([0, 0, 0, 1, 1, 2, 2, 2]) + folded = fold_small(x, labels) + assert set(folded[3:5].tolist()) == {folded[5]} + assert min(Counter(folded.tolist()).values()) >= topic_map.MIN_CLUSTER + + +def test_no_cluster_in_the_file_is_under_the_minimum(): + stray = _group(4, 2, ["agents"], 5) + data = _build(_world() + stray) + assert all(c["size"] >= topic_map.MIN_CLUSTER for c in data["clusters"]) + + +def test_centres_keep_their_distance(): + data = _build(_world() + _group(4, 6, ["agents", "voice"], 6)) + centres = np.array([[c["x"], c["y"]] for c in data["clusters"]]) + gaps = [ + np.linalg.norm(a - b) for i, a in enumerate(centres) for b in centres[i + 1 :] + ] + assert min(gaps) >= topic_map.MIN_GAP - 0.01 + + +# ─── labels ────────────────────────────────────────────────────────────────── + + +def test_labels_name_the_distinctive_tag_not_the_common_one(): + data = _build(_world()) + labels = {c["label"] for c in data["clusters"]} + assert labels == {"retrieval", "open weights", "evals", "inference"} + + +def test_a_cluster_with_no_distinctive_tag_is_other(): + points = _group(0, 8, ["agents"], 1) + _group(1, 8, ["agents"], 2) + data = _build(points) + assert {c["label"] for c in data["clusters"]} == {"other"} + + +# ─── schema ────────────────────────────────────────────────────────────────── + + +def test_the_file_matches_schema_v1(): + data = _build(_world()) + assert set(data) == { + "v", + "generated_at", + "window_days", + "since", + "count", + "clusters", + "points", + } + assert data["v"] == 1 + assert data["window_days"] == WEEK_DAYS + assert datetime.fromisoformat(data["generated_at"]) == NOW + assert datetime.fromisoformat(data["since"]) == NOW - timedelta(days=WEEK_DAYS) + assert data["count"] == len(data["points"]) == 45 + assert 4 <= len(data["clusters"]) <= 8 + + for c in data["clusters"]: + assert set(c) == {"id", "label", "x", "y", "size"} + assert 0.12 <= c["x"] <= 0.88 and 0.12 <= c["y"] <= 0.88 + assert sum(c["size"] for c in data["clusters"]) == data["count"] + assert [c["id"] for c in data["clusters"]] == list(range(len(data["clusters"]))) + + ids = {c["id"] for c in data["clusters"]} + for p in data["points"]: + assert set(p) == { + "id", + "x", + "y", + "cluster", + "score", + "title", + "url", + "publisher", + } + assert 0 <= p["x"] <= 1 and 0 <= p["y"] <= 1 + assert p["cluster"] in ids + json.dumps(data) + + +def test_an_empty_window_still_writes_a_file(): + data = _build([]) + assert data["count"] == 0 + assert data["clusters"] == [] and data["points"] == [] + + +# ─── writing ───────────────────────────────────────────────────────────────── + + +def test_the_write_lands_whole(tmp_path: Path): + path = tmp_path / "data" / "topic-map.json" + write_atomic(path, _build(_world())) + assert json.loads(path.read_text())["count"] == 45 + assert path.stat().st_mode & 0o777 == 0o644 + assert [p.name for p in path.parent.iterdir()] == ["topic-map.json"] + + +def test_an_interrupted_write_leaves_the_last_good_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + path = tmp_path / "topic-map.json" + write_atomic(path, _build([])) + + def crash(*_args, **_kwargs): + raise KeyboardInterrupt + + monkeypatch.setattr(json, "dump", crash) + with pytest.raises(KeyboardInterrupt): + write_atomic(path, _build(_world())) + assert json.loads(path.read_text())["count"] == 0 + assert [p.name for p in tmp_path.iterdir()] == ["topic-map.json"] + + +def test_the_path_comes_from_the_environment(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("TOPIC_MAP_PATH", "/srv/x/topic-map.json") + assert topic_map.output_path() == Path("/srv/x/topic-map.json") + monkeypatch.delenv("TOPIC_MAP_PATH") + assert topic_map.output_path().parts[-3:] == ("public", "data", "topic-map.json") + + +# ─── against the schema ────────────────────────────────────────────────────── + +TAG = random_lower_string()[:8] + + +@pytest.fixture +def publisher(db: Session) -> Generator[Publisher]: + p = Publisher( + name=f"{TAG} Blog", + slug=f"{TAG}-blog", + kind=PublisherKind.individual, + type=PublisherType.rss, + ) + db.add(p) + db.commit() + yield p + articles = select(Article.id).where(col(Article.publisher_id) == p.id) + db.exec(delete(ArticleTag).where(col(ArticleTag.article_id).in_(articles))) + db.exec(delete(Article).where(col(Article.publisher_id) == p.id)) + db.exec(delete(Tag).where(col(Tag.slug).startswith(TAG))) + db.delete(p) + db.commit() + + +def _add(db: Session, publisher: Publisher, n: int, days_ago: float) -> None: + rng = np.random.default_rng(n) + for i in range(n): + db.add( + Article( + title=f"{TAG} {days_ago} {i}", + url=f"https://example.com/{TAG}/{days_ago}/{i}", + publisher_id=publisher.id or 0, + score=40 + i % 60, + embedding=rng.standard_normal(DIM).tolist(), + created_at=NOW - timedelta(days=days_ago), + ) + ) + db.commit() + + +def test_a_sparse_week_reads_the_month(db: Session, publisher: Publisher): + _add(db, publisher, MIN_POINTS - 1, days_ago=2) + _add(db, publisher, 10, days_ago=20) + days, since, points = topic_map.read_points(db, NOW) + assert days == MONTH_DAYS + assert since == NOW - timedelta(days=MONTH_DAYS) + assert len(points) == MIN_POINTS - 1 + 10 + + +def test_a_full_week_reads_the_week_by_score(db: Session, publisher: Publisher): + _add(db, publisher, MIN_POINTS, days_ago=2) + _add(db, publisher, 10, days_ago=20) + tag = Tag(slug=f"{TAG}-evals", name="Evals") + db.add(tag) + db.commit() + top = db.exec( + select(Article) + .where(col(Article.publisher_id) == publisher.id) + .order_by(col(Article.score).desc(), col(Article.id)) + ).first() + assert top is not None + db.add(ArticleTag(article_id=top.id or 0, tag_id=tag.id or 0)) + db.commit() + + days, _, points = topic_map.read_points(db, NOW) + assert days == WEEK_DAYS + assert len(points) == MIN_POINTS + assert [p.score for p in points] == sorted((p.score for p in points), reverse=True) + assert points[0].tags == ["Evals"] + assert points[0].publisher == publisher.name + assert len(points[0].vector) == DIM diff --git a/deploy/Caddyfile b/deploy/Caddyfile index 7f3ba1db..86528dd9 100644 --- a/deploy/Caddyfile +++ b/deploy/Caddyfile @@ -21,6 +21,15 @@ agentique.ch { reverse_proxy 127.0.0.1:8000 } + # Static data files the pipeline writes on the box (the topic map). They + # live outside frontend/dist because the deploy's rsync --delete replaces + # that folder wholesale. + handle_path /data/* { + root * /var/lib/agentique/public + header Cache-Control "public, max-age=900" + file_server + } + # SPA fallback for everything else. handle { try_files {path} /index.html diff --git a/deploy/README.md b/deploy/README.md index 2edab2da..7b6fe0b3 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -11,6 +11,7 @@ How agentique runs in production. Docker runs only the database | `agentique-backend.service` | FastAPI under systemd, 2 workers, bound to `127.0.0.1:8000` | | `agentique-pipeline.service` + `.timer` | the article pipeline, one run daily at 04:00 | | `agentique-report.service` + `.timer` | the night's one report email at 06:00 | +| `agentique-topicmap.service` + `.timer` | the topic map's static data file at 06:10, served by Caddy at `/data/topic-map.json` | | `agentique-backup.service` + `.timer` | nightly DB dump to off-box storage at 03:30 | | `agentique-sql` | run SQL against the prod DB from stdin, inside the `db` container | | `sql-roles.sql` | one-time: creates the read-only and read/write login roles `agentique-sql` uses | @@ -23,7 +24,8 @@ How agentique runs in production. Docker runs only the database the backend. Additive migrations only; rollback is `git revert` + push. - The pipeline needs no restart — the timer starts a fresh process each run. - The night runs in three steps an hour apart: 04:00 the pipeline queues - candidates, 05:00 the agent judges them, 06:00 the report goes out. The box + candidates, 05:00 the agent judges them, 06:00 the report goes out. At + 06:10 the topic map is rebuilt from what landed. The box owns the first and the last as separate timers; the agent runs off-box (see Curation). A failure in one does not silence the others — a night the agent never ran still gets a report, and that report says nothing landed. @@ -128,3 +130,20 @@ Read as raw `os.environ` beside each consumer (ADR 7), so they belong in | --- | --- | --- | | `OPENROUTER_API_KEY` | `pipeline/jev.py` | Jev, which classifies each newsletter link (sponsor, plumbing, on-topic, kind). Unset fails every link, so the newsletter channel yields nothing. | | `PIPELINE_ALERT_EMAIL` | `pipeline/report.py` | Where the daily mail goes (liveness, failing sources and feeds, what landed); falls back to `EMAILS_FROM_EMAIL`. | +| `TOPIC_MAP_PATH` | `pipeline/topic_map.py` | Where the topic map's JSON is written. Must be `/var/lib/agentique/public/topic-map.json`, the folder Caddy serves at `/data/`. Unset writes to `frontend/public/data/` in the repo, the dev default, which no one serves in production. | + +## Topic map + +A static file, not an API route: `pipeline/topic_map.py` writes it nightly and +Caddy serves it at `/data/topic-map.json`. It lives outside `frontend/dist` +because the deploy's `rsync --delete` replaces that folder on every push. A +failed build keeps the last good file (the write is a rename), and the panel +hides itself once the file is more than 3 days old. + +One-time setup on the box: + +1. `sudo install -d -o agentique -g agentique -m 755 /var/lib/agentique/public` +2. Add `TOPIC_MAP_PATH=/var/lib/agentique/public/topic-map.json` to `/opt/agentique/.env`. +3. `sudo cp /opt/agentique/deploy/agentique-topicmap.{service,timer} /etc/systemd/system/ && sudo systemctl daemon-reload && sudo systemctl enable --now agentique-topicmap.timer` +4. `sudo cp /opt/agentique/deploy/Caddyfile /etc/caddy/Caddyfile && sudo systemctl reload caddy` +5. First run: `sudo systemctl start agentique-topicmap`, then `journalctl -u agentique-topicmap -n 5` and `curl -sI https://agentique.ch/data/topic-map.json`. diff --git a/deploy/agentique-topicmap.service b/deploy/agentique-topicmap.service new file mode 100644 index 00000000..843cbf7d --- /dev/null +++ b/deploy/agentique-topicmap.service @@ -0,0 +1,18 @@ +# Install target: /etc/systemd/system/agentique-topicmap.service +# The topic map's data file (see agentique-topicmap.timer for the schedule). +# Manual run: sudo systemctl start agentique-topicmap +[Unit] +Description=agentique topic map (one build) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=agentique +Group=agentique +WorkingDirectory=/opt/agentique/backend +# pipeline/topic_map.py reads raw os.environ and loads no dotenv, same as +# pipeline/run.py -- EnvironmentFile is what feeds it, TOPIC_MAP_PATH included. +EnvironmentFile=/opt/agentique/.env +Environment=HOME=/var/lib/agentique +ExecStart=/opt/agentique/.venv/bin/python -m pipeline.topic_map diff --git a/deploy/agentique-topicmap.timer b/deploy/agentique-topicmap.timer new file mode 100644 index 00000000..f7909e87 --- /dev/null +++ b/deploy/agentique-topicmap.timer @@ -0,0 +1,13 @@ +# Install target: /etc/systemd/system/agentique-topicmap.timer +# After the curation session (05:00) and the report (06:00), so the map holds +# the night's articles. Its own timer, not chained to either: a failed build +# leaves the last good file in place and touches nothing else. +[Unit] +Description=Daily agentique topic map + +[Timer] +OnCalendar=*-*-* 06:10:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/docs/glossary.md b/docs/glossary.md index 4a6e0ed3..e959e04b 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -30,6 +30,8 @@ Glossary for words the code uses. If a term below and the code disagree, the cod - **Issue** - one weekly newsletter email and the blog post with the same text: a 250-450 word explainer of one topic, what it is, how it works and how people are taking it, linking the sources it was written from. The newsletter agent (`.claude/skills/newsletter/SKILL.md`) drafts three a week, one per topic, through `week`, `related` and `draft_issue` (`app/newsletter/`), and a person sends one. Each is a file in `frontend/content/blog/-.md`, the frontmatter giving the title (the subject), the description (the preheader) and the topic; the routine opens one PR with the three, and git stores them. +- **Topic map** - the left-rail panel that draws recent articles as dots grouped by meaning. Built nightly from the stored embeddings by `pipeline/topic_map.py` (06:10, its own timer) into a static JSON file the frontend fetches directly, with no API call: the last 7 days, or 30 when the week has under 60 articles, at most the 300 highest-scored; k-means into 4-8 clusters, each named by its most distinctive tags. Served from `/data/topic-map.json`. + - **Traction** - outside signal that people found a story worth reading, independent of the agent's score: Hacker News points/comments past a minimum, which hold back low-traction submissions from the "everyone can post" source; a repo's stars or a model's downloads, which the curation agent reads with `check_link`; and coverage (above). A first-party URL (`pipeline/first_party.py`) skips the gate - a lab's own announcement counts as news at zero votes. - **Reject** (`Reject`, table `scored_url`) - a URL the funnel turned down, one row per URL. `stage` says which step dropped it - `below_threshold` for the agent's rejects; `thin_repo`, `prefilter`, `duplicate` and `unusable_summary` are on old rows only, from gates that are gone - or `pending`, the one stage that is not a rejection at all: a candidate still waiting on the agent. It keeps what that step saw: title, source, publisher, the first 2000 chars of content, traction, the repo / model / paper / docs `links` the article body makes, score, and the judge's `reason`. `filter_known_urls` reads it so a reject is never judged twice. Holds in a source (HN traction, recency) and a failed LLM call are deliberately not rejects: those come back next run. Rows from before 2026-09-13 carry only the URL. diff --git a/docs/product.md b/docs/product.md index 46623e24..34120a18 100644 --- a/docs/product.md +++ b/docs/product.md @@ -18,6 +18,10 @@ tags and kinds, e.g. "Open weights" or "Harnesses"), and by time window (last 3 days / week / month). A left rail beside the feed shows how that window splits by origin and how its articles were found (publisher feeds, Hacker News, newsletters); a row sets the From filter. +**A topic map beside the feed.** A panel in the left rail draws the week's articles +(the month's, when the week is thin) as dots grouped by meaning into labelled +clusters. A nightly job builds it from the stored embeddings into a static file. + **Semantic search.** Typing a query searches by meaning, not keyword — it embeds your query and finds the nearest articles in vector space, so "how do I run a model locally" surfaces relevant pieces even if none of them use that exact phrase. diff --git a/frontend/src/components/Rails/RailPanel.tsx b/frontend/src/components/Rails/RailPanel.tsx index fbdb1e4e..2c475cad 100644 --- a/frontend/src/components/Rails/RailPanel.tsx +++ b/frontend/src/components/Rails/RailPanel.tsx @@ -12,8 +12,14 @@ export function RailPanel({ panel }: { panel: RailPanelDef }) { const collapsed = isCollapsed(panel.id) const { Component } = panel + // A panel with nothing to show renders null, and an open panel with an + // empty body hides whole, header included. A folded one stays, since its + // body is empty only because it is folded. return ( -
+
+ ) + })} + + {hover && ( +
0.5 ? "-translate-x-full" : "", + hover.y > 0.5 ? "-translate-y-full" : "", + )} + style={{ + left: `${((px(hover.x) + (hover.x > 0.5 ? -8 : 8)) / W) * 100}%`, + top: `${((py(hover.y) + (hover.y > 0.5 ? -8 : 8)) / H) * 100}%`, + }} + > + {hover.title} + + {hover.publisher} + +
+ )} + + +
    + {data.clusters.map((c) => ( +
  • + {c.label}: {c.size} articles +
    • + ))} +
+ + ) +} diff --git a/frontend/src/index.css b/frontend/src/index.css index e36087fd..d3d1a219 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -55,6 +55,7 @@ --font-wire: "IBM Plex Mono", ui-monospace, monospace; --font-body: "IBM Plex Sans Variable", ui-sans-serif, system-ui, sans-serif; --color-ink: var(--ink); + --color-ink-2: var(--ink-2); --color-paper: var(--paper); --color-wire: var(--wire); --color-dim: var(--dim); @@ -77,6 +78,8 @@ /* Landing palette. Ink is warm/olive-tinted, not neutral black. */ --ink: #14160e; + /* A step up from ink: the ground of a chart drawn inside a panel. */ + --ink-2: #1b1d14; --paper: #edeade; --wire: #33362a; --dim: #8a8c77; diff --git a/frontend/tests/topic-map.spec.ts b/frontend/tests/topic-map.spec.ts new file mode 100644 index 00000000..0308e385 --- /dev/null +++ b/frontend/tests/topic-map.spec.ts @@ -0,0 +1,103 @@ +import { expect, type Page, test } from "@playwright/test" + +// Wide enough that the rails sit inline beside the feed, always open. +test.use({ viewport: { width: 1440, height: 900 } }) + +const CLUSTERS = [ + { id: 0, label: "Coding agents", x: 0.3, y: 0.3, size: 4 }, + { id: 1, label: "Open weights", x: 0.7, y: 0.3, size: 3 }, + { id: 2, label: "Retrieval", x: 0.5, y: 0.75, size: 3 }, +] + +function fixture(generatedAt = new Date()) { + const points = CLUSTERS.flatMap((c) => + Array.from({ length: c.size }, (_, i) => ({ + id: c.id * 100 + i, + x: c.x + (i - 1) * 0.04, + y: c.y + (i % 2) * 0.04, + cluster: c.id, + score: i === 0 ? 88 : 55, + title: `${c.label} article ${i}`, + url: `https://example.com/${c.id}/${i}`, + publisher: `Publisher ${c.id}`, + })), + ) + return { + v: 1, + generated_at: generatedAt.toISOString(), + window_days: 7, + since: new Date(generatedAt.getTime() - 7 * 864e5).toISOString(), + count: points.length, + clusters: CLUSTERS, + points, + } +} + +async function serve(page: Page, body: unknown, status = 200) { + await page.route("/data/topic-map.json", (route) => + route.fulfill({ + status, + contentType: "application/json", + body: JSON.stringify(body), + }), + ) +} + +test("the topic map draws a dot per article and a label per cluster", async ({ + page, +}) => { + await serve(page, fixture()) + await page.goto("/") + + const map = page.getByTestId("topic-map") + await expect(map).toBeVisible() + await expect(map.getByText("this week · 10")).toBeVisible() + await expect(map.getByTestId("topic-map-dot")).toHaveCount(10) + for (const c of CLUSTERS) { + await expect( + map.getByRole("button", { name: `Highlight ${c.label}` }), + ).toBeVisible() + } + await expect(map.getByRole("img")).toHaveAttribute( + "aria-label", + "10 articles in 3 clusters: Coding agents (4), Open weights (3), Retrieval (3)", + ) +}) + +test("clicking a label keeps its cluster and dims the rest", async ({ + page, +}) => { + await serve(page, fixture()) + await page.goto("/") + + const map = page.getByTestId("topic-map") + const label = map.getByRole("button", { name: "Highlight Retrieval" }) + await label.click() + await expect(label).toHaveAttribute("aria-pressed", "true") + await expect( + map.locator('[data-testid="topic-map-dot"].opacity-15'), + ).toHaveCount(7) + await label.click() + await expect( + map.locator('[data-testid="topic-map-dot"].opacity-15'), + ).toHaveCount(0) +}) + +test("a missing file hides the panel", async ({ page }) => { + await page.route("/data/topic-map.json", (route) => + route.fulfill({ status: 404, body: "" }), + ) + await page.goto("/") + + await expect(page.getByTestId("article-row").first()).toBeVisible() + await expect(page.getByTestId("topic-map")).toHaveCount(0) + await expect(page.locator('[data-panel="topic-map"]')).toBeHidden() +}) + +test("a stale file hides the panel", async ({ page }) => { + await serve(page, fixture(new Date(Date.now() - 4 * 864e5))) + await page.goto("/") + + await expect(page.getByTestId("article-row").first()).toBeVisible() + await expect(page.locator('[data-panel="topic-map"]')).toBeHidden() +})