From 18024156ac0e8e1b9c00dcae1e3835f7de5548bb Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 08:36:29 +0000 Subject: [PATCH] feat: add agent-named stories to the right rail --- .claude/skills/stories/SKILL.md | 105 +++++ CLAUDE.md | 1 + .../versions/d1e2f3a4b5c6_add_story_tables.py | 49 ++ backend/app/catalog/models.py | 76 +++- backend/app/catalog/routes.py | 13 +- backend/app/catalog/stories.py | 417 ++++++++++++++++++ backend/app/catalog/tests/test_stories.py | 398 +++++++++++++++++ backend/app/mcp/server.py | 10 +- backend/app/mcp/tests/test_curation_tools.py | 6 + backend/app/mcp/tools.py | 68 ++- backend/app/router.py | 2 + backend/scripts/seed_stories.py | 109 +++++ deploy/README.md | 35 ++ docs/adr/0016-stories-named-by-a-routine.md | 64 +++ docs/glossary.md | 2 + docs/product.md | 8 + frontend/src/client/schemas.gen.ts | 103 +++++ frontend/src/client/sdk.gen.ts | 24 +- frontend/src/client/types.gen.ts | 27 ++ .../src/components/Rails/RailsProvider.tsx | 18 +- frontend/src/components/Rails/panels.tsx | 10 +- .../src/components/Rails/panels/Stories.tsx | 183 ++++++++ frontend/tests/stories.spec.ts | 121 +++++ 23 files changed, 1838 insertions(+), 11 deletions(-) create mode 100644 .claude/skills/stories/SKILL.md create mode 100644 backend/app/alembic/versions/d1e2f3a4b5c6_add_story_tables.py create mode 100644 backend/app/catalog/stories.py create mode 100644 backend/app/catalog/tests/test_stories.py create mode 100644 backend/scripts/seed_stories.py create mode 100644 docs/adr/0016-stories-named-by-a-routine.md create mode 100644 frontend/src/components/Rails/panels/Stories.tsx create mode 100644 frontend/tests/stories.spec.ts diff --git a/.claude/skills/stories/SKILL.md b/.claude/skills/stories/SKILL.md new file mode 100644 index 00000000..181928f6 --- /dev/null +++ b/.claude/skills/stories/SKILL.md @@ -0,0 +1,105 @@ +--- +name: stories +description: Keep the right rail's stories - name the threads that run over days or weeks in agentique's feed, add new articles to them, and close the ones that went quiet. Use when running the daily stories session, or when asked to update, name or close stories. +--- + +# Keep the stories + +A story is a theme a builder follows over days or weeks: "Codemode", +"Decision models". The right rail shows each one with a two-sentence blurb and +its articles in order. You decide what is a story, name it, and keep it up to +date. Nothing else writes stories. + +A story is wider than coverage. Coverage is one event told by several outlets +(curation's `similar` and `stories`, at 0.30). A story is several events on +one theme: a launch, a repo built on it, a writer who measured it, a rival's +take a week later. + +## Tools + +From the `agentique` MCP server, with the curation token: + +- `story_candidates(days=14)` - `open`: every open story with its newest 5 + articles and `quiet_days` (since its newest article was published). + `clusters`: the last `days` of articles in no open story, grouped by + embedding, the most publishers first. Each has `articles` (with the + `summary` curation wrote), `publishers`, and `near`: open stories one of its + articles sits close to. A single article close to a story comes back as a + cluster of one with that story in `near`. +- `save_story(slug, name, blurb, urls)` - create a story, or update an open + one: set its name and blurb, add `urls` (ones already in it are skipped). + Refuses the whole call, saying why, on an unknown URL, an article already in + another open story, a bad slug, a name over 4 words, or a blurb over 2 + sentences or 280 characters. +- `close_story(slug)` - close an open story. It leaves the rail and its + articles are free to join another. +- `sql_query(sql)` - read-only, for anything the tools do not answer. + +The rail shows an open story once it has 3 articles from 2 publishers. + +## Rounds + +1. **`story_candidates()` once.** Read every open story and every cluster + before saving anything. +2. **Extend.** For each cluster with a story in `near`: add the articles that + belong to that story's theme. Pass the story's current name and blurb + unless one is now wrong (see **Keeping them stable**). +3. **Start.** For each cluster left that is a story (see **What is a story**), + create one. Before you do, check the open stories again: an article that + fits an open story goes there, even when `near` did not name it. +4. **Close.** Close every open story with `quiet_days` of 21 or more. +5. **Report.** See **Finish**. + +## What is a story + +All of these: + +- 3 or more articles from 2 or more publishers, on one theme. +- A theme a builder can act on: a technique, a protocol, a kind of model, a + way of working. Something they could try, adopt or decide against. +- More than one event. A single launch and its retellings are coverage, not a + story, however many outlets carried it. +- Not one vendor's product line. "Claude releases" is a feed filter; "Computer + use" is a story when several makers and writers are working on it. + +A cluster is a hint, not a verdict. Split one that holds two themes, and leave +out an article that is only near by wording. A cluster of two can start a story +when a third article in the window plainly belongs to it. + +## Naming + +- **Name:** 1-3 words, the term people use. "Codemode", not "Code execution + with MCP tools". "Decision models", not "Small models for agent routing". + If the articles use one name for it, use that name. +- **Slug:** the name in lowercase with hyphens: `codemode`, + `decision-models`. It never changes, so pick it with the name. +- **Blurb:** two plain sentences. The first says what it is, the second why it + matters now. Under 280 characters. The newsletter's plain tone without the + "I": no hype words, no em dashes, no "game-changer", no "landscape". + "The agent writes a short program against your tools instead of calling them + one at a time. Tool definitions stay out of the context; only the result + comes back." + +## Keeping them stable + +- Add to an open story before starting a new one. Two open stories on one + theme split the rail. +- Never rename a story without cause. Cause is: the field settled on another + name for the thing, or the name was wrong. A new article is not a cause. +- Rewrite a blurb only when "why it matters now" is no longer true. +- A closed story stays closed. If its theme comes back, start a new story + under a new slug. + +## Finish + +Report, short: + +- created: slug, name, article count, and the line that decided it is a story +- extended: slug and how many articles added +- closed: slug and its quiet days +- clusters passed over that came close, and why +- anything a tool refused and what you did + +Do not edit files or commit anything; this session reads and writes stories, +nothing else. If a tool answers "This tool needs the curation token", stop and +say so: the session has the read token, not the write token. diff --git a/CLAUDE.md b/CLAUDE.md index edaf5930..f7c2590c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,6 +35,7 @@ - `.mcp.json` connects to the `agentique` MCP server (`backend/app/mcp/`), exposing `sql_query` (read-only prod Postgres), `web_fetch`, `web_search`. - With the write token it also exposes `list_candidates`, `get_content`, `similar`, `stories`, `check_link`, `vocabulary`, `approve`, `reject`, `reject_many` - the curation agent's tools. They publish to prod. Don't call them outside a curation session. - The write token also reaches `week`, `related`, `draft_issue` - the newsletter agent's tools (`.claude/skills/newsletter/SKILL.md`). `draft_issue` creates a Resend draft addressed to every subscriber; sending is done by hand in Resend. +- And `story_candidates`, `save_story`, `close_story` - the stories agent's tools (`.claude/skills/stories/SKILL.md`). They write the right rail's stories in prod. - Don't use any of these unless the user specifically asks for them. Prefer local tools (Read, Grep, the local db) for everything else. ## Curation diff --git a/backend/app/alembic/versions/d1e2f3a4b5c6_add_story_tables.py b/backend/app/alembic/versions/d1e2f3a4b5c6_add_story_tables.py new file mode 100644 index 00000000..0639f2e7 --- /dev/null +++ b/backend/app/alembic/versions/d1e2f3a4b5c6_add_story_tables.py @@ -0,0 +1,49 @@ +"""add story and story_article tables + +Named threads for the right rail, written by the stories routine (ADR 16). + +Revision ID: d1e2f3a4b5c6 +Revises: c5d6e7f8a9b0 +Create Date: 2026-10-10 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op + +revision = "d1e2f3a4b5c6" +down_revision = "c5d6e7f8a9b0" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "story", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("slug", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("blurb", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("last_article_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("closed_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("slug"), + ) + op.create_table( + "story_article", + sa.Column("story_id", sa.Integer(), nullable=False), + sa.Column("article_id", sa.Integer(), nullable=False), + sa.Column("added_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["story_id"], ["story.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["article_id"], ["article.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("story_id", "article_id"), + ) + op.create_index("ix_story_article_article_id", "story_article", ["article_id"]) + + +def downgrade() -> None: + op.drop_index("ix_story_article_article_id", table_name="story_article") + op.drop_table("story_article") + op.drop_table("story") diff --git a/backend/app/catalog/models.py b/backend/app/catalog/models.py index 62b011f1..871ee715 100644 --- a/backend/app/catalog/models.py +++ b/backend/app/catalog/models.py @@ -8,7 +8,7 @@ from enum import StrEnum from pgvector.sqlalchemy import Vector # type: ignore[import-untyped] -from sqlalchemy import JSON, Column, DateTime +from sqlalchemy import JSON, Column, DateTime, Index from sqlmodel import Field, SQLModel from app.platform.dates import get_datetime_utc @@ -221,6 +221,54 @@ class ArticleTag(SQLModel, table=True): tag_id: int = Field(foreign_key="tag.id", primary_key=True) +# ─── Story ───────────────────────────────────────────────────────────────── +# A named thread that runs over days or weeks. Wider than coverage (one event +# told by several outlets): an agent names and keeps stories, see ADR 16. + + +class Story(SQLModel, table=True): + id: int | None = Field(default=None, primary_key=True) + slug: str = Field(unique=True) + name: str + blurb: str + created_at: datetime = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) + updated_at: datetime = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) + # When its newest article was published. What the rail orders by and what + # "quiet" is measured from. + last_article_at: datetime | None = Field( + default=None, + sa_type=DateTime(timezone=True), # type: ignore + ) + closed_at: datetime | None = Field( + default=None, + sa_type=DateTime(timezone=True), # type: ignore + ) + + +class StoryArticle(SQLModel, table=True): + """An article in a story. At most one open story per article, enforced in + `catalog.stories.save_story`: a closed story keeps its articles.""" + + __tablename__ = "story_article" + # The primary key leads with story_id; "which story holds this article" is + # asked on every save. + __table_args__ = (Index("ix_story_article_article_id", "article_id"),) + story_id: int = Field(foreign_key="story.id", primary_key=True, ondelete="CASCADE") + article_id: int = Field( + foreign_key="article.id", primary_key=True, ondelete="CASCADE" + ) + added_at: datetime = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) + + # ─── Read API shapes ─────────────────────────────────────────────────────── # Consumer-facing views. `publisher` is nested (replaces the old `source` string) # and `tags` come from the article_tag join. Pipeline provenance and bulk fields @@ -303,3 +351,29 @@ class OriginCounts(SQLModel): # names its kind. unlabelled: int found_via: list[FoundViaCount] + + +# ─── Stories (right rail) ────────────────────────────────────────────────── + + +class StoryArticlePublic(SQLModel): + id: int + title: str + url: str + publisher: str + published_at: datetime | None = None + score: int + + +class StoryPublic(SQLModel): + slug: str + name: str + blurb: str + article_count: int + publisher_count: int + first_at: datetime | None = None + last_at: datetime | None = None + # An article joined it in the last 24 hours. + grew_today: bool + # Newest first, at most `stories.STORY_ARTICLES` of `article_count`. + articles: list[StoryArticlePublic] diff --git a/backend/app/catalog/routes.py b/backend/app/catalog/routes.py index acd427e1..a64f5a70 100644 --- a/backend/app/catalog/routes.py +++ b/backend/app/catalog/routes.py @@ -1,4 +1,4 @@ -"""Article listing, semantic search, and facet endpoints for the public feed.""" +"""Article listing, semantic search, facet and story endpoints for the public feed.""" from __future__ import annotations @@ -6,7 +6,7 @@ from fastapi import APIRouter, HTTPException, Query -from app.catalog import facets, semantic_search +from app.catalog import facets, semantic_search, stories from app.catalog.articles import list_articles, stats from app.catalog.models import ( ArticleFacets, @@ -14,11 +14,13 @@ Origin, OriginCounts, PublisherFacet, + StoryPublic, TagFacet, ) from app.deps import CurrentUserOptional, SessionDep router = APIRouter(prefix="/articles", tags=["articles"]) +stories_router = APIRouter(prefix="/stories", tags=["stories"]) # Reads are public and unbounded. A caller may still send a token — it only # decides whether `liked_by_me` is filled in. @@ -112,3 +114,10 @@ def search_tags( @router.get("/stats") def article_stats(session: SessionDep) -> dict[str, int | str | None]: return stats(session) + + +@stories_router.get("/", response_model=list[StoryPublic]) +def read_stories( + session: SessionDep, limit: int = Query(default=5, ge=1, le=20) +) -> list[StoryPublic]: + return stories.list_stories(session, limit) diff --git a/backend/app/catalog/stories.py b/backend/app/catalog/stories.py new file mode 100644 index 00000000..7cbceaa5 --- /dev/null +++ b/backend/app/catalog/stories.py @@ -0,0 +1,417 @@ +"""Stories: named threads that run over days or weeks, for the right rail. + +A story is wider than coverage. Coverage is one event told by several outlets, +within `DEDUP_DIST_THRESHOLD`; a story is a theme a builder follows, which +takes judgment, so an agent names and keeps them (ADR 16). This module is what +that agent drives through the MCP tools in `app/mcp/tools.py`, and what the +public rail reads. + +An article sits in at most one open story. Postgres cannot say that without a +trigger (the rule spans two tables), so `save_story` enforces it; it is the +only writer. +""" + +from __future__ import annotations + +import re +from collections import defaultdict +from datetime import UTC, datetime, timedelta +from typing import TypedDict + +import numpy as np +from sqlmodel import Session, col, func, select + +from app.catalog.models import ( + Article, + Publisher, + Story, + StoryArticle, + StoryArticlePublic, + StoryPublic, +) +from pipeline.curation import cosine_distances, story_groups + +# What the rail shows: a story carried by fewer is coverage, or not yet a story. +MIN_ARTICLES = 3 +MIN_PUBLISHERS = 2 +# Articles per story on the rail, newest first. The panel shows six and +# "Show all" the rest, so this is a payload cap rather than a page size. +STORY_ARTICLES = 30 +GREW_WINDOW = timedelta(hours=24) + +# Looser than coverage (0.30) and than `similar`'s related cut (0.45 lists, it +# does not group): a story's articles share a theme, not an event. A first +# value, to tune against what the routine actually groups. +STORY_DIST = 0.40 +# What `story_candidates` shows of each open story. +CANDIDATE_TAIL = 5 +# Clusters per `story_candidates` call, the most publishers first. +CANDIDATE_CLUSTERS = 30 + +SLUG = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +SLUG_MAX = 60 +NAME_WORDS = 4 +BLURB_SENTENCES = 2 +BLURB_CHARS = 280 +# A sentence ends at . ! or ? followed by space and a capital, a digit or a +# quote. Close enough for two plain sentences; "e.g. Claude" would count two. +SENTENCE_BREAK = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9\"'“(])") + + +class StoryError(ValueError): + """A save or close the agent can correct, returned as a message.""" + + +def _published(article_at: datetime | None, created_at: datetime) -> datetime: + return article_at or created_at + + +# ─── Public read ───────────────────────────────────────────────────────────── + + +def list_stories( + session: Session, limit: int = 5, now: datetime | None = None +) -> list[StoryPublic]: + """Open stories with at least `MIN_ARTICLES` articles from + `MIN_PUBLISHERS` publishers, the most recently active first.""" + now = now or datetime.now(UTC) + stats = session.exec( + select( + Story, + func.count(col(Article.id)), + func.count(func.distinct(Article.publisher_id)), + func.max(StoryArticle.added_at), + ) + .join(StoryArticle, col(StoryArticle.story_id) == col(Story.id)) + .join(Article, col(Article.id) == col(StoryArticle.article_id)) + .where(col(Story.closed_at).is_(None)) + .group_by(col(Story.id)) + .having(func.count(col(Article.id)) >= MIN_ARTICLES) + .having(func.count(func.distinct(Article.publisher_id)) >= MIN_PUBLISHERS) + .order_by(col(Story.last_article_at).desc().nulls_last(), col(Story.id)) + .limit(limit) + ).all() + if not stats: + return [] + + # Every member, newest first: the rail shows the head, the span reads both ends. + by_story: dict[int, list[StoryArticlePublic]] = defaultdict(list) + for story_id, article, publisher in _members( + session, [s.id for s, *_ in stats if s.id is not None] + ): + by_story[story_id].append( + StoryArticlePublic( + id=article.id or 0, + title=article.title, + url=article.url, + publisher=publisher, + published_at=_published(article.published_at, article.created_at), + score=article.score, + ) + ) + + out: list[StoryPublic] = [] + for story, count, publishers, last_added in stats: + articles = by_story[story.id or 0] + out.append( + StoryPublic( + slug=story.slug, + name=story.name, + blurb=story.blurb, + article_count=count, + publisher_count=publishers, + first_at=articles[-1].published_at, + last_at=articles[0].published_at, + grew_today=last_added >= now - GREW_WINDOW, + articles=articles[:STORY_ARTICLES], + ) + ) + return out + + +def _members(session: Session, story_ids: list[int]) -> list[tuple[int, Article, str]]: + """Every article in these stories with its publisher's name, newest first.""" + return list( + session.exec( + select(StoryArticle.story_id, Article, Publisher.name) + .join(Article, col(Article.id) == col(StoryArticle.article_id)) + .join(Publisher, col(Publisher.id) == col(Article.publisher_id)) + .where(col(StoryArticle.story_id).in_(story_ids)) + .order_by( + func.coalesce(Article.published_at, Article.created_at).desc(), + col(Article.id).desc(), + ) + ).all() + ) + + +# ─── Agent writes ──────────────────────────────────────────────────────────── + + +class SavedStory(TypedDict): + slug: str + created: bool + # URLs this call added; ones already in the story are not counted again. + added: int + article_count: int + + +def _validate(slug: str, name: str, blurb: str) -> tuple[str, str]: + if len(slug) > SLUG_MAX or not SLUG.match(slug): + raise StoryError( + f"Bad slug {slug!r}: lowercase letters, digits and single hyphens, " + f"up to {SLUG_MAX} characters, e.g. 'decision-models'." + ) + name = " ".join(name.split()) + if not name: + raise StoryError("The name is empty.") + if len(name.split()) > NAME_WORDS: + raise StoryError( + f"Name {name!r} is over {NAME_WORDS} words: use the term people use." + ) + blurb = " ".join(blurb.split()) + if not blurb: + raise StoryError("The blurb is empty.") + if len(blurb) > BLURB_CHARS: + raise StoryError(f"The blurb is {len(blurb)} characters, over {BLURB_CHARS}.") + if len(SENTENCE_BREAK.split(blurb)) > BLURB_SENTENCES: + raise StoryError(f"The blurb is over {BLURB_SENTENCES} sentences.") + return name, blurb + + +def save_story( + session: Session, + slug: str, + name: str, + blurb: str, + urls: list[str], + now: datetime | None = None, +) -> SavedStory: + """Create a story, or rename, reblurb and add articles to an open one. + + All or nothing: an unknown URL, or an article already in another open + story, refuses the whole call and nothing is written. + """ + now = now or datetime.now(UTC) + name, blurb = _validate(slug, name, blurb) + urls = list(dict.fromkeys(u.strip() for u in urls if u.strip())) + + story = session.exec(select(Story).where(col(Story.slug) == slug)).first() + if story is not None and story.closed_at is not None: + raise StoryError( + f"Story {slug!r} is closed. Start a new one under another slug." + ) + if story is None and not urls: + raise StoryError("A new story needs at least one article URL.") + + articles = ( + session.exec(select(Article).where(col(Article.url).in_(urls))).all() + if urls + else [] + ) + unknown = sorted(set(urls) - {a.url for a in articles}) + if unknown: + raise StoryError("Not a published article: " + ", ".join(unknown)) + + article_ids = [a.id for a in articles if a.id is not None] + held = session.exec( + select(Article.url, Story.slug) + .join(StoryArticle, col(StoryArticle.article_id) == col(Article.id)) + .join(Story, col(Story.id) == col(StoryArticle.story_id)) + .where( + col(StoryArticle.article_id).in_(article_ids), + col(Story.closed_at).is_(None), + col(Story.slug) != slug, + ) + ).all() + if held: + raise StoryError( + "Already in another open story: " + + ", ".join(f"{url} ({other})" for url, other in sorted(held)) + ) + + created = story is None + if story is None: + story = Story(slug=slug, name=name, blurb=blurb, created_at=now) + story.name = name + story.blurb = blurb + story.updated_at = now + session.add(story) + session.flush() + assert story.id is not None + + present = set( + session.exec( + select(StoryArticle.article_id).where( + col(StoryArticle.story_id) == story.id + ) + ).all() + ) + added = [i for i in article_ids if i not in present] + for article_id in added: + session.add( + StoryArticle(story_id=story.id, article_id=article_id, added_at=now) + ) + session.flush() + + count, last = session.exec( + select( + func.count(col(Article.id)), + func.max(func.coalesce(Article.published_at, Article.created_at)), + ) + .join(StoryArticle, col(StoryArticle.article_id) == col(Article.id)) + .where(col(StoryArticle.story_id) == story.id) + ).one() + story.last_article_at = last + session.add(story) + session.commit() + return { + "slug": slug, + "created": created, + "added": len(added), + "article_count": int(count), + } + + +def close_story(session: Session, slug: str, now: datetime | None = None) -> None: + """Close an open story. Its articles stay with it and are free to join + another.""" + story = session.exec(select(Story).where(col(Story.slug) == slug)).first() + if story is None: + raise StoryError(f"No story {slug!r}.") + if story.closed_at is not None: + raise StoryError(f"Story {slug!r} is already closed.") + story.closed_at = now or datetime.now(UTC) + story.updated_at = story.closed_at + session.add(story) + session.commit() + + +# ─── Agent reads ───────────────────────────────────────────────────────────── + + +class CandidateArticle(TypedDict): + url: str + title: str + publisher: str + published_at: str + score: int + summary: str + + +class OpenStory(TypedDict): + slug: str + name: str + blurb: str + article_count: int + # Days since its newest article was published. + quiet_days: int + # The newest `CANDIDATE_TAIL`, newest first. + articles: list[CandidateArticle] + + +class Cluster(TypedDict): + # Open stories one of these articles sits close to: add to one of them + # before starting a new story. + near: list[str] + publishers: list[str] + articles: list[CandidateArticle] + + +class Candidates(TypedDict): + open: list[OpenStory] + clusters: list[Cluster] + + +def _candidate(article: Article, publisher: str) -> CandidateArticle: + return { + "url": article.url, + "title": article.title, + "publisher": publisher, + "published_at": _published(article.published_at, article.created_at) + .date() + .isoformat(), + "score": article.score, + "summary": article.summary or "", + } + + +def story_candidates( + session: Session, days: int = 14, now: datetime | None = None +) -> Candidates: + """Open stories with their newest articles, and the last ``days`` of + articles in no open story grouped at `STORY_DIST`. + + Grouped with curation's `story_groups`, the open stories' articles taking + the place its feed rows do: an edge needs a free article at one end, so + two stories never merge through each other, and a cluster that reaches a + story's article names that story in `near`. + """ + now = now or datetime.now(UTC) + since = now - timedelta(days=days) + + stories = session.exec( + select(Story) + .where(col(Story.closed_at).is_(None)) + .order_by(col(Story.last_article_at).desc().nulls_last(), col(Story.id)) + ).all() + members = _members(session, [s.id for s in stories if s.id is not None]) + held: set[int] = set() + by_story: dict[int, list[CandidateArticle]] = defaultdict(list) + # (slug, publisher, vector) for each open story's article that has one. + anchored: list[tuple[str, str, list[float]]] = [] + slug_of = {s.id: s.slug for s in stories} + for story_id, article, publisher in members: + held.add(article.id or 0) + by_story[story_id].append(_candidate(article, publisher)) + # pgvector hands back a numpy array, and bool(array) raises. + if article.embedding is not None: + anchored.append((slug_of[story_id], publisher, list(article.embedding))) + + open_stories: list[OpenStory] = [ + { + "slug": s.slug, + "name": s.name, + "blurb": s.blurb, + "article_count": len(by_story[s.id or 0]), + "quiet_days": (now - s.last_article_at).days if s.last_article_at else 0, + "articles": by_story[s.id or 0][:CANDIDATE_TAIL], + } + for s in stories + ] + + free: list[tuple[Article, str, list[float]]] = [ + (a, p, list(a.embedding)) + for a, p in session.exec( + select(Article, Publisher.name) + .join(Publisher, col(Publisher.id) == col(Article.publisher_id)) + .where( + func.coalesce(Article.published_at, Article.created_at) >= since, + col(Article.embedding).is_not(None), + ) + ).all() + if a.id not in held and a.embedding is not None + ] + if not free: + return {"open": open_stories, "clusters": []} + + vecs = np.array( + [v for _, _, v in free] + [v for _, _, v in anchored], dtype=np.float32 + ) + anchors = [True] * len(free) + [False] * len(anchored) + groups = story_groups(cosine_distances(vecs, vecs), anchors, STORY_DIST) + + clusters: list[Cluster] = [] + for group in groups: + articles = [_candidate(free[i][0], free[i][1]) for i in group if i < len(free)] + reached = [anchored[i - len(free)] for i in group if i >= len(free)] + near = sorted({slug for slug, _, _ in reached}) + publishers = {a["publisher"] for a in articles} | {p for _, p, _ in reached} + articles.sort(key=lambda a: a["published_at"]) + clusters.append( + {"near": near, "publishers": sorted(publishers), "articles": articles} + ) + clusters.sort( + key=lambda c: (len(c["publishers"]), len(c["articles"])), reverse=True + ) + return {"open": open_stories, "clusters": clusters[:CANDIDATE_CLUSTERS]} diff --git a/backend/app/catalog/tests/test_stories.py b/backend/app/catalog/tests/test_stories.py new file mode 100644 index 00000000..4c9f5641 --- /dev/null +++ b/backend/app/catalog/tests/test_stories.py @@ -0,0 +1,398 @@ +"""Stories: the schema, what the rail lists, what the agent may save, and the +story tools' write gate.""" + +from __future__ import annotations + +from collections.abc import Generator +from datetime import UTC, datetime, timedelta + +import numpy as np +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import inspect +from sqlmodel import Session, col, delete, func, select + +from app.catalog import stories +from app.catalog.models import Article, Publisher, Story, StoryArticle +from app.catalog.tests.factories import create_random_article, create_random_publisher +from app.platform.db import engine +from app.platform.settings import settings +from tests.random_data import random_lower_string + +NOW = datetime(2026, 10, 10, 12, tzinfo=UTC) +# Slugs are unique per run: the tests share a database with whatever a +# developer restored, stories included. +RUN = random_lower_string()[:6] + + +def S(slug: str) -> str: + return f"{slug}-{RUN}" + + +DIM = 256 + + +@pytest.fixture +def made(db: Session) -> Generator[dict[str, list[int]]]: + """Ids of what a test created, deleted afterwards with every story made + meanwhile, so the next test starts from the same rows.""" + created: dict[str, list[int]] = {"articles": [], "publishers": []} + before = db.exec(select(func.max(Story.id))).one() or 0 + yield created + db.rollback() + ours = select(Story.id).where(col(Story.id) > before) + db.exec(delete(StoryArticle).where(col(StoryArticle.story_id).in_(ours))) # type: ignore[call-overload] + db.exec(delete(Story).where(col(Story.id) > before)) # type: ignore[call-overload] + db.exec(delete(Article).where(col(Article.id).in_(created["articles"]))) # type: ignore[call-overload] + db.exec(delete(Publisher).where(col(Publisher.id).in_(created["publishers"]))) # type: ignore[call-overload] + db.commit() + + +def _publisher(db: Session, made: dict[str, list[int]]) -> Publisher: + p = create_random_publisher(db) + assert p.id is not None + made["publishers"].append(p.id) + return p + + +def _article( + db: Session, + made: dict[str, list[int]], + publisher: Publisher, + days_ago: float = 1, + **overrides: object, +) -> Article: + a = create_random_article( + db, + publisher_id=publisher.id, + published_at=NOW - timedelta(days=days_ago), + url=f"https://stories.example/{random_lower_string()}", + **overrides, + ) + assert a.id is not None + made["articles"].append(a.id) + return a + + +def _save( + db: Session, + slug: str, + urls: list[str], + name: str | None = None, + blurb: str = "What it is. Why it matters now.", + now: datetime = NOW, +) -> stories.SavedStory: + name = name or slug.replace("-", " ").title() + return stories.save_story(db, slug, name, blurb, urls, now=now) + + +def _story( + db: Session, made: dict[str, list[int]], slug: str, days_ago: float = 1 +) -> list[Article]: + """A story the rail shows: three articles from two publishers.""" + p1, p2 = _publisher(db, made), _publisher(db, made) + articles = [ + _article(db, made, p1, days_ago + 2), + _article(db, made, p2, days_ago + 1), + _article(db, made, p1, days_ago), + ] + _save(db, slug, [a.url for a in articles]) + return articles + + +def _mine(listed: list, slugs: set[str]) -> list[str]: + return [s.slug for s in listed if s.slug in slugs] + + +# ─── Migration ─────────────────────────────────────────────────────────────── + + +def test_the_migration_creates_both_tables(): + schema = inspect(engine) + story = {c["name"] for c in schema.get_columns("story")} + assert story >= { + "id", + "slug", + "name", + "blurb", + "created_at", + "updated_at", + "last_article_at", + "closed_at", + } + assert any( + u["column_names"] == ["slug"] for u in schema.get_unique_constraints("story") + ) + pk = schema.get_pk_constraint("story_article") + assert sorted(pk["constrained_columns"]) == ["article_id", "story_id"] + assert {c["name"] for c in schema.get_columns("story_article")} == { + "story_id", + "article_id", + "added_at", + } + + +# ─── list_stories ──────────────────────────────────────────────────────────── + + +def test_list_needs_three_articles_from_two_publishers( + db: Session, made: dict[str, list[int]] +): + one_publisher = _publisher(db, made) + p1, p2 = _publisher(db, made), _publisher(db, made) + _save( + db, + S("one-outlet"), + [_article(db, made, one_publisher).url for _ in range(3)], + ) + _save( + db, S("two-articles"), [_article(db, made, p1).url, _article(db, made, p2).url] + ) + _story(db, made, S("shown")) + + listed = stories.list_stories(db, limit=20, now=NOW) + assert _mine(listed, {S("one-outlet"), S("two-articles"), S("shown")}) == [ + S("shown") + ] + + +def test_list_hides_closed_stories(db: Session, made: dict[str, list[int]]): + _story(db, made, S("closed-one")) + stories.close_story(db, S("closed-one"), now=NOW) + + assert _mine(stories.list_stories(db, limit=20, now=NOW), {S("closed-one")}) == [] + + +def test_list_orders_by_newest_article_and_shapes_each_story( + db: Session, made: dict[str, list[int]] +): + older = _story(db, made, S("older"), days_ago=5) + _story(db, made, S("newer"), days_ago=1) + + listed = stories.list_stories(db, limit=20, now=NOW) + assert _mine(listed, {S("older"), S("newer")}) == [S("newer"), S("older")] + + shown = next(s for s in listed if s.slug == S("older")) + assert shown.article_count == 3 + assert shown.publisher_count == 2 + assert [a.url for a in shown.articles] == [a.url for a in reversed(older)] + assert shown.first_at == older[0].published_at + assert shown.last_at == older[-1].published_at + + +def test_grew_today_reads_when_an_article_was_added( + db: Session, made: dict[str, list[int]] +): + _story(db, made, S("quiet")) + later = NOW + timedelta(days=2) + + listed = stories.list_stories(db, limit=20, now=later) + assert next(s for s in listed if s.slug == S("quiet")).grew_today is False + + _save(db, S("quiet"), [_article(db, made, _publisher(db, made)).url], now=later) + listed = stories.list_stories(db, limit=20, now=later) + assert next(s for s in listed if s.slug == S("quiet")).grew_today is True + + +def test_the_rail_caps_the_articles_it_sends( + db: Session, made: dict[str, list[int]], monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(stories, "STORY_ARTICLES", 8) + p1, p2 = _publisher(db, made), _publisher(db, made) + urls = [_article(db, made, p1 if i % 2 else p2, i).url for i in range(10)] + _save(db, S("long-one"), urls) + + shown = next( + s + for s in stories.list_stories(db, limit=20, now=NOW) + if s.slug == S("long-one") + ) + assert shown.article_count == 10 + assert [a.url for a in shown.articles] == urls[:8] + + +def test_the_route_serves_the_list( + client: TestClient, db: Session, made: dict[str, list[int]] +): + _story(db, made, S("served")) + + r = client.get(f"{settings.API_V1_STR}/stories/", params={"limit": 20}) + assert r.status_code == 200 + story = next(s for s in r.json() if s["slug"] == S("served")) + assert set(story["articles"][0]) == { + "id", + "title", + "url", + "publisher", + "published_at", + "score", + } + + +# ─── save_story ────────────────────────────────────────────────────────────── + + +def test_save_creates_then_updates_in_place(db: Session, made: dict[str, list[int]]): + p = _publisher(db, made) + first, second = _article(db, made, p), _article(db, made, p) + + saved = _save(db, S("codemode"), [first.url], name="Codemode") + assert saved == { + "slug": S("codemode"), + "created": True, + "added": 1, + "article_count": 1, + } + + saved = _save( + db, + S("codemode"), + [first.url, second.url], + name="Code mode", + blurb="Agents write code that calls tools. It saves tokens.", + ) + assert saved == { + "slug": S("codemode"), + "created": False, + "added": 1, + "article_count": 2, + } + story = db.exec(select(Story).where(Story.slug == S("codemode"))).one() + db.refresh(story) + assert story.name == "Code mode" + assert story.blurb == "Agents write code that calls tools. It saves tokens." + + +@pytest.mark.parametrize( + ("slug", "name", "blurb", "message"), + [ + ("Bad Slug", "Ok", "Fine.", "Bad slug"), + ("trailing-", "Ok", "Fine.", "Bad slug"), + ("ok", "one two three four five", "Fine.", "over 4 words"), + ("ok", " ", "Fine.", "name is empty"), + ("ok", "Ok", "One. Two. Three.", "over 2 sentences"), + ("ok", "Ok", "x" * 281, "over 280"), + ("ok", "Ok", " ", "blurb is empty"), + ], +) +def test_save_refuses_a_bad_label( + db: Session, + made: dict[str, list[int]], + slug: str, + name: str, + blurb: str, + message: str, +): + url = _article(db, made, _publisher(db, made)).url + with pytest.raises(stories.StoryError, match=message): + stories.save_story(db, slug, name, blurb, [url], now=NOW) + + +def test_save_refuses_unknown_urls_and_writes_nothing( + db: Session, made: dict[str, list[int]] +): + known = _article(db, made, _publisher(db, made)).url + with pytest.raises(stories.StoryError, match="Not a published article"): + _save(db, S("nothing-written"), [known, "https://never.example/x"]) + db.rollback() + assert ( + db.exec(select(Story).where(Story.slug == S("nothing-written"))).first() is None + ) + + +def test_save_refuses_a_new_story_with_no_articles(db: Session): + with pytest.raises(stories.StoryError, match="at least one"): + _save(db, S("empty"), []) + + +def test_an_article_sits_in_one_open_story(db: Session, made: dict[str, list[int]]): + p = _publisher(db, made) + shared, other = _article(db, made, p), _article(db, made, p) + _save(db, S("first"), [shared.url]) + + with pytest.raises( + stories.StoryError, match=rf"another open story.*\({S('first')}\)" + ): + _save(db, S("second"), [other.url, shared.url]) + db.rollback() + assert db.exec(select(Story).where(Story.slug == S("second"))).first() is None + + stories.close_story(db, S("first"), now=NOW) + assert _save(db, S("second"), [shared.url])["created"] is True + + +def test_a_closed_story_cannot_be_saved_or_closed_again( + db: Session, made: dict[str, list[int]] +): + url = _article(db, made, _publisher(db, made)).url + _save(db, S("done"), [url]) + stories.close_story(db, S("done"), now=NOW) + + with pytest.raises(stories.StoryError, match="closed"): + _save(db, S("done"), [url]) + with pytest.raises(stories.StoryError, match="already closed"): + stories.close_story(db, S("done"), now=NOW) + with pytest.raises(stories.StoryError, match="No story"): + stories.close_story(db, S("never-was"), now=NOW) + + +# ─── story_candidates ──────────────────────────────────────────────────────── + + +def _near(axis: int, wobble: float = 0.0) -> list[float]: + """A vector on one high axis, far from the seeded random ones.""" + v = np.zeros(DIM, dtype=np.float32) + v[220 + axis] = 1.0 + v[255] = wobble + return v.tolist() + + +def test_candidates_cluster_free_articles_and_name_the_story_they_reach( + db: Session, made: dict[str, list[int]] +): + p1, p2 = _publisher(db, made), _publisher(db, made) + held = _article(db, made, p1, embedding=_near(0)) + _save(db, S("held"), [held.url]) + joins = _article(db, made, p2, embedding=_near(0, 0.1)) + a = _article(db, made, p1, embedding=_near(1)) + b = _article(db, made, p2, embedding=_near(1, 0.2)) + alone = _article(db, made, p1, embedding=_near(2)) + + found = stories.story_candidates(db, days=14, now=NOW) + + mine = [s for s in found["open"] if s["slug"] == S("held")] + assert [s["articles"][0]["url"] for s in mine] == [held.url] + ours = {held.url, joins.url, a.url, b.url, alone.url} + clusters = [ + c for c in found["clusters"] if {x["url"] for x in c["articles"]} & ours + ] + by_urls = {frozenset(x["url"] for x in c["articles"]): c for c in clusters} + assert set(by_urls) == {frozenset({joins.url}), frozenset({a.url, b.url})} + assert by_urls[frozenset({joins.url})]["near"] == [S("held")] + assert by_urls[frozenset({a.url, b.url})]["near"] == [] + assert by_urls[frozenset({a.url, b.url})]["publishers"] == sorted( + [p1.name, p2.name] + ) + + +# ─── Tool gate ─────────────────────────────────────────────────────────────── + + +def test_save_story_tool_reports_what_it_did( + db: Session, made: dict[str, list[int]], monkeypatch: pytest.MonkeyPatch +): + from fastmcp.server.auth import AccessToken + + from app.mcp import tools + + monkeypatch.setattr( + tools, + "get_access_token", + lambda: AccessToken(token="t", client_id="c", scopes=[tools.WRITE_SCOPE]), + ) + url = _article(db, made, _publisher(db, made)).url + slug = S("tool-made") + assert tools.save_story(slug, "Tool made", "It works.", [url]) == ( + f"Created {slug}: added 1, 1 articles in all" + ) + assert tools.close_story(slug) == f"Closed {slug}" diff --git a/backend/app/mcp/server.py b/backend/app/mcp/server.py index d2857776..84de414f 100644 --- a/backend/app/mcp/server.py +++ b/backend/app/mcp/server.py @@ -11,15 +11,18 @@ WRITE_SCOPE, approve, check_link, + close_story, draft_issue, get_content, list_candidates, reject, reject_many, related, + save_story, similar, sql_query, stories, + story_candidates, vocabulary, web_fetch, web_search, @@ -76,7 +79,9 @@ async def verify_token(self, token: str) -> AccessToken | None: "newsletter, with the write token only: `week` ranks the week's " "topics, `related` finds what else covers one, and `draft_issue` " "drafts an issue for a person to send and says where to save it as a " - "blog post." + "blog post. Stories, with the write token only: `story_candidates` " + "lists open stories and unassigned clusters, `save_story` creates or " + "extends one, `close_story` closes one." ), auth=SharedSecret(), tools=[ @@ -95,6 +100,9 @@ async def verify_token(self, token: str) -> AccessToken | None: week, related, draft_issue, + story_candidates, + save_story, + close_story, ], ) diff --git a/backend/app/mcp/tests/test_curation_tools.py b/backend/app/mcp/tests/test_curation_tools.py index c8a76e8a..deae272d 100644 --- a/backend/app/mcp/tests/test_curation_tools.py +++ b/backend/app/mcp/tests/test_curation_tools.py @@ -28,6 +28,9 @@ "week", "related", "draft_issue", + "story_candidates", + "save_story", + "close_story", ) @@ -63,6 +66,9 @@ def _call(name: str): "week": lambda: tools.week(), "related": lambda: tools.related("https://example.com"), "draft_issue": lambda: tools.draft_issue("---\ntitle: t\n---\nb"), + "story_candidates": lambda: tools.story_candidates(), + "save_story": lambda: tools.save_story("s", "n", "b", []), + "close_story": lambda: tools.close_story("s"), }[name]() diff --git a/backend/app/mcp/tools.py b/backend/app/mcp/tools.py index 245e378c..84cdfd8a 100644 --- a/backend/app/mcp/tools.py +++ b/backend/app/mcp/tools.py @@ -1,10 +1,10 @@ -"""What an agent can do against agentique: read the db and the web, curate, and -draft the weekly newsletter. +"""What an agent can do against agentique: read the db and the web, curate, +draft the weekly newsletter, and keep the rail's stories. Two groups, two tokens. `sql_query`, `web_fetch` and `web_search` read and are reachable with `MCP_TOKEN`. The curation tools publish and reject articles, and -the newsletter tools at the bottom draft emails addressed to every subscriber; -both need `MCP_WRITE_TOKEN` — see `WRITE_SCOPE` below. +the newsletter tools draft emails addressed to every subscriber, and the story +tools at the bottom name what the right rail shows; all three need `MCP_WRITE_TOKEN` — see `WRITE_SCOPE` below. """ from __future__ import annotations @@ -18,6 +18,7 @@ from fastmcp.server.dependencies import get_access_token from sqlmodel import Session +from app.catalog import stories as story_domain from app.newsletter import broadcast, digest from app.platform.db import engine from app.platform.settings import settings @@ -440,3 +441,62 @@ def draft_issue(post: str) -> str: return broadcast.draft(post) except broadcast.IssueError as exc: raise ToolError(str(exc)) + + +# ─── Stories ───────────────────────────────────────────────────────────────── +# The right rail's named threads, kept by their own daily routine (ADR 16). +# Behind the write scope: a saved story is on the public site at once. + + +def story_candidates(days: int = 14) -> story_domain.Candidates: + """What the stories routine decides from. + + `open`: every open story, the most recently active first, with `slug`, + `name`, `blurb`, `article_count`, `quiet_days` (since its newest article + was published) and its newest 5 `articles`. `clusters`: the last `days` + of published articles that sit in no open story, grouped by embedding at + a looser cut than coverage, the most publishers first. Each cluster has + `articles` (`url`, `title`, `publisher`, `published_at`, `score`, + `summary`), its `publishers`, and `near`: the open stories one of its + articles is close to. A single article close to a story comes back as a + cluster of one with that story in `near`. + """ + _require_write() + with _curation_session() as session: + return story_domain.story_candidates(session, days) + + +def save_story(slug: str, name: str, blurb: str, urls: list[str]) -> str: + """Create a story, or update an open one: set its `name` and `blurb` and + add `urls` to it (ones already in it are skipped). + + `slug` is lowercase words joined by hyphens and never changes. `name` is + at most 4 words, `blurb` at most 2 sentences and 280 characters. Every URL + must be a published article and in no other open story. Any problem + refuses the whole call, saying which, and nothing is written. A closed + story cannot be saved to. The rail shows a story once it has 3 articles + from 2 publishers. + """ + _require_write() + with _curation_session() as session: + try: + saved = story_domain.save_story(session, slug, name, blurb, urls) + except story_domain.StoryError as exc: + raise ToolError(str(exc)) + verb = "Created" if saved["created"] else "Updated" + return ( + f"{verb} {saved['slug']}: added {saved['added']}, " + f"{saved['article_count']} articles in all" + ) + + +def close_story(slug: str) -> str: + """Close an open story: it leaves the rail, and its articles are free to + join another.""" + _require_write() + with _curation_session() as session: + try: + story_domain.close_story(session, slug) + except story_domain.StoryError as exc: + raise ToolError(str(exc)) + return f"Closed {slug}" diff --git a/backend/app/router.py b/backend/app/router.py index 25e687dd..6d12b130 100644 --- a/backend/app/router.py +++ b/backend/app/router.py @@ -7,6 +7,7 @@ from app.analytics.routes import router as analytics_router from app.audience.routes import likes_router, login_router, users_router from app.catalog.routes import router as catalog_router +from app.catalog.routes import stories_router # Registration order is the order the endpoints appear in the OpenAPI document, # and the generated client follows that document. Reordering churns the client @@ -17,3 +18,4 @@ api_router.include_router(catalog_router) api_router.include_router(likes_router) api_router.include_router(analytics_router) +api_router.include_router(stories_router) diff --git a/backend/scripts/seed_stories.py b/backend/scripts/seed_stories.py new file mode 100644 index 00000000..03e248a6 --- /dev/null +++ b/backend/scripts/seed_stories.py @@ -0,0 +1,109 @@ +"""Seeds two or three stories into a local database, through `save_story`. + +With a JSON file argument, saves each `{slug, name, blurb, urls}` in it: write +one by hand against a restored dump. Without one, makes demo stories from the +newest articles, a few per story across publishers, so the rail has something +to show. Never touches production. +""" + +from __future__ import annotations + +import json +import logging +import sys +from collections import defaultdict +from pathlib import Path +from typing import TypedDict + +from sqlmodel import Session, col, select + +from app.catalog import stories +from app.catalog.models import Article +from app.platform.db import engine +from app.platform.settings import settings + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +DEMO = [ + ( + "codemode", + "Codemode", + "Agents write code that calls their tools instead of calling each tool " + "in turn. It cuts tokens and round trips on long tasks.", + ), + ( + "decision-models", + "Decision models", + "Small models trained to pick the next step, not to write the answer. " + "They make agents cheaper to run at every turn.", + ), + ( + "local-inference", + "Local inference", + "Running open weights on a laptop or one GPU. Quantized releases now " + "come the same week as the full model.", + ), +] +PER_STORY = 4 + + +class StorySpec(TypedDict): + slug: str + name: str + blurb: str + urls: list[str] + + +def demo_stories(session: Session) -> list[StorySpec]: + """Each demo story gets `PER_STORY` of the newest articles, taken round + robin across publishers so it passes the rail's two-publisher bar.""" + newest = session.exec( + select(Article).order_by(col(Article.published_at).desc().nulls_last()) + ).all() + by_publisher: dict[int, list[Article]] = defaultdict(list) + for article in newest: + by_publisher[article.publisher_id].append(article) + queues = list(by_publisher.values()) + picked: list[str] = [] + while len(picked) < PER_STORY * len(DEMO) and any(queues): + for queue in queues: + if queue: + picked.append(queue.pop(0).url) + return [ + { + "slug": slug, + "name": name, + "blurb": blurb, + "urls": picked[i * PER_STORY : (i + 1) * PER_STORY], + } + for i, (slug, name, blurb) in enumerate(DEMO) + ] + + +def main() -> None: + if settings.ENVIRONMENT == "production": + raise SystemExit("Refusing to run seed_stories in production.") + with Session(engine) as session: + if len(sys.argv) > 1: + wanted: list[StorySpec] = json.loads(Path(sys.argv[1]).read_text()) + else: + wanted = demo_stories(session) + for story in wanted: + try: + saved = stories.save_story( + session, + story["slug"], + story["name"], + story["blurb"], + story["urls"], + ) + except stories.StoryError as exc: + session.rollback() + logger.warning("Skipped %s: %s", story["slug"], exc) + continue + logger.info("Saved %s", saved) + + +if __name__ == "__main__": + main() diff --git a/deploy/README.md b/deploy/README.md index 2edab2da..3de1ea72 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -109,6 +109,41 @@ The routine's prompt, to paste there when it changes: Run the `newsletter` skill (`.claude/skills/newsletter/SKILL.md`). ``` +## Stories + +The right rail's stories are kept by a third off-box routine, the same way as +curation and the newsletter: a scheduled task on claude.ai/code against this +repo, with the same write token, invoking `/stories` +(`.claude/skills/stories/SKILL.md`, ADR 16). It runs daily at 06:17 +Europe/Zurich (`CRON_TZ=Europe/Zurich 17 6 * * *`), after that morning's +curation and report. + +- It reads `story_candidates`, extends open stories, starts new ones and + closes the ones quiet for 21 days, through `save_story` and `close_story`. +- It writes nothing else and commits nothing. A saved story is public at once + once it has 3 articles from 2 publishers. +- A night it does not run, the rail stays as it was; the next run catches up + over its 14-day window. + +The routine's prompt, to paste there when it changes: + +```text +Run the agentique stories session. + +Invoke the `stories` skill (`.claude/skills/stories/SKILL.md` in this repo) and follow it end to end: + +1. `story_candidates()` on the agentique MCP server, once. Read every open story and every cluster before saving anything. +2. Add articles that belong to an open story with `save_story`, keeping its slug, name and blurb unless the skill gives a cause to change them. +3. Start a story with `save_story` only for 3+ articles from 2+ publishers on one theme a builder can act on: not a single launch, not one vendor's product line. +4. `close_story` every open story with `quiet_days` of 21 or more. + +Do not edit files or commit anything; this session reads and writes stories, nothing else. + +If a tool comes back with "This tool needs the curation token", the environment's AGENTIQUE_MCP_TOKEN is the read token rather than MCP_WRITE_TOKEN. Stop and say so plainly - do not try to work around it. + +Finish with a short report: stories created, extended and closed, and anything a tool refused. +``` + ## Newsletter environment Read by the backend through `app/platform/settings.py`. diff --git a/docs/adr/0016-stories-named-by-a-routine.md b/docs/adr/0016-stories-named-by-a-routine.md new file mode 100644 index 00000000..8b5a9a09 --- /dev/null +++ b/docs/adr/0016-stories-named-by-a-routine.md @@ -0,0 +1,64 @@ +# 16. A separate routine names the rail's stories + +## Status + +Accepted. + +## Context + +The feed's right rail shows stories: named threads that run over days or +weeks, like "Codemode" or "Decision models", each with a two-sentence blurb +and its articles in order. + +A story is wider than coverage. Coverage (`similar`, `stories`, 0.30 cosine) +is one event told by several outlets, and the embedding maths finds it on its +own. A story is a theme a builder follows across several events: a launch, a +repo built on it, a writer who measured it. Grouping by distance alone either +misses those or chains unrelated articles together, and nothing in the maths +can name the result. That takes judgment, so an agent names and keeps stories. + +The curation agent (ADR 9) already reads every article and calls `stories()` +each night, so the obvious place was a new `approve()` argument naming the +story an article joins. + +## Decision + + daily 06:17: story_candidates -> extend open stories, start new ones, close quiet ones + the rail: GET /api/v1/stories reads what the routine saved + +- **A third routine.** A scheduled task on claude.ai/code, like curation and + the newsletter, runs `.claude/skills/stories/SKILL.md` with the write token, + daily after curation and the report. +- **Two tables.** `story` (slug, name, blurb, last_article_at, closed_at) and + `story_article`. An article sits in at most one open story; `save_story` + enforces it, being the only writer. +- **Three tools.** `story_candidates` shows the open stories and the recent + articles in none, clustered at `STORY_DIST` (0.40) with curation's + `story_groups`; `save_story` creates or extends one and checks the labels; + `close_story` closes one. +- **The rail filters.** A story is public once it has 3 articles from 2 + publishers; below that it is the routine's working state. + +## Why not in `approve` + +- The curation rubric stays untouched. Changing it means re-running its + regression set (CLAUDE.md), and a story field on every approval is a rubric + change. +- The curation session is already long. Naming a story needs the open stories + and the last two weeks in view at once, which is a different read from + judging one candidate. +- A story is decided over days, not per article. Most articles join a story + after it exists, and one session a day can look back over the window. + +## Consequences + +Stories cost one short session a day and two tables. An article approved at +05:00 joins its story at 06:17, so the rail is at most a day behind the feed. + +`STORY_DIST` is a first value. It was not tuned on production data when this +shipped; the routine's reports will show whether clusters come back too broad +or too thin. + +A wrong story stays on the rail until the next run fixes it or a person closes +it with `close_story`. Stories close after 21 quiet days by the skill's rule, +not by a timer. diff --git a/docs/glossary.md b/docs/glossary.md index 4a6e0ed3..011ee000 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -28,6 +28,8 @@ Glossary for words the code uses. If a term below and the code disagree, the cod - **Coverage** - how many distinct publishers carry one story, counted by `similar` and `stories` (`pipeline/curation.py`): rows within 0.30 cosine distance of each other across the feed and the ledger. Reach evidence for the curation agent, like traction. +- **Story** (`Story`, `app/catalog/stories.py`) - a named thread that runs over days or weeks, like "Codemode": a name of 1-3 words, a two-sentence blurb, and its articles (`story_article`). Wider than coverage: coverage is one event told by several outlets, a story is a theme a builder follows across several events, which takes judgment. The stories routine (`.claude/skills/stories/SKILL.md`, ADR 16) names and keeps them through `story_candidates`, `save_story` and `close_story`. An article sits in at most one open story. The right rail (`GET /api/v1/stories`) shows open stories with 3+ articles from 2+ publishers. + - **Issue** - one weekly newsletter email and the blog post with the same text: a 250-450 word explainer of one topic, what it is, how it works and how people are taking it, linking the sources it was written from. The newsletter agent (`.claude/skills/newsletter/SKILL.md`) drafts three a week, one per topic, through `week`, `related` and `draft_issue` (`app/newsletter/`), and a person sends one. Each is a file in `frontend/content/blog/-.md`, the frontmatter giving the title (the subject), the description (the preheader) and the topic; the routine opens one PR with the three, and git stores them. - **Traction** - outside signal that people found a story worth reading, independent of the agent's score: Hacker News points/comments past a minimum, which hold back low-traction submissions from the "everyone can post" source; a repo's stars or a model's downloads, which the curation agent reads with `check_link`; and coverage (above). A first-party URL (`pipeline/first_party.py`) skips the gate - a lab's own announcement counts as news at zero votes. diff --git a/docs/product.md b/docs/product.md index 46623e24..ca41f1e8 100644 --- a/docs/product.md +++ b/docs/product.md @@ -31,6 +31,14 @@ discussions say about each, and writes three short explainers, one per topic: wh thing is, how it works, how people are taking it, with links to the sources. They land as Resend drafts and a person sends one (docs/adr/0012). +**Stories in the right rail.** Beside the feed, on wide screens, a rail lists +up to five stories: named threads that run over days or weeks, like "Codemode", +each with a two-sentence blurb and its articles in order. A Claude Code session +names and keeps them every morning after curation: it adds new articles to open +stories, starts a story when three articles from two publishers share a theme, +and closes one after 21 quiet days (docs/adr/0016). With no stories the rail +does not show. + **A developer API — public, paid tier not live yet.** The article endpoints (list, search, facets, publishers, tags, stats) are open and unauthenticated, with no date window or rate limit; a token is optional and only fills in `liked_by_me`. Liking an diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index edf3071f..6136e14f 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -553,6 +553,109 @@ export const SignInRequestSchema = { title: 'SignInRequest' } as const; +export const StoryArticlePublicSchema = { + properties: { + id: { + type: 'integer', + title: 'Id' + }, + title: { + type: 'string', + title: 'Title' + }, + url: { + type: 'string', + title: 'Url' + }, + publisher: { + type: 'string', + title: 'Publisher' + }, + published_at: { + anyOf: [ + { + type: 'string', + format: 'date-time' + }, + { + type: 'null' + } + ], + title: 'Published At' + }, + score: { + type: 'integer', + title: 'Score' + } + }, + type: 'object', + required: ['id', 'title', 'url', 'publisher', 'score'], + title: 'StoryArticlePublic' +} as const; + +export const StoryPublicSchema = { + properties: { + slug: { + type: 'string', + title: 'Slug' + }, + name: { + type: 'string', + title: 'Name' + }, + blurb: { + type: 'string', + title: 'Blurb' + }, + article_count: { + type: 'integer', + title: 'Article Count' + }, + publisher_count: { + type: 'integer', + title: 'Publisher Count' + }, + first_at: { + anyOf: [ + { + type: 'string', + format: 'date-time' + }, + { + type: 'null' + } + ], + title: 'First At' + }, + last_at: { + anyOf: [ + { + type: 'string', + format: 'date-time' + }, + { + type: 'null' + } + ], + title: 'Last At' + }, + grew_today: { + type: 'boolean', + title: 'Grew Today' + }, + articles: { + items: { + '$ref': '#/components/schemas/StoryArticlePublic' + }, + type: 'array', + title: 'Articles' + } + }, + type: 'object', + required: ['slug', 'name', 'blurb', 'article_count', 'publisher_count', 'grew_today', 'articles'], + title: 'StoryPublic' +} as const; + export const TagFacetSchema = { properties: { slug: { diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index ef2f6a9a..a44fed79 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { AnalyticsCollectEventData, AnalyticsCollectEventResponse, AnalyticsReadReportData, AnalyticsReadReportResponse, ArticlesReadArticlesData, ArticlesReadArticlesResponse, ArticlesSearchArticlesData, ArticlesSearchArticlesResponse, ArticlesArticleFacetsData, ArticlesArticleFacetsResponse, ArticlesArticleOriginsData, ArticlesArticleOriginsResponse, ArticlesSearchPublishersData, ArticlesSearchPublishersResponse, ArticlesSearchTagsData, ArticlesSearchTagsResponse, ArticlesArticleStatsResponse, LikesLikeArticleData, LikesLikeArticleResponse, LikesUnlikeArticleData, LikesUnlikeArticleResponse, LikesReadLikedArticlesResponse, LoginRequestSignInLinkData, LoginRequestSignInLinkResponse, LoginLoginWithLinkData, LoginLoginWithLinkResponse, LoginRefreshTokenResponse, LoginTestTokenResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UtilsHealthCheckResponse } from './types.gen'; +import type { AnalyticsCollectEventData, AnalyticsCollectEventResponse, AnalyticsReadReportData, AnalyticsReadReportResponse, ArticlesReadArticlesData, ArticlesReadArticlesResponse, ArticlesSearchArticlesData, ArticlesSearchArticlesResponse, ArticlesArticleFacetsData, ArticlesArticleFacetsResponse, ArticlesArticleOriginsData, ArticlesArticleOriginsResponse, ArticlesSearchPublishersData, ArticlesSearchPublishersResponse, ArticlesSearchTagsData, ArticlesSearchTagsResponse, ArticlesArticleStatsResponse, LikesLikeArticleData, LikesLikeArticleResponse, LikesUnlikeArticleData, LikesUnlikeArticleResponse, LikesReadLikedArticlesResponse, LoginRequestSignInLinkData, LoginRequestSignInLinkResponse, LoginLoginWithLinkData, LoginLoginWithLinkResponse, LoginRefreshTokenResponse, LoginTestTokenResponse, StoriesReadStoriesData, StoriesReadStoriesResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UtilsHealthCheckResponse } from './types.gen'; export class AnalyticsService { /** @@ -329,6 +329,28 @@ export class LoginService { } } +export class StoriesService { + /** + * Read Stories + * @param data The data for the request. + * @param data.limit + * @returns StoryPublic Successful Response + * @throws ApiError + */ + public static readStories(data: StoriesReadStoriesData = {}): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/stories/', + query: { + limit: data.limit + }, + errors: { + 422: 'Validation Error' + } + }); + } +} + export class UsersService { /** * Read User Me diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 48c073ab..7af2a908 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -134,6 +134,27 @@ export type SignInRequest = { email: string; }; +export type StoryArticlePublic = { + id: number; + title: string; + url: string; + publisher: string; + published_at?: (string | null); + score: number; +}; + +export type StoryPublic = { + slug: string; + name: string; + blurb: string; + article_count: number; + publisher_count: number; + first_at?: (string | null); + last_at?: (string | null); + grew_today: boolean; + articles: Array; +}; + export type TagFacet = { slug: string; name: string; @@ -276,6 +297,12 @@ export type LoginRefreshTokenResponse = (Token); export type LoginTestTokenResponse = (UserPublic); +export type StoriesReadStoriesData = { + limit?: number; +}; + +export type StoriesReadStoriesResponse = (Array); + export type UsersReadUserMeResponse = (UserPublic); export type UsersDeleteUserMeResponse = (Message); diff --git a/frontend/src/components/Rails/RailsProvider.tsx b/frontend/src/components/Rails/RailsProvider.tsx index 2797c65a..fb7bdb28 100644 --- a/frontend/src/components/Rails/RailsProvider.tsx +++ b/frontend/src/components/Rails/RailsProvider.tsx @@ -11,6 +11,7 @@ import { } from "react" import { trackEvent } from "@/lib/analytics" import { LEFT, type RailPanels, RIGHT, type Side } from "./panels" +import { STORIES_PANEL_ID, useHasStories } from "./panels/Stories" /** * wide: rails sit inline in the gutters beside the feed, always open. @@ -97,13 +98,28 @@ type RailsContextType = { const RailsContext = createContext(null) +const REGISTERED: RailPanels = { left: LEFT, right: RIGHT } + export function RailsProvider({ children, - panels = { left: LEFT, right: RIGHT }, + panels: registered = REGISTERED, }: { children: ReactNode panels?: RailPanels }) { + // A panel with nothing to show is left out, so a side whose panels are all + // empty renders no rail, strip or button. Stories is the one that can be. + const hasStories = useHasStories() + const panels = useMemo( + () => + hasStories + ? registered + : { + ...registered, + right: registered.right.filter((p) => p.id !== STORIES_PANEL_ID), + }, + [registered, hasStories], + ) const mode = useSyncExternalStore( subscribeMode, readMode, diff --git a/frontend/src/components/Rails/panels.tsx b/frontend/src/components/Rails/panels.tsx index f9e4a8e3..a8425d20 100644 --- a/frontend/src/components/Rails/panels.tsx +++ b/frontend/src/components/Rails/panels.tsx @@ -1,4 +1,5 @@ import type { ComponentType } from "react" +import { STORIES_PANEL_ID, Stories } from "./panels/Stories" import { WhoWrote } from "./panels/WhoWrote" export type Side = "left" | "right" @@ -19,7 +20,14 @@ export type RailPanels = Record export const LEFT: RailPanelDef[] = [ { id: "who-wrote", title: "Who wrote the wire", Component: WhoWrote }, ] -export const RIGHT: RailPanelDef[] = [] +export const RIGHT: RailPanelDef[] = [ + { + id: STORIES_PANEL_ID, + title: "Stories", + meta: "named by the agent", + Component: Stories, + }, +] export const RAIL_TITLES: Record = { left: "People and topics", diff --git a/frontend/src/components/Rails/panels/Stories.tsx b/frontend/src/components/Rails/panels/Stories.tsx new file mode 100644 index 00000000..1b4b21fb --- /dev/null +++ b/frontend/src/components/Rails/panels/Stories.tsx @@ -0,0 +1,183 @@ +import { useQuery } from "@tanstack/react-query" +import { useId, useState } from "react" +import { StoriesService, type StoryPublic } from "@/client" +import { articleClickHandlers, trackEvent } from "@/lib/analytics" +import { cn } from "@/lib/utils" +import { MiniRail, ScoreBars } from "../ScoreMarks" + +export const STORIES_PANEL_ID = "stories" + +// The timeline shows this many; "Show all" opens the rest in place. +const TIMELINE = 6 +// Enough bars to read a story's shape without crowding its name. +const BARS = 12 + +export function useStories() { + return useQuery({ + queryKey: ["stories"], + queryFn: () => StoriesService.readStories({ limit: 5 }), + staleTime: 5 * 60 * 1000, + retry: false, + }) +} + +/** Whether there is anything to show. An empty list, a failed fetch, or one + * still loading hides the panel, and the rail with it. */ +export function useHasStories() { + const { data } = useStories() + return !!data && data.length > 0 +} + +function day(iso: string | null | undefined): string { + if (!iso) return "" + return new Date(iso).toLocaleDateString("en-US", { + month: "short", + day: "numeric", + }) +} + +function span(story: StoryPublic): string { + const first = day(story.first_at) + const last = day(story.last_at) + return first === last ? last : `${first} – ${last}` +} + +export function Stories() { + const { data } = useStories() + // undefined until the reader picks: the first story starts open. + const [open, setOpen] = useState() + if (!data || data.length === 0) return null + + const current = open === undefined ? data[0].slug : open + + function toggle(slug: string) { + const next = current === slug ? null : slug + setOpen(next) + if (next) trackEvent("story_open", { slug }) + } + + return ( +
    + {data.map((story) => ( + toggle(story.slug)} + /> + ))} +
+ ) +} + +function StoryRow({ + story, + open, + onToggle, +}: { + story: StoryPublic + open: boolean + onToggle: () => void +}) { + const bodyId = useId() + const bars = story.articles + .slice(0, BARS) + .map((a) => a.score) + .reverse() + + return ( +
  • + +
    +
    + +
    +
    +
    • + ) +} + +function StoryBody({ story }: { story: StoryPublic }) { + const [all, setAll] = useState(false) + const shown = all ? story.articles : story.articles.slice(0, TIMELINE) + const rest = story.articles.length - TIMELINE + + return ( +
    +

    + {story.blurb} +

    +
      + {shown.map((article) => ( +
    1. + + {day(article.published_at)} + + + + + {article.publisher} + + + {article.title} + + +
      2. + ))} +
    + {rest > 0 && !all && ( + + )} +
    + ) +} diff --git a/frontend/tests/stories.spec.ts b/frontend/tests/stories.spec.ts new file mode 100644 index 00000000..d964ba7e --- /dev/null +++ b/frontend/tests/stories.spec.ts @@ -0,0 +1,121 @@ +import { expect, type Page, test } from "@playwright/test" + +// Wide enough that the rails sit inline beside the feed, always open. +test.use({ viewport: { width: 1440, height: 900 } }) + +function article(id: number, daysAgo: number, score: number) { + return { + id, + title: `Article ${id}`, + url: `https://example.com/${id}`, + publisher: id % 2 ? "Cloudflare" : "Simon Willison", + published_at: new Date(Date.now() - daysAgo * 86_400_000).toISOString(), + score, + } +} + +function story(slug: string, name: string, count: number, grewToday = false) { + const articles = Array.from({ length: count }, (_, i) => + article(slug.length * 100 + i, i, 40 + ((i * 17) % 60)), + ) + return { + slug, + name, + blurb: `What ${name} is. Why it matters now.`, + article_count: count, + publisher_count: 2, + first_at: articles[articles.length - 1].published_at, + last_at: articles[0].published_at, + grew_today: grewToday, + articles, + } +} + +const STORIES = [ + story("codemode", "Codemode", 9, true), + story("decision-models", "Decision models", 4), + story("local-inference", "Local inference", 3), +] + +async function mockStories(page: Page, body: unknown, status = 200) { + await page.route("**/api/v1/stories/**", (route) => + route.fulfill({ status, json: body }), + ) +} + +/** The body that directly follows a story's row button. */ +async function bodyAfter(page: Page, name: string) { + return page + .getByRole("button", { name }) + .evaluate((row) => row.nextElementSibling?.getAttribute("data-testid")) +} + +test("the first story is open, its body right under its row", async ({ + page, +}) => { + await mockStories(page, STORIES) + await page.goto("/") + + const rows = page.getByTestId("story-row") + await expect(rows).toHaveCount(3) + await expect(rows.nth(0)).toHaveAttribute("aria-expanded", "true") + await expect(rows.nth(1)).toHaveAttribute("aria-expanded", "false") + + expect(await bodyAfter(page, "Codemode")).toBe("story-body") + const open = page.locator('[data-testid="story-body"][data-open="true"]') + await expect(open).toHaveCount(1) + await expect(open).toContainText("What Codemode is.") + await expect(open.getByRole("link")).toHaveCount(6) +}) + +test("clicking another row moves the open body to it", async ({ page }) => { + await mockStories(page, STORIES) + await page.goto("/") + + await page.getByRole("button", { name: "Decision models" }).click() + + const rows = page.getByTestId("story-row") + await expect(rows.nth(0)).toHaveAttribute("aria-expanded", "false") + await expect(rows.nth(1)).toHaveAttribute("aria-expanded", "true") + const open = page.locator('[data-testid="story-body"][data-open="true"]') + await expect(open).toHaveCount(1) + await expect(open).toContainText("What Decision models is.") + // The open body sits between its row and the next story, not below the list. + const openBodyFollowsRow = await rows + .nth(1) + .evaluate( + (row) => row.nextElementSibling?.getAttribute("data-open") === "true", + ) + expect(openBodyFollowsRow).toBe(true) + const third = await rows.nth(2).boundingBox() + const body = await open.boundingBox() + expect(body && third && body.y < third.y).toBe(true) +}) + +test("Show all expands the timeline in place", async ({ page }) => { + await mockStories(page, STORIES) + await page.goto("/") + + const open = page.locator('[data-testid="story-body"][data-open="true"]') + await expect(open.getByRole("link")).toHaveCount(6) + await open.getByRole("button", { name: "Show all 9" }).click() + await expect(open.getByRole("link")).toHaveCount(9) + await expect(open.getByRole("button", { name: /Show all/ })).toHaveCount(0) + expect(await bodyAfter(page, "Codemode")).toBe("story-body") +}) + +test("no stories hides the right rail", async ({ page }) => { + await mockStories(page, []) + await page.goto("/") + + await expect(page.getByTestId("article-row").first()).toBeVisible() + await expect(page.locator('[data-rail="right"]')).toHaveCount(0) +}) + +test("a failed fetch hides the right rail", async ({ page }) => { + await mockStories(page, { detail: "boom" }, 500) + await page.goto("/") + + await expect(page.getByTestId("article-row").first()).toBeVisible() + await expect(page.locator('[data-rail="right"]')).toHaveCount(0) +})