diff --git a/backend/app/catalog/facets.py b/backend/app/catalog/facets.py index 6ade5662..90889978 100644 --- a/backend/app/catalog/facets.py +++ b/backend/app/catalog/facets.py @@ -1,16 +1,25 @@ -"""Facet counts for the feed's filter sidebar: how many articles each publisher and tag has.""" +"""Facet counts for the feed's filters and rails: how many articles each publisher, tag and origin has.""" from __future__ import annotations -from sqlalchemy import func +from datetime import datetime + +from sqlalchemy import ColumnElement, func from sqlmodel import Session, col, select +from app.catalog.articles import origin_condition from app.catalog.models import ( Article, ArticleFacets, + ArticleKind, ArticleTag, + FoundViaCount, + Origin, + OriginCount, + OriginCounts, Publisher, PublisherFacet, + PublisherKind, Tag, TagFacet, ) @@ -58,3 +67,37 @@ def all_facets(session: Session, limit: int = 8) -> ArticleFacets: publishers=publisher_facets(session, None, limit), tags=tag_facets(session, None, limit), ) + + +def origin_counts(session: Session, since: datetime) -> OriginCounts: + """How the articles published since `since` split by origin and by how + they were found. Each origin is counted with the From filter's own + condition, so a row's number is what that filter shows for the window.""" + in_window = col(Article.published_at) >= since + + def count(*conditions: ColumnElement[bool]) -> int: + return session.exec( + select(func.count()).select_from(Article).where(in_window, *conditions) + ).one() + + unlabelled = count( + col(Article.kind) == ArticleKind.post, + col(Article.publisher_id).in_( + select(Publisher.id).where(Publisher.kind == PublisherKind.unknown) + ), + ) + found_via_rows = session.exec( + select(Article.found_via, func.count()) + .where(in_window) + .group_by(col(Article.found_via)) + .order_by(func.count().desc()) + ).all() + return OriginCounts( + since=since, + total=count(), + origins=[ + OriginCount(origin=o, count=count(origin_condition(o))) for o in Origin + ], + unlabelled=unlabelled, + found_via=[FoundViaCount(name=n, count=c) for n, c in found_via_rows], + ) diff --git a/backend/app/catalog/models.py b/backend/app/catalog/models.py index 191c5968..62b011f1 100644 --- a/backend/app/catalog/models.py +++ b/backend/app/catalog/models.py @@ -279,3 +279,27 @@ class TagFacet(SQLModel): class ArticleFacets(SQLModel): publishers: list[PublisherFacet] tags: list[TagFacet] + + +# ─── Origin counts (the left rail's "Who wrote this week") ───────────────── + + +class OriginCount(SQLModel): + origin: Origin + count: int + + +class FoundViaCount(SQLModel): + # Null: the publisher's own feed. + name: str | None + count: int + + +class OriginCounts(SQLModel): + since: datetime + total: int + origins: list[OriginCount] + # Posts by a publisher still `unknown`: under no origin until curation + # names its kind. + unlabelled: int + found_via: list[FoundViaCount] diff --git a/backend/app/catalog/routes.py b/backend/app/catalog/routes.py index 5fc3b02d..acd427e1 100644 --- a/backend/app/catalog/routes.py +++ b/backend/app/catalog/routes.py @@ -12,6 +12,7 @@ ArticleFacets, ArticlesPublic, Origin, + OriginCounts, PublisherFacet, TagFacet, ) @@ -81,6 +82,15 @@ def article_facets( return facets.all_facets(session, limit) +@router.get("/origins", response_model=OriginCounts) +def article_origins(session: SessionDep, since: str) -> OriginCounts: + try: + since_dt = datetime.fromisoformat(since) + except ValueError: + raise HTTPException(status_code=422, detail="Invalid 'since' datetime") + return facets.origin_counts(session, since_dt) + + @router.get("/publishers", response_model=list[PublisherFacet]) def search_publishers( session: SessionDep, diff --git a/backend/app/catalog/tests/test_routes.py b/backend/app/catalog/tests/test_routes.py index 550faeac..4af6d0ae 100644 --- a/backend/app/catalog/tests/test_routes.py +++ b/backend/app/catalog/tests/test_routes.py @@ -4,6 +4,7 @@ import uuid from collections.abc import Generator +from dataclasses import dataclass from datetime import UTC, datetime, timedelta import pytest @@ -15,7 +16,7 @@ from app.audience import service from app.audience.tests.factories import authentication_token_from_email from app.catalog import semantic_search -from app.catalog.models import Article, ArticleKind, PublisherKind +from app.catalog.models import Article, ArticleKind, Origin, PublisherKind from app.catalog.tests.factories import ( create_random_article, create_random_publisher, @@ -64,6 +65,7 @@ def auth_client( "/publishers", "/tags", "/stats", + "/origins?since=2026-01-01", ] @@ -529,3 +531,89 @@ def test_search_articles_has_like_count_and_liked_by_me( for article in r.json()["data"]: assert "like_count" in article assert "liked_by_me" in article + + +# --- origin counts ----------------------------------------------------------- + + +@dataclass +class _Origins: + total: int + unlabelled: int + by_origin: dict[str, int] + by_found_via: dict[str | None, int] + + +def _origins(client: TestClient, since: str) -> _Origins: + r = client.get(f"{ARTICLES_URL}/origins", params={"since": since}) + assert r.status_code == 200 + body = r.json() + return _Origins( + total=body["total"], + unlabelled=body["unlabelled"], + by_origin={o["origin"]: o["count"] for o in body["origins"]}, + by_found_via={f["name"]: f["count"] for f in body["found_via"]}, + ) + + +def _grew[K](after: dict[K, int], before: dict[K, int]) -> dict[K, int]: + """The keys whose count changed, and by how much.""" + return { + k: after.get(k, 0) - before.get(k, 0) + for k in after.keys() | before.keys() + if after.get(k, 0) != before.get(k, 0) + } + + +def test_origin_counts_lists_every_origin(client: TestClient) -> None: + assert set(_origins(client, RECENT_SINCE).by_origin) == {o.value for o in Origin} + + +def test_origin_counts_follow_the_from_filter(client: TestClient, db: Session) -> None: + """A company's repo is a repo, a community post is media, and an unknown + publisher's post falls under no origin.""" + now = datetime.now(UTC) + company = create_random_publisher(db, kind=PublisherKind.company) + community = create_random_publisher(db, kind=PublisherKind.community) + unknown = create_random_publisher(db, kind=PublisherKind.unknown) + + before = _origins(client, RECENT_SINCE) + create_random_article( + db, publisher_id=company.id, kind=ArticleKind.repo, published_at=now + ) + create_random_article(db, publisher_id=community.id, published_at=now) + create_random_article(db, publisher_id=unknown.id, published_at=now) + after = _origins(client, RECENT_SINCE) + + assert _grew(after.by_origin, before.by_origin) == {"repo": 1, "media": 1} + assert after.unlabelled == before.unlabelled + 1 + assert after.total == before.total + 3 + + +def test_origin_counts_are_bounded_by_since(client: TestClient, db: Session) -> None: + lab = create_random_publisher(db, kind=PublisherKind.lab) + before = _origins(client, RECENT_SINCE) + create_random_article( + db, publisher_id=lab.id, published_at=datetime.now(UTC) - timedelta(days=10) + ) + assert _origins(client, RECENT_SINCE) == before + + +def test_origin_counts_group_found_via(client: TestClient, db: Session) -> None: + now = datetime.now(UTC) + before = _origins(client, RECENT_SINCE) + create_random_article(db, published_at=now) + create_random_article(db, published_at=now, found_via="Hacker News") + create_random_article(db, published_at=now, found_via="Hacker News") + create_random_article(db, published_at=now, found_via="AINews") + after = _origins(client, RECENT_SINCE) + assert _grew(after.by_found_via, before.by_found_via) == { + None: 1, + "Hacker News": 2, + "AINews": 1, + } + + +def test_origin_counts_malformed_since_returns_422(client: TestClient) -> None: + r = client.get(f"{ARTICLES_URL}/origins", params={"since": "not-a-date"}) + assert r.status_code == 422 diff --git a/docs/product.md b/docs/product.md index 866fa662..46623e24 100644 --- a/docs/product.md +++ b/docs/product.md @@ -16,6 +16,7 @@ Each entry shows its publisher, and its kind when it is a repo, paper or model. Companies, Writers, Media, Repos, Papers, Models), by topic (a hand-curated set of tags and kinds, e.g. "Open weights" or "Harnesses"), and by time window (last 3 days / week / month). +A left rail beside the feed shows how that window splits by origin and how its articles were found (publisher feeds, Hacker News, newsletters); a row sets the From filter. **Semantic search.** Typing a query searches by meaning, not keyword — it embeds your query and finds the nearest articles in vector space, so "how do I run a model locally" diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 5761385a..edf3071f 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -254,6 +254,29 @@ export const ArticlesPublicSchema = { title: 'ArticlesPublic' } as const; +export const FoundViaCountSchema = { + properties: { + name: { + anyOf: [ + { + type: 'string' + }, + { + type: 'null' + } + ], + title: 'Name' + }, + count: { + type: 'integer', + title: 'Count' + } + }, + type: 'object', + required: ['name', 'count'], + title: 'FoundViaCount' +} as const; + export const HTTPValidationErrorSchema = { properties: { detail: { @@ -302,6 +325,56 @@ repo, a paper, a model), otherwise who published it. Not stored; each article falls under exactly one, see \`catalog.articles.origin_condition\`.` } as const; +export const OriginCountSchema = { + properties: { + origin: { + '$ref': '#/components/schemas/Origin' + }, + count: { + type: 'integer', + title: 'Count' + } + }, + type: 'object', + required: ['origin', 'count'], + title: 'OriginCount' +} as const; + +export const OriginCountsSchema = { + properties: { + since: { + type: 'string', + format: 'date-time', + title: 'Since' + }, + total: { + type: 'integer', + title: 'Total' + }, + origins: { + items: { + '$ref': '#/components/schemas/OriginCount' + }, + type: 'array', + title: 'Origins' + }, + unlabelled: { + type: 'integer', + title: 'Unlabelled' + }, + found_via: { + items: { + '$ref': '#/components/schemas/FoundViaCount' + }, + type: 'array', + title: 'Found Via' + } + }, + type: 'object', + required: ['since', 'total', 'origins', 'unlabelled', 'found_via'], + title: 'OriginCounts' +} as const; + export const PublisherFacetSchema = { properties: { slug: { diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index d5293af3..ef2f6a9a 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { AnalyticsCollectEventData, AnalyticsCollectEventResponse, AnalyticsReadReportData, AnalyticsReadReportResponse, ArticlesReadArticlesData, ArticlesReadArticlesResponse, ArticlesSearchArticlesData, ArticlesSearchArticlesResponse, ArticlesArticleFacetsData, ArticlesArticleFacetsResponse, ArticlesSearchPublishersData, ArticlesSearchPublishersResponse, ArticlesSearchTagsData, ArticlesSearchTagsResponse, ArticlesArticleStatsResponse, LikesLikeArticleData, LikesLikeArticleResponse, LikesUnlikeArticleData, LikesUnlikeArticleResponse, LikesReadLikedArticlesResponse, LoginRequestSignInLinkData, LoginRequestSignInLinkResponse, LoginLoginWithLinkData, LoginLoginWithLinkResponse, LoginRefreshTokenResponse, LoginTestTokenResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UtilsHealthCheckResponse } from './types.gen'; +import type { AnalyticsCollectEventData, AnalyticsCollectEventResponse, AnalyticsReadReportData, AnalyticsReadReportResponse, ArticlesReadArticlesData, ArticlesReadArticlesResponse, ArticlesSearchArticlesData, ArticlesSearchArticlesResponse, ArticlesArticleFacetsData, ArticlesArticleFacetsResponse, ArticlesArticleOriginsData, ArticlesArticleOriginsResponse, ArticlesSearchPublishersData, ArticlesSearchPublishersResponse, ArticlesSearchTagsData, ArticlesSearchTagsResponse, ArticlesArticleStatsResponse, LikesLikeArticleData, LikesLikeArticleResponse, LikesUnlikeArticleData, LikesUnlikeArticleResponse, LikesReadLikedArticlesResponse, LoginRequestSignInLinkData, LoginRequestSignInLinkResponse, LoginLoginWithLinkData, LoginLoginWithLinkResponse, LoginRefreshTokenResponse, LoginTestTokenResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UtilsHealthCheckResponse } from './types.gen'; export class AnalyticsService { /** @@ -130,6 +130,26 @@ export class ArticlesService { }); } + /** + * Article Origins + * @param data The data for the request. + * @param data.since + * @returns OriginCounts Successful Response + * @throws ApiError + */ + public static articleOrigins(data: ArticlesArticleOriginsData): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/articles/origins', + query: { + since: data.since + }, + errors: { + 422: 'Validation Error' + } + }); + } + /** * Search Publishers * @param data The data for the request. diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 83e82c0c..48c073ab 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -49,6 +49,11 @@ export type ArticlesPublic = { count: number; }; +export type FoundViaCount = { + name: (string | null); + count: number; +}; + export type HTTPValidationError = { detail?: Array; }; @@ -68,6 +73,19 @@ export type Message = { */ export type Origin = 'lab' | 'company' | 'individual' | 'media' | 'repo' | 'paper' | 'model'; +export type OriginCount = { + origin: Origin; + count: number; +}; + +export type OriginCounts = { + since: string; + total: number; + origins: Array; + unlabelled: number; + found_via: Array; +}; + export type PublisherFacet = { slug: string; name: string; @@ -200,6 +218,12 @@ export type ArticlesArticleFacetsData = { export type ArticlesArticleFacetsResponse = (ArticleFacets); +export type ArticlesArticleOriginsData = { + since: string; +}; + +export type ArticlesArticleOriginsResponse = (OriginCounts); + export type ArticlesSearchPublishersData = { limit?: number; q?: (string | null); diff --git a/frontend/src/components/Articles/ArticlesList.tsx b/frontend/src/components/Articles/ArticlesList.tsx index acb45706..0b91243a 100644 --- a/frontend/src/components/Articles/ArticlesList.tsx +++ b/frontend/src/components/Articles/ArticlesList.tsx @@ -7,7 +7,12 @@ import { FilterOptions } from "./FilterOption" import { SCORE_STANDOUT } from "./ScoreRail" import { findTopic } from "./topics" -const PUBLISHED_DAYS: Record = { "3d": 3, "1w": 7, "1m": 30 } +/** The Published filter's windows, in days. The left rail's panels follow it. */ +export const PUBLISHED_DAYS: Record = { + "3d": 3, + "1w": 7, + "1m": 30, +} const DATE_OPTIONS = [ { value: "3d", label: "3 days" }, @@ -21,7 +26,7 @@ const SORT_OPTIONS = [ { value: "likes-desc", label: "Popular" }, ] -function cutoffIso(days: number): string { +export function cutoffIso(days: number): string { const d = new Date() d.setDate(d.getDate() - days) return d.toISOString() diff --git a/frontend/src/components/Articles/FeedFilters.tsx b/frontend/src/components/Articles/FeedFilters.tsx index c312844a..f814f1e6 100644 --- a/frontend/src/components/Articles/FeedFilters.tsx +++ b/frontend/src/components/Articles/FeedFilters.tsx @@ -7,7 +7,7 @@ import { TOPICS } from "./topics" // Who published it, or what it is when that is settled. Each article falls // under exactly one: a repo is a repo whoever published it. -const ORIGIN_OPTIONS = [ +export const ORIGIN_OPTIONS = [ { value: "", label: "All" }, { value: "lab", label: "Labs" }, { value: "company", label: "Companies" }, diff --git a/frontend/src/components/Rails/panels.tsx b/frontend/src/components/Rails/panels.tsx index 2317b511..f9e4a8e3 100644 --- a/frontend/src/components/Rails/panels.tsx +++ b/frontend/src/components/Rails/panels.tsx @@ -1,4 +1,5 @@ import type { ComponentType } from "react" +import { WhoWrote } from "./panels/WhoWrote" export type Side = "left" | "right" @@ -15,7 +16,9 @@ export type RailPanels = Record // Planned: left = "Who wrote this week", "Topic map"; right = "Stories". // A side with no panels renders nothing at all: no rail, strip or button. -export const LEFT: RailPanelDef[] = [] +export const LEFT: RailPanelDef[] = [ + { id: "who-wrote", title: "Who wrote the wire", Component: WhoWrote }, +] export const RIGHT: RailPanelDef[] = [] export const RAIL_TITLES: Record = { diff --git a/frontend/src/components/Rails/panels/WhoWrote.tsx b/frontend/src/components/Rails/panels/WhoWrote.tsx new file mode 100644 index 00000000..7ec0ed50 --- /dev/null +++ b/frontend/src/components/Rails/panels/WhoWrote.tsx @@ -0,0 +1,151 @@ +import { useQuery } from "@tanstack/react-query" +import { ArticlesService, type OriginCounts } from "@/client" +import { cutoffIso, PUBLISHED_DAYS } from "@/components/Articles/ArticlesList" +import { ORIGIN_OPTIONS } from "@/components/Articles/FeedFilters" +import { useFilters } from "@/context/filters" +import { cn } from "@/lib/utils" + +const HACKER_NEWS = "Hacker News" + +const LABELS = Object.fromEntries(ORIGIN_OPTIONS.map((o) => [o.value, o.label])) + +function shortDate(d: Date): string { + return d.toLocaleDateString("en-US", { month: "short", day: "numeric" }) +} + +/** How the wire's current window splits by origin, and how it was found. + * Follows the feed's Published filter; a row sets the From filter. */ +export function WhoWrote() { + const { filters, setFilter } = useFilters() + const { dateRange, origin, search } = filters + const days = PUBLISHED_DAYS[dateRange] ?? 7 + + const { data, isError } = useQuery({ + queryKey: ["articles", "origins", dateRange], + queryFn: () => ArticlesService.articleOrigins({ since: cutoffIso(days) }), + staleTime: 5 * 60 * 1000, + }) + + const start = new Date() + start.setDate(start.getDate() - days) + + return ( + // A search is all-time, so the window doesn't apply to it; greyed out + // like the filters. +
+

+ {shortDate(start)} – {shortDate(new Date())} +

+ {isError ? ( +

Failed to load.

+ ) : data ? ( + setFilter("origin", value === origin ? "" : value)} + /> + ) : null} +
+ ) +} + +function Counts({ + data, + active, + onPick, +}: { + data: OriginCounts + active: string + onPick: (origin: string) => void +}) { + // Busiest first; a tie keeps the From filter's order. + const rows = [...data.origins].sort((a, b) => b.count - a.count) + const max = Math.max(1, data.unlabelled, ...rows.map((r) => r.count)) + + let own = 0 + let hn = 0 + let newsletters = 0 + for (const f of data.found_via) { + if (f.name === null || f.name === undefined) own += f.count + else if (f.name === HACKER_NEWS) hn += f.count + else newsletters += f.count + } + + return ( + <> +
    + {rows.map((r) => ( +
  • + +
    • + ))} + {data.unlabelled > 0 && ( +
  • + +
    • + )} +
+

+ Of {data.total}: {own} from publishers' own feeds, {hn} found on Hacker + News, {newsletters} via newsletters. +

+ + ) +} + +function Row({ + label, + count, + max, + active, +}: { + label: string + count: number + max: number + active: boolean +}) { + return ( + + + {label} + + + + + {count} + + ) +} diff --git a/frontend/tests/rails.spec.ts b/frontend/tests/rails.spec.ts new file mode 100644 index 00000000..445261ec --- /dev/null +++ b/frontend/tests/rails.spec.ts @@ -0,0 +1,112 @@ +import { expect, type Page, test } from "@playwright/test" + +const RAIL = "People and topics" + +function leftRail(page: Page) { + return page.getByRole("complementary", { name: RAIL }) +} + +function whoWrote(page: Page) { + return page.locator('[data-panel="who-wrote"]') +} + +test.describe("wide screen", () => { + test.use({ viewport: { width: 1440, height: 900 } }) + + test("the left rail sits inline and a folded panel stays folded after a reload", async ({ + page, + }) => { + await page.goto("/") + await expect(leftRail(page)).toBeVisible() + + // Wide rails are always open: the key that toggles them on smaller + // screens does nothing here. + await page.keyboard.press("[") + await expect(leftRail(page)).toBeVisible() + + const header = whoWrote(page).getByRole("button", { + name: "Who wrote the wire", + }) + await header.click() + await expect(header).toHaveAttribute("aria-expanded", "false") + await page.reload() + await expect(header).toHaveAttribute("aria-expanded", "false") + await header.click() + await expect(header).toHaveAttribute("aria-expanded", "true") + await page.reload() + await expect(header).toHaveAttribute("aria-expanded", "true") + }) + + test("clicking Writers sets the From filter and the feed shows only writers", async ({ + page, + }) => { + await page.goto("/") + const writers = whoWrote(page).getByRole("button", { name: /^Writers/ }) + await expect(writers).toBeVisible() + + const feed = page.waitForResponse( + (r) => + r.url().includes("/api/v1/articles/?") && + r.url().includes("origin=individual"), + ) + await writers.click() + const { data } = await (await feed).json() + + await expect(writers).toHaveAttribute("aria-pressed", "true") + await expect( + page + .getByRole("group", { name: "From" }) + .getByRole("button", { name: "Writers" }), + ).toHaveAttribute("aria-pressed", "true") + + expect(data.length).toBeGreaterThan(0) + for (const a of data) { + expect(a.kind).toBe("post") + expect(a.publisher.kind).toBe("individual") + } + await expect(page.getByTestId("article-row")).toHaveCount(data.length) + + // Clicking the active row clears the filter. + await writers.click() + await expect(writers).toHaveAttribute("aria-pressed", "false") + await expect( + page.getByRole("group", { name: "From" }).getByRole("button", { + name: "All", + }), + ).toHaveAttribute("aria-pressed", "true") + }) +}) + +test.describe("mid screen", () => { + test.use({ viewport: { width: 1000, height: 800 } }) + + test("the edge strip opens an overlay and Esc closes it", async ({ page }) => { + await page.goto("/") + await expect(leftRail(page)).toHaveCount(0) + + await page + .getByRole("navigation", { name: `${RAIL} rail, collapsed` }) + .getByRole("button", { name: "Who wrote the wire" }) + .click() + await expect(leftRail(page)).toBeVisible() + await expect(whoWrote(page).getByText(/^Of \d+:/)).toBeVisible() + + await page.keyboard.press("Escape") + await expect(leftRail(page)).toHaveCount(0) + }) +}) + +test.describe("phone", () => { + test.use({ viewport: { width: 390, height: 844 } }) + + test("the header button opens the rail as a sheet", async ({ page }) => { + await page.goto("/") + await expect(leftRail(page)).toHaveCount(0) + + await page + .getByRole("button", { name: "Toggle the people and topics rail" }) + .click() + await expect(page.getByRole("dialog", { name: RAIL })).toBeVisible() + await expect(whoWrote(page)).toBeVisible() + }) +})