Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 45 additions & 2 deletions backend/app/catalog/facets.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,25 @@
"""Facet counts for the feed's filter sidebar: how many articles each publisher and tag has."""
"""Facet counts for the feed's filters and rails: how many articles each publisher, tag and origin has."""

from __future__ import annotations

from sqlalchemy import func
from datetime import datetime

from sqlalchemy import ColumnElement, func
from sqlmodel import Session, col, select

from app.catalog.articles import origin_condition
from app.catalog.models import (
Article,
ArticleFacets,
ArticleKind,
ArticleTag,
FoundViaCount,
Origin,
OriginCount,
OriginCounts,
Publisher,
PublisherFacet,
PublisherKind,
Tag,
TagFacet,
)
Expand Down Expand Up @@ -58,3 +67,37 @@ def all_facets(session: Session, limit: int = 8) -> ArticleFacets:
publishers=publisher_facets(session, None, limit),
tags=tag_facets(session, None, limit),
)


def origin_counts(session: Session, since: datetime) -> OriginCounts:
"""How the articles published since `since` split by origin and by how
they were found. Each origin is counted with the From filter's own
condition, so a row's number is what that filter shows for the window."""
in_window = col(Article.published_at) >= since

def count(*conditions: ColumnElement[bool]) -> int:
return session.exec(
select(func.count()).select_from(Article).where(in_window, *conditions)
).one()

unlabelled = count(
col(Article.kind) == ArticleKind.post,
col(Article.publisher_id).in_(
select(Publisher.id).where(Publisher.kind == PublisherKind.unknown)
),
)
found_via_rows = session.exec(
select(Article.found_via, func.count())
.where(in_window)
.group_by(col(Article.found_via))
.order_by(func.count().desc())
).all()
return OriginCounts(
since=since,
total=count(),
origins=[
OriginCount(origin=o, count=count(origin_condition(o))) for o in Origin
],
unlabelled=unlabelled,
found_via=[FoundViaCount(name=n, count=c) for n, c in found_via_rows],
)
24 changes: 24 additions & 0 deletions backend/app/catalog/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -279,3 +279,27 @@ class TagFacet(SQLModel):
class ArticleFacets(SQLModel):
publishers: list[PublisherFacet]
tags: list[TagFacet]


# ─── Origin counts (the left rail's "Who wrote this week") ─────────────────


class OriginCount(SQLModel):
origin: Origin
count: int


class FoundViaCount(SQLModel):
# Null: the publisher's own feed.
name: str | None
count: int


class OriginCounts(SQLModel):
since: datetime
total: int
origins: list[OriginCount]
# Posts by a publisher still `unknown`: under no origin until curation
# names its kind.
unlabelled: int
found_via: list[FoundViaCount]
10 changes: 10 additions & 0 deletions backend/app/catalog/routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
ArticleFacets,
ArticlesPublic,
Origin,
OriginCounts,
PublisherFacet,
TagFacet,
)
Expand Down Expand Up @@ -81,6 +82,15 @@ def article_facets(
return facets.all_facets(session, limit)


@router.get("/origins", response_model=OriginCounts)
def article_origins(session: SessionDep, since: str) -> OriginCounts:
try:
since_dt = datetime.fromisoformat(since)
except ValueError:
raise HTTPException(status_code=422, detail="Invalid 'since' datetime")
return facets.origin_counts(session, since_dt)


@router.get("/publishers", response_model=list[PublisherFacet])
def search_publishers(
session: SessionDep,
Expand Down
90 changes: 89 additions & 1 deletion backend/app/catalog/tests/test_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import uuid
from collections.abc import Generator
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta

import pytest
Expand All @@ -15,7 +16,7 @@
from app.audience import service
from app.audience.tests.factories import authentication_token_from_email
from app.catalog import semantic_search
from app.catalog.models import Article, ArticleKind, PublisherKind
from app.catalog.models import Article, ArticleKind, Origin, PublisherKind
from app.catalog.tests.factories import (
create_random_article,
create_random_publisher,
Expand Down Expand Up @@ -64,6 +65,7 @@ def auth_client(
"/publishers",
"/tags",
"/stats",
"/origins?since=2026-01-01",
]


Expand Down Expand Up @@ -529,3 +531,89 @@ def test_search_articles_has_like_count_and_liked_by_me(
for article in r.json()["data"]:
assert "like_count" in article
assert "liked_by_me" in article


# --- origin counts -----------------------------------------------------------


@dataclass
class _Origins:
total: int
unlabelled: int
by_origin: dict[str, int]
by_found_via: dict[str | None, int]


def _origins(client: TestClient, since: str) -> _Origins:
r = client.get(f"{ARTICLES_URL}/origins", params={"since": since})
assert r.status_code == 200
body = r.json()
return _Origins(
total=body["total"],
unlabelled=body["unlabelled"],
by_origin={o["origin"]: o["count"] for o in body["origins"]},
by_found_via={f["name"]: f["count"] for f in body["found_via"]},
)


def _grew[K](after: dict[K, int], before: dict[K, int]) -> dict[K, int]:
"""The keys whose count changed, and by how much."""
return {
k: after.get(k, 0) - before.get(k, 0)
for k in after.keys() | before.keys()
if after.get(k, 0) != before.get(k, 0)
}


def test_origin_counts_lists_every_origin(client: TestClient) -> None:
assert set(_origins(client, RECENT_SINCE).by_origin) == {o.value for o in Origin}


def test_origin_counts_follow_the_from_filter(client: TestClient, db: Session) -> None:
"""A company's repo is a repo, a community post is media, and an unknown
publisher's post falls under no origin."""
now = datetime.now(UTC)
company = create_random_publisher(db, kind=PublisherKind.company)
community = create_random_publisher(db, kind=PublisherKind.community)
unknown = create_random_publisher(db, kind=PublisherKind.unknown)

before = _origins(client, RECENT_SINCE)
create_random_article(
db, publisher_id=company.id, kind=ArticleKind.repo, published_at=now
)
create_random_article(db, publisher_id=community.id, published_at=now)
create_random_article(db, publisher_id=unknown.id, published_at=now)
after = _origins(client, RECENT_SINCE)

assert _grew(after.by_origin, before.by_origin) == {"repo": 1, "media": 1}
assert after.unlabelled == before.unlabelled + 1
assert after.total == before.total + 3


def test_origin_counts_are_bounded_by_since(client: TestClient, db: Session) -> None:
lab = create_random_publisher(db, kind=PublisherKind.lab)
before = _origins(client, RECENT_SINCE)
create_random_article(
db, publisher_id=lab.id, published_at=datetime.now(UTC) - timedelta(days=10)
)
assert _origins(client, RECENT_SINCE) == before


def test_origin_counts_group_found_via(client: TestClient, db: Session) -> None:
now = datetime.now(UTC)
before = _origins(client, RECENT_SINCE)
create_random_article(db, published_at=now)
create_random_article(db, published_at=now, found_via="Hacker News")
create_random_article(db, published_at=now, found_via="Hacker News")
create_random_article(db, published_at=now, found_via="AINews")
after = _origins(client, RECENT_SINCE)
assert _grew(after.by_found_via, before.by_found_via) == {
None: 1,
"Hacker News": 2,
"AINews": 1,
}


def test_origin_counts_malformed_since_returns_422(client: TestClient) -> None:
r = client.get(f"{ARTICLES_URL}/origins", params={"since": "not-a-date"})
assert r.status_code == 422
1 change: 1 addition & 0 deletions docs/product.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ Each entry shows its publisher, and its kind when it is a repo, paper or model.
Companies, Writers, Media, Repos, Papers, Models), by topic (a hand-curated set of
tags and kinds, e.g. "Open weights" or "Harnesses"), and by time window
(last 3 days / week / month).
A left rail beside the feed shows how that window splits by origin and how its articles were found (publisher feeds, Hacker News, newsletters); a row sets the From filter.

**Semantic search.** Typing a query searches by meaning, not keyword — it embeds your
query and finds the nearest articles in vector space, so "how do I run a model locally"
Expand Down
73 changes: 73 additions & 0 deletions frontend/src/client/schemas.gen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,29 @@ export const ArticlesPublicSchema = {
title: 'ArticlesPublic'
} as const;

export const FoundViaCountSchema = {
properties: {
name: {
anyOf: [
{
type: 'string'
},
{
type: 'null'
}
],
title: 'Name'
},
count: {
type: 'integer',
title: 'Count'
}
},
type: 'object',
required: ['name', 'count'],
title: 'FoundViaCount'
} as const;

export const HTTPValidationErrorSchema = {
properties: {
detail: {
Expand Down Expand Up @@ -302,6 +325,56 @@ repo, a paper, a model), otherwise who published it. Not stored; each
article falls under exactly one, see \`catalog.articles.origin_condition\`.`
} as const;

export const OriginCountSchema = {
properties: {
origin: {
'$ref': '#/components/schemas/Origin'
},
count: {
type: 'integer',
title: 'Count'
}
},
type: 'object',
required: ['origin', 'count'],
title: 'OriginCount'
} as const;

export const OriginCountsSchema = {
properties: {
since: {
type: 'string',
format: 'date-time',
title: 'Since'
},
total: {
type: 'integer',
title: 'Total'
},
origins: {
items: {
'$ref': '#/components/schemas/OriginCount'
},
type: 'array',
title: 'Origins'
},
unlabelled: {
type: 'integer',
title: 'Unlabelled'
},
found_via: {
items: {
'$ref': '#/components/schemas/FoundViaCount'
},
type: 'array',
title: 'Found Via'
}
},
type: 'object',
required: ['since', 'total', 'origins', 'unlabelled', 'found_via'],
title: 'OriginCounts'
} as const;

export const PublisherFacetSchema = {
properties: {
slug: {
Expand Down
22 changes: 21 additions & 1 deletion frontend/src/client/sdk.gen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import type { CancelablePromise } from './core/CancelablePromise';
import { OpenAPI } from './core/OpenAPI';
import { request as __request } from './core/request';
import type { AnalyticsCollectEventData, AnalyticsCollectEventResponse, AnalyticsReadReportData, AnalyticsReadReportResponse, ArticlesReadArticlesData, ArticlesReadArticlesResponse, ArticlesSearchArticlesData, ArticlesSearchArticlesResponse, ArticlesArticleFacetsData, ArticlesArticleFacetsResponse, ArticlesSearchPublishersData, ArticlesSearchPublishersResponse, ArticlesSearchTagsData, ArticlesSearchTagsResponse, ArticlesArticleStatsResponse, LikesLikeArticleData, LikesLikeArticleResponse, LikesUnlikeArticleData, LikesUnlikeArticleResponse, LikesReadLikedArticlesResponse, LoginRequestSignInLinkData, LoginRequestSignInLinkResponse, LoginLoginWithLinkData, LoginLoginWithLinkResponse, LoginRefreshTokenResponse, LoginTestTokenResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UtilsHealthCheckResponse } from './types.gen';
import type { AnalyticsCollectEventData, AnalyticsCollectEventResponse, AnalyticsReadReportData, AnalyticsReadReportResponse, ArticlesReadArticlesData, ArticlesReadArticlesResponse, ArticlesSearchArticlesData, ArticlesSearchArticlesResponse, ArticlesArticleFacetsData, ArticlesArticleFacetsResponse, ArticlesArticleOriginsData, ArticlesArticleOriginsResponse, ArticlesSearchPublishersData, ArticlesSearchPublishersResponse, ArticlesSearchTagsData, ArticlesSearchTagsResponse, ArticlesArticleStatsResponse, LikesLikeArticleData, LikesLikeArticleResponse, LikesUnlikeArticleData, LikesUnlikeArticleResponse, LikesReadLikedArticlesResponse, LoginRequestSignInLinkData, LoginRequestSignInLinkResponse, LoginLoginWithLinkData, LoginLoginWithLinkResponse, LoginRefreshTokenResponse, LoginTestTokenResponse, UsersReadUserMeResponse, UsersDeleteUserMeResponse, UsersUpdateUserMeData, UsersUpdateUserMeResponse, UsersRegisterUserData, UsersRegisterUserResponse, UtilsHealthCheckResponse } from './types.gen';

export class AnalyticsService {
/**
Expand Down Expand Up @@ -130,6 +130,26 @@ export class ArticlesService {
});
}

/**
* Article Origins
* @param data The data for the request.
* @param data.since
* @returns OriginCounts Successful Response
* @throws ApiError
*/
public static articleOrigins(data: ArticlesArticleOriginsData): CancelablePromise<ArticlesArticleOriginsResponse> {
return __request(OpenAPI, {
method: 'GET',
url: '/api/v1/articles/origins',
query: {
since: data.since
},
errors: {
422: 'Validation Error'
}
});
}

/**
* Search Publishers
* @param data The data for the request.
Expand Down
Loading
Loading