diff --git a/src/__tests__/china-sources.test.ts b/src/__tests__/china-sources.test.ts new file mode 100644 index 0000000..762f47d --- /dev/null +++ b/src/__tests__/china-sources.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it, vi } from "vitest"; + +const adapterMocks = vi.hoisted(() => ({ + kr36: vi.fn().mockRejectedValue(new Error("kr36 failed")), + infoqCn: vi.fn().mockRejectedValue(new Error("infoq failed")), + gitee: vi.fn().mockRejectedValue(new Error("gitee failed")), + oschina: vi.fn().mockRejectedValue(new Error("oschina failed")), + juejin: vi.fn().mockRejectedValue(new Error("juejin failed")), +})); + +vi.mock("../kr36.ts", () => ({ fetchKr36Data: adapterMocks.kr36 })); +vi.mock("../infoq-cn.ts", () => ({ fetchInfoqCnData: adapterMocks.infoqCn })); +vi.mock("../gitee.ts", () => ({ fetchGiteeData: adapterMocks.gitee })); +vi.mock("../oschina.ts", () => ({ fetchOschinaData: adapterMocks.oschina })); +vi.mock("../juejin.ts", () => ({ fetchJuejinData: adapterMocks.juejin })); + +import { fetchChinaSourcesData } from "../china-sources.ts"; + +describe("fetchChinaSourcesData", () => { + it("returns structured error statuses when adapters reject", async () => { + const result = await fetchChinaSourcesData(); + + expect(Object.values(result).map(({ status }) => status.state)).toEqual([ + "error", + "error", + "error", + "error", + "error", + ]); + }); +}); diff --git a/src/__tests__/generate-manifest.test.ts b/src/__tests__/generate-manifest.test.ts index 75c0d63..0659f3c 100644 --- a/src/__tests__/generate-manifest.test.ts +++ b/src/__tests__/generate-manifest.test.ts @@ -1,5 +1,10 @@ -import { describe, it, expect } from "vitest"; -import { toRfc822, escapeXml, getReportFiles } from "../generate-manifest.ts"; +import fs from "node:fs"; +import { afterEach, describe, it, expect, vi } from "vitest"; +import { toRfc822, escapeXml, getReportFiles, generateSearchIndex } from "../generate-manifest.ts"; + +afterEach(() => { + vi.restoreAllMocks(); +}); // --------------------------------------------------------------------------- // toRfc822 @@ -79,3 +84,35 @@ describe("getReportFiles", () => { expect(reports).toContain("ai-weekly-en"); }); }); + +describe("generateSearchIndex", () => { + it("indexes topic-pool candidates and legacy topics", () => { + vi.spyOn(fs, "existsSync").mockImplementation((file) => String(file).endsWith("topic-pool.json")); + vi.spyOn(fs, "readFileSync").mockImplementation((file) => { + const text = String(file); + if (text.includes("2026-06-01")) { + return JSON.stringify({ + candidates: [ + { title: "New candidate", score: 88, category: "AI 产品与用户入口", source: "Dev.to" }, + ], + }); + } + return JSON.stringify({ + topics: [{ topic: "Legacy topic", score: 70, category: "模型与技术突破", evidence: ["来源：ArXiv"] }], + }); + }); + const writeSpy = vi.spyOn(fs, "writeFileSync").mockImplementation(() => undefined); + + generateSearchIndex([ + { date: "2026-06-01", reports: ["ai-topic-radar"] }, + { date: "2026-05-31", reports: ["ai-topic-radar"] }, + ]); + + const [, content] = writeSpy.mock.calls.find(([file]) => String(file).endsWith("search-index.json"))!; + const index = JSON.parse(String(content)); + expect(index.topics).toEqual([ + expect.objectContaining({ title: "New candidate", source: "Dev.to" }), + expect.objectContaining({ title: "Legacy topic", source: "来源：ArXiv" }), + ]); + }); +}); diff --git a/src/__tests__/prompt-builders.test.ts b/src/__tests__/prompt-builders.test.ts index fb2aea2..5bd5e07 100644 --- a/src/__tests__/prompt-builders.test.ts +++ b/src/__tests__/prompt-builders.test.ts @@ -18,6 +18,7 @@ import type { RepoDigest } from "../prompts.ts"; import type { TrendingData } from "../trending.ts"; import type { HnData } from "../hn.ts"; import type { WebFetchResult } from "../web.ts"; +import type { SourceState, SourceStatus } from "../source-status.ts"; // --------------------------------------------------------------------------- // Fixtures @@ -189,6 +190,7 @@ describe("buildTrendingPrompt", () => { ], searchRepos: [], trendingFetchSuccess: true, + status: status("github-trending"), }; const result = buildTrendingPrompt(data, "2026-03-09"); expect(result).toContain("org/repo"); @@ -198,7 +200,12 @@ describe("buildTrendingPrompt", () => { }); it("shows fetch failure message when trending fails", () => { - const data: TrendingData = { trendingRepos: [], searchRepos: [], trendingFetchSuccess: false }; + const data: TrendingData = { + trendingRepos: [], + searchRepos: [], + trendingFetchSuccess: false, + status: status("github-trending", "error"), + }; const result = buildTrendingPrompt(data, "2026-03-09"); expect(result).toContain("未能抓取"); }); @@ -218,6 +225,7 @@ describe("buildTrendingPrompt", () => { }, ], trendingFetchSuccess: false, + status: status("github-trending", "error"), }; const result = buildTrendingPrompt(data, "2026-03-09"); expect(result).toContain("[topic:ai-agent]"); @@ -247,6 +255,7 @@ describe("buildWebReportPrompt", () => { }, ], totalDiscovered: 50, + status: status("web-anthropic"), }, ]; const result = buildWebReportPrompt(results, "2026-03-09"); @@ -257,7 +266,14 @@ describe("buildWebReportPrompt", () => { it("shows incremental mode for non-first-run", () => { const results: WebFetchResult[] = [ - { site: "openai", siteName: "OpenAI", isFirstRun: false, newItems: [], totalDiscovered: 100 }, + { + site: "openai", + siteName: "OpenAI", + isFirstRun: false, + newItems: [], + totalDiscovered: 100, + status: status("web-openai", "empty"), + }, ]; const result = buildWebReportPrompt(results, "2026-03-09"); expect(result).toContain("增量更新"); @@ -352,6 +368,7 @@ describe("buildHnPrompt", () => { }, ], fetchSuccess: true, + status: status("hn"), }; const result = buildHnPrompt(data, "2026-03-09"); expect(result).toContain("AI News"); @@ -376,6 +393,7 @@ describe("buildHnPrompt", () => { }, ], fetchSuccess: true, + status: status("hn"), }; const result = buildHnPrompt(data, "2026-03-09", "en"); expect(result).toContain("Score: 10"); @@ -383,3 +401,6 @@ describe("buildHnPrompt", () => { expect(result).toContain("Hacker News"); }); }); +function status(id: string, state: SourceState = "ok"): SourceStatus { + return { id, label: id, state, fetchedCount: 0, acceptedCount: 0 }; +} diff --git a/src/__tests__/report-savers.test.ts b/src/__tests__/report-savers.test.ts new file mode 100644 index 0000000..3a91eb8 --- /dev/null +++ b/src/__tests__/report-savers.test.ts @@ -0,0 +1,125 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { ChinaSourcesData } from "../china-sources.ts"; + +const { callLlmMock } = vi.hoisted(() => ({ + callLlmMock: vi.fn(), +})); + +vi.mock("../report.ts", () => ({ + callLlm: callLlmMock, + saveFile: vi.fn(), + LLM_TOKENS_WEB: 4096, +})); + +vi.mock("../github.ts", () => ({ + createGitHubIssue: vi.fn(), +})); + +import { hasChinaSourcesData } from "../china-sources.ts"; +import { + saveArxivReport, + saveChinaTechReport, + saveHfReport, + saveHnReport, + savePhReport, +} from "../report-savers.ts"; + +function emptyChinaSources(): ChinaSourcesData { + return { + kr36: { articles: [], fetchSuccess: false, status: status("kr36") }, + infoqCn: { + articles: [], + fetchSuccess: true, + status: { + id: "infoq-cn", + label: "InfoQ 中国", + state: "empty", + fetchedCount: 0, + acceptedCount: 0, + }, + }, + gitee: { + projects: [], + fetchSuccess: true, + status: { + id: "gitee", + label: "Gitee", + state: "empty", + fetchedCount: 0, + acceptedCount: 0, + }, + }, + oschina: { news: [], fetchSuccess: false, status: status("oschina") }, + juejin: { + articles: [], + fetchSuccess: true, + status: { + id: "juejin", + label: "掘金", + state: "empty", + fetchedCount: 0, + acceptedCount: 0, + }, + }, + }; +} + +beforeEach(() => { + callLlmMock.mockReset(); +}); + +describe("saveArxivReport", () => { + it("skips LLM generation when a successful fetch contains no papers", async () => { + await saveArxivReport( + { + papers: [], + fetchSuccess: true, + status: { + id: "arxiv", + label: "ArXiv", + state: "empty", + fetchedCount: 0, + acceptedCount: 0, + }, + }, + "2026-06-01 00:00", + "2026-06-01", + "", + "", + ); + + expect(callLlmMock).not.toHaveBeenCalled(); + }); +}); + +describe("empty source reports", () => { + it("skips HN, Product Hunt, and Hugging Face LLM generation when successful fetches are empty", async () => { + await saveHnReport({ stories: [], fetchSuccess: true, status: { ...status("hn") } }, "", "", "", ""); + await savePhReport( + { products: [], fetchSuccess: true, status: { ...status("product-hunt") } }, + "", + "", + "", + "", + ); + await saveHfReport({ models: [], fetchSuccess: true, status: { ...status("hf") } }, "", "", "", ""); + + expect(callLlmMock).not.toHaveBeenCalled(); + }); +}); + +describe("China sources empty results", () => { + it("does not treat successful empty fetches as China source content", () => { + expect(hasChinaSourcesData(emptyChinaSources())).toBe(false); + }); + + it("skips China tech LLM generation when all sources contain no entries", async () => { + await saveChinaTechReport(emptyChinaSources(), "2026-06-01 00:00", "2026-06-01", "", ""); + + expect(callLlmMock).not.toHaveBeenCalled(); + }); +}); + +function status(id: string) { + return { id, label: id, state: "empty" as const, fetchedCount: 0, acceptedCount: 0 }; +} diff --git a/src/__tests__/source-adapters.test.ts b/src/__tests__/source-adapters.test.ts new file mode 100644 index 0000000..5e0991f --- /dev/null +++ b/src/__tests__/source-adapters.test.ts @@ -0,0 +1,505 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { fetchArxivData } from "../arxiv.ts"; +import { fetchGiteeData } from "../gitee.ts"; +import { fetchHfData } from "../hf.ts"; +import { fetchHnData } from "../hn.ts"; +import { fetchInfoqCnData } from "../infoq-cn.ts"; +import { fetchJuejinData } from "../juejin.ts"; +import { fetchKr36Data } from "../kr36.ts"; +import { fetchOschinaData } from "../oschina.ts"; +import { fetchPhData } from "../ph.ts"; +import { fetchTrendingData } from "../trending.ts"; + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); +}); + +describe("source adapters", () => { + it("maps the current InfoQ recommendation response fields", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + Response.json({ + code: 0, + data: [ + { + id: 1, + uuid: "infoq-1", + article_title: "大模型应用进入企业工作流", + article_summary: "企业开始把大模型能力接入真实业务流程。", + publish_time: 1_800_000_000, + author: [{ nickname: "Alice" }], + topic: [{ name: "AI" }], + }, + ], + error: {}, + }), + ), + ); + + const result = await fetchInfoqCnData(); + + expect(result.fetchSuccess).toBe(true); + expect(result.articles).toEqual([ + expect.objectContaining({ + title: "大模型应用进入企业工作流", + summary: "企业开始把大模型能力接入真实业务流程。", + author: "Alice", + topics: ["AI"], + }), + ]); + }); + + it("keeps mapping legacy InfoQ recommendation response fields", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + Response.json({ + data: [ + { + id: 2, + uuid: "infoq-legacy", + title: "LLM 工程落地复盘", + description: "团队总结大模型应用上线经验。", + url: "https://www.infoq.cn/article/infoq-legacy", + publish_time: 1_800_000_000, + author: { name: "Legacy Author" }, + topics: ["LLM"], + }, + ], + }), + ), + ); + + await expect(fetchInfoqCnData()).resolves.toMatchObject({ + fetchSuccess: true, + articles: [ + { + id: "2", + title: "LLM 工程落地复盘", + summary: "团队总结大模型应用上线经验。", + author: "Legacy Author", + topics: ["LLM"], + }, + ], + }); + }); + + it("accepts Juejin responses whose err_msg is success", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + Response.json({ + err_no: 0, + err_msg: "success", + data: [ + { + article_id: "juejin-1", + article_info: { + title: "AI Agent 工程实践", + brief_content: "如何构建稳定的 AI Agent。", + digg_count: 12, + view_count: 340, + publish_time: 1_800_000_000, + }, + author_user_info: { user_name: "Bob" }, + tags: [{ tag_name: "AI" }], + }, + ], + }), + ), + ); + + const result = await fetchJuejinData(); + + expect(result.fetchSuccess).toBe(true); + expect(result.articles).toEqual([ + expect.objectContaining({ + id: "juejin-1", + title: "AI Agent 工程实践", + }), + ]); + }); + + it("uses the Gitee repository search endpoint", async () => { + vi.stubEnv("GITEE_TOKEN", ""); + const fetchMock = vi.fn().mockImplementation(() => Promise.resolve(Response.json([]))); + vi.stubGlobal("fetch", fetchMock); + + await fetchGiteeData(); + + const paths = fetchMock.mock.calls.map(([url]) => new URL(String(url)).pathname); + expect(paths.length).toBeGreaterThan(0); + expect(paths.every((path) => path === "/api/v5/search/repositories")).toBe(true); + }); + + it("queries Gitee with multiple AI keywords and deduplicates repositories", async () => { + vi.stubEnv("GITEE_TOKEN", ""); + const fetchMock = vi.fn().mockImplementation(() => + Promise.resolve( + Response.json([ + { + id: 1, + name: "agent-platform", + full_name: "example/agent-platform", + html_url: "https://gitee.com/example/agent-platform", + description: "AI agent platform", + language: "TypeScript", + stargazers_count: 12, + forks_count: 3, + updated_at: "2026-05-31T00:00:00Z", + namespace: { path: "example" }, + }, + ]), + ), + ); + vi.stubGlobal("fetch", fetchMock); + + const result = await fetchGiteeData(); + + const queries = fetchMock.mock.calls.map(([url]) => new URL(String(url)).searchParams.get("q")); + expect(queries).toEqual(["ai", "llm", "大模型", "agent", "rag"]); + expect(result.projects).toHaveLength(1); + expect(result.fetchSuccess).toBe(true); + expect(result.status.state).toBe("ok"); + }); + + it("sorts Gitee projects globally after querying every keyword", async () => { + vi.stubEnv("GITEE_TOKEN", ""); + const lowStarProjects = Array.from({ length: 30 }, (_, index) => ({ + id: index + 1, + name: `low-${index + 1}`, + full_name: `example/low-${index + 1}`, + html_url: `https://gitee.com/example/low-${index + 1}`, + description: "AI project", + language: "TypeScript", + stargazers_count: 30 - index, + forks_count: 0, + updated_at: "2026-05-31T00:00:00Z", + namespace: { path: "example" }, + })); + const highStarProject = { + id: 31, + name: "high-star", + full_name: "example/high-star", + html_url: "https://gitee.com/example/high-star", + description: "LLM project", + language: "TypeScript", + stargazers_count: 1000, + forks_count: 10, + updated_at: "2026-05-31T00:00:00Z", + namespace: { path: "example" }, + }; + const fetchMock = vi.fn().mockImplementation((url) => { + const query = new URL(String(url)).searchParams.get("q"); + return Promise.resolve( + Response.json(query === "ai" ? lowStarProjects : query === "llm" ? [highStarProject] : []), + ); + }); + vi.stubGlobal("fetch", fetchMock); + + const result = await fetchGiteeData(); + + expect(fetchMock).toHaveBeenCalledTimes(5); + expect(result.projects).toHaveLength(30); + expect(result.projects[0]!.id).toBe(31); + }); + + it("keeps successful Gitee projects when later keywords fail", async () => { + const token = "gitee-secret-token"; + vi.stubEnv("GITEE_TOKEN", token); + const project = (id: number, stars: number) => ({ + id, + name: `project-${id}`, + full_name: `example/project-${id}`, + html_url: `https://gitee.com/example/project-${id}`, + description: "AI project", + language: "TypeScript", + stargazers_count: stars, + forks_count: 0, + updated_at: "2026-05-31T00:00:00Z", + namespace: { path: "example" }, + }); + const fetchMock = vi.fn().mockImplementation((url) => { + const query = new URL(String(url)).searchParams.get("q"); + if (query === "ai") return Promise.resolve(Response.json([project(1, 10)])); + if (query === "llm") return Promise.resolve(new Response("", { status: 503 })); + if (query === "大模型") return Promise.resolve(Response.json([project(2, 20)])); + if (query === "agent") return Promise.resolve(Response.json({ unexpected: true })); + return Promise.reject(new Error(`failed ${String(url)}`)); + }); + vi.stubGlobal("fetch", fetchMock); + + const result = await fetchGiteeData(); + + expect(fetchMock).toHaveBeenCalledTimes(5); + expect(result.projects.map(({ id }) => id)).toEqual([2, 1]); + expect(result.fetchSuccess).toBe(false); + expect(result.status).toMatchObject({ + state: "error", + acceptedCount: 2, + }); + expect(result.status.detail).toContain("partial failure"); + expect(result.status.detail).not.toContain(token); + }); + + it("distinguishes an empty Gitee search from an HTTP error", async () => { + vi.stubEnv("GITEE_TOKEN", ""); + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation(() => Promise.resolve(Response.json([]))), + ); + + await expect(fetchGiteeData()).resolves.toMatchObject({ + projects: [], + fetchSuccess: true, + status: { state: "empty" }, + }); + + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("", { status: 503 }))); + + await expect(fetchGiteeData()).resolves.toMatchObject({ + projects: [], + fetchSuccess: false, + status: { state: "error" }, + }); + }); + + it("redacts the Gitee token from fetch failure logs", async () => { + const token = "gitee-secret-token"; + vi.stubEnv("GITEE_TOKEN", token); + vi.stubGlobal( + "fetch", + vi.fn().mockRejectedValue(new Error(`request failed: https://gitee.com/api?access_token=${token}`)), + ); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + + await fetchGiteeData(); + + expect(errorSpy.mock.calls.flat().join("\n")).not.toContain(token); + errorSpy.mockRestore(); + }); + + it("queries ArXiv categories in one request", async () => { + vi.useFakeTimers(); + const fetchMock = vi.fn().mockImplementation(() => + Promise.resolve( + new Response( + ` + `, + { status: 200 }, + ), + ), + ); + vi.stubGlobal("fetch", fetchMock); + + const resultPromise = fetchArxivData(); + await vi.runAllTimersAsync(); + await resultPromise; + + expect(fetchMock).toHaveBeenCalledTimes(1); + const query = new URL(String(fetchMock.mock.calls[0]![0])).searchParams.get("search_query"); + expect(query).toContain("cat:cs.AI"); + expect(query).toContain("cat:cs.CL"); + expect(query).toContain("cat:cs.LG"); + }); + + it("retries ArXiv 429 responses before reporting an error", async () => { + vi.useFakeTimers(); + const fetchMock = vi.fn().mockResolvedValue(new Response("", { status: 429 })); + vi.stubGlobal("fetch", fetchMock); + + const resultPromise = fetchArxivData(); + await vi.runAllTimersAsync(); + + await expect(resultPromise).resolves.toMatchObject({ + papers: [], + fetchSuccess: false, + status: { state: "error", detail: "HTTP 429" }, + }); + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it("backs off ArXiv 429 retries by 3s then 6s when Retry-After is absent", async () => { + vi.useFakeTimers(); + const fetchMock = vi.fn().mockResolvedValue(new Response("", { status: 429 })); + vi.stubGlobal("fetch", fetchMock); + + const resultPromise = fetchArxivData(); + await vi.advanceTimersByTimeAsync(2_999); + expect(fetchMock).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + expect(fetchMock).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(5_999); + expect(fetchMock).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(1); + await resultPromise; + + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it("prefers a valid ArXiv Retry-After delay", async () => { + vi.useFakeTimers(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response("", { status: 429, headers: { "Retry-After": "7" } })) + .mockResolvedValueOnce(new Response("", { status: 429 })) + .mockResolvedValueOnce(new Response("", { status: 503 })); + vi.stubGlobal("fetch", fetchMock); + + const resultPromise = fetchArxivData(); + await vi.advanceTimersByTimeAsync(6_999); + expect(fetchMock).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + expect(fetchMock).toHaveBeenCalledTimes(2); + await vi.runAllTimersAsync(); + await expect(resultPromise).resolves.toMatchObject({ + fetchSuccess: false, + status: { state: "error", detail: "HTTP 503" }, + }); + }); + + it("reports an empty ArXiv feed as a successful empty fetch", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + new Response( + ` + `, + { status: 200 }, + ), + ), + ); + + await expect(fetchArxivData()).resolves.toMatchObject({ + papers: [], + fetchSuccess: true, + status: { state: "empty" }, + }); + }); + + it("reports HTTP 200 ArXiv block pages as errors", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("blocked", { status: 200 }))); + + await expect(fetchArxivData()).resolves.toMatchObject({ + papers: [], + fetchSuccess: false, + status: { state: "error", detail: "unexpected Atom feed shape" }, + }); + }); + + it("rejects Juejin responses when err_no is non-zero", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + Response.json({ + err_no: 403, + err_msg: "", + data: [ + { + article_id: "juejin-1", + article_info: { + title: "AI Agent 工程实践", + brief_content: "如何构建稳定的 AI Agent。", + digg_count: 12, + view_count: 340, + publish_time: 1_800_000_000, + }, + author_user_info: { user_name: "Bob" }, + tags: [{ tag_name: "AI" }], + }, + ], + }), + ), + ); + + await expect(fetchJuejinData()).resolves.toMatchObject({ + articles: [], + fetchSuccess: false, + status: { state: "error" }, + }); + }); + + it("reports malformed Juejin responses as errors", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(Response.json(null))); + + await expect(fetchJuejinData()).resolves.toMatchObject({ + articles: [], + fetchSuccess: false, + status: { state: "error", detail: "unexpected response shape" }, + }); + }); + + it("reports successful empty legacy adapters as empty", async () => { + vi.stubEnv("PRODUCTHUNT_TOKEN", "configured"); + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation((url) => { + const value = String(url); + if (value.includes("hn.algolia.com")) return Promise.resolve(Response.json({ hits: [] })); + if (value.includes("producthunt.com")) + return Promise.resolve(Response.json({ data: { posts: { edges: [] } } })); + if (value.includes("huggingface.co")) return Promise.resolve(Response.json([])); + if (value.includes("36kr.com")) return Promise.resolve(new Response("", { status: 200 })); + if (value.includes("oschina.net")) + return Promise.resolve(new Response("", { status: 200 })); + throw new Error(`Unexpected URL: ${value}`); + }), + ); + + await expect(fetchHnData()).resolves.toMatchObject({ fetchSuccess: true, status: { state: "empty" } }); + await expect(fetchPhData()).resolves.toMatchObject({ fetchSuccess: true, status: { state: "empty" } }); + await expect(fetchHfData()).resolves.toMatchObject({ fetchSuccess: true, status: { state: "empty" } }); + await expect(fetchKr36Data()).resolves.toMatchObject({ fetchSuccess: true, status: { state: "empty" } }); + await expect(fetchOschinaData()).resolves.toMatchObject({ + fetchSuccess: true, + status: { state: "empty" }, + }); + }); + + it("reports HTTP 200 RSS block pages as errors", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockImplementation(() => Promise.resolve(new Response("blocked", { status: 200 }))), + ); + + await expect(fetchKr36Data()).resolves.toMatchObject({ + fetchSuccess: false, + status: { state: "error", detail: "unexpected RSS response shape" }, + }); + await expect(fetchOschinaData()).resolves.toMatchObject({ + fetchSuccess: false, + status: { state: "error", detail: "unexpected RSS response shape" }, + }); + }); + + it("reports Product Hunt without a token as skipped", async () => { + vi.stubEnv("PRODUCTHUNT_TOKEN", ""); + + await expect(fetchPhData()).resolves.toMatchObject({ + fetchSuccess: false, + status: { state: "skipped" }, + }); + }); + + it("reports GitHub Trending HTML parse failures with a structured status", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation((url) => { + const value = String(url); + if (value.includes("github.com/trending")) + return Promise.resolve(new Response("", { status: 200 })); + return Promise.resolve(Response.json({ items: [] })); + }), + ); + + await expect(fetchTrendingData()).resolves.toMatchObject({ + trendingFetchSuccess: false, + status: { id: "github-trending", state: "error" }, + }); + }); +}); diff --git a/src/__tests__/source-status.test.ts b/src/__tests__/source-status.test.ts new file mode 100644 index 0000000..d45e2de --- /dev/null +++ b/src/__tests__/source-status.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; +import { createSourceStatus } from "../source-status.ts"; + +describe("createSourceStatus", () => { + it("returns ok when accepted content exists", () => { + expect( + createSourceStatus({ + id: "infoq-cn", + label: "InfoQ 中国", + fetchedCount: 12, + acceptedCount: 3, + }), + ).toEqual({ + id: "infoq-cn", + label: "InfoQ 中国", + state: "ok", + fetchedCount: 12, + acceptedCount: 3, + }); + }); + + it("returns empty when fetching succeeds without accepted content", () => { + expect( + createSourceStatus({ + id: "infoq-cn", + label: "InfoQ 中国", + fetchedCount: 12, + acceptedCount: 0, + }), + ).toEqual({ + id: "infoq-cn", + label: "InfoQ 中国", + state: "empty", + fetchedCount: 12, + acceptedCount: 0, + }); + }); + + it("returns error when the adapter reports a failure", () => { + expect( + createSourceStatus({ + id: "gitee", + label: "Gitee", + fetchedCount: 0, + acceptedCount: 0, + error: "HTTP 404", + }), + ).toEqual({ + id: "gitee", + label: "Gitee", + state: "error", + fetchedCount: 0, + acceptedCount: 0, + detail: "HTTP 404", + }); + }); + + it("returns skipped when an optional source is disabled", () => { + expect( + createSourceStatus({ + id: "product-hunt", + label: "Product Hunt", + fetchedCount: 0, + acceptedCount: 0, + skipped: "PRODUCTHUNT_TOKEN 未配置", + }), + ).toEqual({ + id: "product-hunt", + label: "Product Hunt", + state: "skipped", + fetchedCount: 0, + acceptedCount: 0, + detail: "PRODUCTHUNT_TOKEN 未配置", + }); + }); +}); diff --git a/src/__tests__/topic-radar.test.ts b/src/__tests__/topic-radar.test.ts index c1a9d55..bc1dacb 100644 --- a/src/__tests__/topic-radar.test.ts +++ b/src/__tests__/topic-radar.test.ts @@ -1,6 +1,21 @@ -import { describe, expect, it } from "vitest"; -import { buildTopicRadar, buildTopicRadarHtml, buildTopicRadarMarkdown } from "../topic-radar.ts"; +import fs from "node:fs"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + buildTopicRadar, + buildTopicRadarHtml, + buildTopicRadarMarkdown, + saveTopicRadar, +} from "../topic-radar.ts"; import type { TopicRadarInput } from "../topic-radar.ts"; +import type { SourceStatus } from "../source-status.ts"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function status(id: string, state: SourceStatus["state"] = "empty"): SourceStatus { + return { id, label: id, state, fetchedCount: 0, acceptedCount: 0 }; +} function baseInput(): TopicRadarInput { return { @@ -11,22 +26,33 @@ function baseInput(): TopicRadarInput { trendingRepos: [], searchRepos: [], trendingFetchSuccess: true, + status: status("github-trending"), }, hnData: { stories: [], fetchSuccess: true, + status: status("hn"), }, phData: { products: [], fetchSuccess: true, + status: status("product-hunt"), }, arxivData: { papers: [], fetchSuccess: true, + status: { + id: "arxiv", + label: "ArXiv", + state: "empty", + fetchedCount: 0, + acceptedCount: 0, + }, }, hfData: { models: [], fetchSuccess: true, + status: status("hf"), }, webResults: [ { @@ -35,6 +61,7 @@ function baseInput(): TopicRadarInput { isFirstRun: false, newItems: [], totalDiscovered: 0, + status: status("web-openai"), }, ], }; @@ -70,18 +97,284 @@ describe("buildTopicRadar", () => { expect(result.candidates[0]!.evidence.join(" ")).toContain("Product Hunt"); }); + it("includes Dev.to and Lobste.rs community sources in candidates", () => { + const input = baseInput(); + input.devtoData = { + fetchSuccess: true, + articles: [ + { + id: 1, + title: "Building reliable AI agents", + description: "Enterprise agent workflow with observability and evaluation.", + url: "https://dev.to/example/agents", + publishedAt: "2026-05-20T00:00:00Z", + positiveReactionsCount: 420, + commentsCount: 30, + readingTimeMinutes: 8, + tags: ["ai", "agents"], + user: "Alice", + }, + ], + }; + input.lobstersData = { + fetchSuccess: true, + stories: [ + { + title: "LLM evals in production", + url: "https://example.com/evals", + commentsUrl: "https://lobste.rs/s/abc", + score: 120, + commentCount: 45, + author: "bob", + publishedAt: "2026-05-20T00:00:00Z", + tags: ["ai"], + }, + ], + }; + + const sources = buildTopicRadar(input).candidates.map((candidate) => candidate.source); + + expect(sources).toContain("Dev.to"); + expect(sources).toContain("Lobste.rs"); + }); + + it("limits GitHub and Gitee candidates after deduplication", () => { + const input = baseInput(); + input.trendingData.searchRepos = Array.from({ length: 30 }, (_, index) => ({ + fullName: `example/repo-${index}`, + description: "Enterprise AI agent workflow platform", + language: "TypeScript", + stargazersCount: 100_000 - index, + pushedAt: "2026-05-20T00:00:00Z", + url: `https://github.com/example/repo-${index}`, + searchQuery: "ai-agent", + })); + input.chinaSourcesData = { + kr36: { articles: [], fetchSuccess: true, status: status("kr36") }, + infoqCn: { articles: [], fetchSuccess: true, status: status("infoq-cn") }, + gitee: { + projects: Array.from({ length: 10 }, (_, index) => ({ + id: index, + name: `gitee-${index}`, + fullName: `gitee/gitee-${index}`, + url: `https://gitee.com/gitee/gitee-${index}`, + description: "AI agent project", + language: "TypeScript", + stars: 50_000 - index, + forks: 100, + updatedAt: "2026-05-20T00:00:00Z", + namespace: "gitee", + })), + fetchSuccess: true, + status: status("gitee"), + }, + oschina: { news: [], fetchSuccess: true, status: status("oschina") }, + juejin: { articles: [], fetchSuccess: true, status: status("juejin") }, + }; + + const candidates = buildTopicRadar(input).candidates; + const repoCount = candidates.filter( + (candidate) => candidate.source.startsWith("GitHub Search") || candidate.source === "Gitee", + ).length; + + expect(repoCount).toBeLessThanOrEqual(20); + }); + it("keeps source warnings without blocking topic pool generation", () => { const input = baseInput(); input.trendingData.trendingFetchSuccess = false; + input.trendingData.status = status("github-trending", "error"); input.hnData.fetchSuccess = false; + input.hnData.status = status("hn", "error"); input.phData.fetchSuccess = false; + input.phData.status = { ...status("product-hunt", "skipped"), label: "Product Hunt" }; const result = buildTopicRadar(input); expect(result.candidates).toEqual([]); expect(result.warnings.join("\n")).toContain("GitHub Trending"); expect(result.warnings.join("\n")).toContain("Hacker News"); - expect(result.warnings.join("\n")).toContain("Product Hunt"); + expect(result.notices.join("\n")).toContain("Product Hunt"); + }); + + it("renders an empty notice instead of an error warning for a successful empty source", () => { + const input = baseInput(); + Object.assign(input.arxivData, { + fetchSuccess: true, + status: { + id: "arxiv", + label: "ArXiv", + state: "empty", + fetchedCount: 0, + acceptedCount: 0, + }, + }); + + const result = buildTopicRadar(input); + const notices = result.notices.join("\n"); + const warnings = result.warnings.join("\n"); + + expect(notices).toContain("ArXiv 暂无"); + expect(warnings).not.toContain("ArXiv 暂无"); + expect(warnings).not.toContain("ArXiv 获取失败"); + }); + + it("renders an error warning when the source status reports an error", () => { + const input = baseInput(); + Object.assign(input.arxivData, { + fetchSuccess: true, + status: { + id: "arxiv", + label: "ArXiv", + state: "error", + fetchedCount: 0, + acceptedCount: 0, + detail: "HTTP 429", + }, + }); + + const result = buildTopicRadar(input); + + expect(result.notices.join("\n")).not.toContain("ArXiv 获取失败"); + expect(result.warnings.join("\n")).toContain("ArXiv 获取失败"); + expect(buildTopicRadarMarkdown(result)).toContain("ArXiv 获取失败"); + expect(buildTopicRadarHtml(result)).toContain("ArXiv 获取失败"); + }); + + it("preserves structured statuses from adapters", () => { + const input = baseInput(); + Object.assign(input.trendingData, { status: status("github-trending") }); + Object.assign(input.hnData, { status: status("hn") }); + Object.assign(input.phData, { status: status("product-hunt", "skipped") }); + Object.assign(input.hfData, { status: status("hf") }); + input.webResults = [ + { + site: "anthropic", + siteName: "Anthropic", + isFirstRun: false, + newItems: [], + totalDiscovered: 0, + status: status("web-anthropic"), + }, + { + site: "openai", + siteName: "OpenAI", + isFirstRun: false, + newItems: [], + totalDiscovered: 0, + status: status("web-openai"), + }, + { + site: "deepmind", + siteName: "DeepMind", + isFirstRun: false, + newItems: [], + totalDiscovered: 0, + status: status("web-deepmind"), + }, + ]; + input.chinaSourcesData = { + kr36: { articles: [], fetchSuccess: true, status: status("kr36") }, + infoqCn: { + articles: [], + fetchSuccess: true, + status: { + id: "infoq-cn", + label: "InfoQ 中国", + state: "empty", + fetchedCount: 12, + acceptedCount: 0, + }, + }, + gitee: { + projects: [], + fetchSuccess: false, + status: { + id: "gitee", + label: "Gitee", + state: "error", + fetchedCount: 0, + acceptedCount: 0, + detail: "HTTP 503", + }, + }, + oschina: { news: [], fetchSuccess: true, status: status("oschina") }, + juejin: { + articles: [], + fetchSuccess: true, + status: { + id: "juejin", + label: "掘金", + state: "empty", + fetchedCount: 20, + acceptedCount: 0, + }, + }, + }; + + expect(buildTopicRadar(input).sourceStatuses).toEqual([ + input.trendingData.status, + input.hnData.status, + input.phData.status, + input.arxivData.status, + input.hfData.status, + input.webResults[0]!.status, + input.webResults[1]!.status, + input.webResults[2]!.status, + input.chinaSourcesData.kr36.status, + input.chinaSourcesData.infoqCn.status, + input.chinaSourcesData.gitee.status, + input.chinaSourcesData.oschina.status, + input.chinaSourcesData.juejin.status, + ]); + }); + + it("shows official source errors as warnings without an all-clear notice", () => { + const input = baseInput(); + input.webResults = [ + { + site: "anthropic", + siteName: "Anthropic", + isFirstRun: false, + newItems: [], + totalDiscovered: 0, + status: status("web-anthropic"), + }, + { + site: "openai", + siteName: "OpenAI", + isFirstRun: false, + newItems: [], + totalDiscovered: 0, + status: status("web-openai", "error"), + }, + { + site: "deepmind", + siteName: "DeepMind", + isFirstRun: false, + newItems: [], + totalDiscovered: 0, + status: status("web-deepmind"), + }, + ]; + + const result = buildTopicRadar(input); + + expect(result.warnings.join("\n")).toContain("OpenAI"); + expect(result.notices.join("\n")).not.toContain("官方内容源今日没有检测到新内容"); + }); + + it("saves structured source statuses in topic-pool JSON", () => { + const input = baseInput(); + const writeSpy = vi.spyOn(fs, "writeFileSync").mockReturnValue(undefined); + vi.spyOn(fs, "mkdirSync").mockReturnValue(undefined); + + const result = buildTopicRadar(input); + saveTopicRadar(result); + + const jsonCall = writeSpy.mock.calls.find(([file]) => String(file).endsWith("topic-pool.json")); + expect(jsonCall).toBeDefined(); + expect(JSON.parse(String(jsonCall![1])).sourceStatuses).toEqual(result.sourceStatuses); }); it("renders a decision-first markdown report", () => { @@ -102,7 +395,9 @@ describe("buildTopicRadar", () => { expect(markdown).toContain("## 今日 Top 深挖选题"); expect(markdown).toContain("## 入池选题"); expect(markdown).toContain("## 按五类选题分类摘要"); - expect(markdown).toContain("## 数据源状态与修复提示"); + expect(markdown).toContain("## 数据源普通状态提示"); + expect(markdown).toContain("## 数据源修复提示"); + expect(markdown).toContain("ArXiv 暂无"); expect(markdown).toContain("| 分数 | 动作 | 题目 | 摘要 | 分类 | 推荐选题 | 推荐理由 | 证据 |"); expect(markdown).toContain("New multimodal model launch"); expect(markdown).toContain( @@ -111,6 +406,12 @@ describe("buildTopicRadar", () => { expect(markdown).toContain("New multimodal model launch 为什么值得关注？（"); }); + it("explains an empty watch list without implying source failure", () => { + const markdown = buildTopicRadarMarkdown(buildTopicRadar(baseInput())); + + expect(markdown).toContain("暂无 50–64 分观察项。这不代表数据源采集失败。"); + }); + it("renders a self-contained html report", () => { const input = baseInput(); input.webResults[0]!.newItems = [ @@ -129,6 +430,9 @@ describe("buildTopicRadar", () => { expect(html).toContain(""); expect(html).toContain("AI 热点选题池 2026-05-20"); expect(html).toContain("今日 Top 深挖选题"); + expect(html).toContain("数据源普通状态提示"); + expect(html).toContain("数据源修复提示"); + expect(html).toContain("ArXiv 暂无"); expect(html).toContain("New multimodal model launch"); expect(html).toContain("topic-pool.json"); expect(html).not.toContain("https://cdn."); diff --git a/src/__tests__/web.test.ts b/src/__tests__/web.test.ts index ae1283b..e56b245 100644 --- a/src/__tests__/web.test.ts +++ b/src/__tests__/web.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from "vitest"; +import { afterEach, describe, it, expect, vi } from "vitest"; import { parseSitemapUrls, isSitemapIndex, @@ -7,8 +7,15 @@ import { urlCategory, titleFromUrl, emptyState, + normalizeWebState, + fetchSiteContent, } from "../web.ts"; +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + // --------------------------------------------------------------------------- // parseSitemapUrls // --------------------------------------------------------------------------- @@ -203,3 +210,220 @@ describe("emptyState", () => { expect(b.anthropic.lastChecked).toBe(""); }); }); + +describe("normalizeWebState", () => { + it("adds missing sites without dropping persisted URLs", () => { + const state = normalizeWebState({ + anthropic: { lastChecked: "2026-05-31", seenUrls: { "https://anthropic.com/news/a": "seen" } }, + openai: { lastChecked: "2026-05-31", seenUrls: { "https://openai.com/news/b": "seen" } }, + }); + + expect(state).toEqual({ + anthropic: { lastChecked: "2026-05-31", seenUrls: { "https://anthropic.com/news/a": "seen" } }, + openai: { lastChecked: "2026-05-31", seenUrls: { "https://openai.com/news/b": "seen" } }, + deepmind: { lastChecked: "", seenUrls: {} }, + }); + }); +}); + +describe("fetchSiteContent", () => { + it("initializes a missing DeepMind state from older persisted data", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + new Response( + ` + `, + { status: 200 }, + ), + ), + ); + + const legacyState = { + anthropic: { lastChecked: "", seenUrls: {} }, + openai: { lastChecked: "", seenUrls: {} }, + }; + + await expect(fetchSiteContent("deepmind", legacyState as never)).resolves.toMatchObject({ + site: "deepmind", + totalDiscovered: 0, + status: { state: "empty" }, + }); + }); + + it("reports newly fetched official content as ok", async () => { + vi.useFakeTimers(); + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation((url) => { + if (String(url).endsWith("/sitemap.xml")) { + return Promise.resolve( + new Response( + `https://deepmind.google/blog/new-model2026-06-02`, + { status: 200 }, + ), + ); + } + return Promise.resolve(new Response("New model
Details
", { status: 200 })); + }), + ); + + const resultPromise = fetchSiteContent("deepmind", emptyState()); + await vi.runAllTimersAsync(); + + await expect(resultPromise).resolves.toMatchObject({ + newItems: [{ title: "New model" }], + status: { state: "ok", acceptedCount: 1 }, + }); + }); + + it("retries body fetches that failed before advancing the seen URL cursor", async () => { + vi.useFakeTimers(); + const pageUrl = "https://deepmind.google/blog/retry-model"; + const state = emptyState(); + let bodyRequests = 0; + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation((url) => { + if (String(url).endsWith("/sitemap.xml")) { + return Promise.resolve( + new Response(`${pageUrl}2026-06-02`, { + status: 200, + }), + ); + } + bodyRequests++; + return Promise.resolve( + bodyRequests === 1 + ? new Response("", { status: 503 }) + : new Response("Retry model
Details
", { status: 200 }), + ); + }), + ); + + const firstResultPromise = fetchSiteContent("deepmind", state); + await vi.runAllTimersAsync(); + await expect(firstResultPromise).resolves.toMatchObject({ + newItems: [], + status: { state: "error" }, + }); + expect(state.deepmind.seenUrls[pageUrl]).toBeUndefined(); + + const secondResultPromise = fetchSiteContent("deepmind", state); + await vi.runAllTimersAsync(); + await expect(secondResultPromise).resolves.toMatchObject({ + newItems: [{ url: pageUrl, title: "Retry model" }], + status: { state: "ok" }, + }); + expect(bodyRequests).toBe(2); + }); + + it("preserves an older seen URL cursor when a changed page body fetch fails", async () => { + vi.useFakeTimers(); + const pageUrl = "https://deepmind.google/blog/changed-model"; + const state = emptyState(); + state.deepmind.seenUrls[pageUrl] = "2026-06-01"; + let bodyRequests = 0; + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation((url) => { + if (String(url).endsWith("/sitemap.xml")) { + return Promise.resolve( + new Response(`${pageUrl}2026-06-02`, { + status: 200, + }), + ); + } + bodyRequests++; + return Promise.resolve( + bodyRequests === 1 + ? new Response("", { status: 503 }) + : new Response("Changed model
Details
", { status: 200 }), + ); + }), + ); + + const firstResultPromise = fetchSiteContent("deepmind", state); + await vi.runAllTimersAsync(); + await expect(firstResultPromise).resolves.toMatchObject({ + newItems: [], + status: { state: "error" }, + }); + expect(state.deepmind.seenUrls[pageUrl]).toBe("2026-06-01"); + + const secondResultPromise = fetchSiteContent("deepmind", state); + await vi.runAllTimersAsync(); + await expect(secondResultPromise).resolves.toMatchObject({ + newItems: [{ url: pageUrl, title: "Changed model" }], + status: { state: "ok" }, + }); + expect(bodyRequests).toBe(2); + }); + + it("reports OpenAI as error when every sub-sitemap fails", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("", { status: 503 }))); + + await expect(fetchSiteContent("openai", emptyState())).resolves.toMatchObject({ + newItems: [], + status: { id: "web-openai", state: "error", acceptedCount: 0 }, + }); + }); + + it("reports HTTP 200 sitemap block pages as errors", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("blocked", { status: 200 }))); + + await expect(fetchSiteContent("deepmind", emptyState())).rejects.toThrow("unexpected sitemap response"); + }); + + it("treats malformed OpenAI sub-sitemaps as partial failures", async () => { + vi.useFakeTimers(); + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation((url) => { + if (String(url).includes("/research/")) { + return Promise.resolve( + new Response( + `https://openai.com/research/new-model2026-06-02`, + { status: 200 }, + ), + ); + } + return Promise.resolve(new Response("blocked", { status: 200 })); + }), + ); + + const resultPromise = fetchSiteContent("openai", emptyState()); + await vi.runAllTimersAsync(); + + await expect(resultPromise).resolves.toMatchObject({ + newItems: [{ url: "https://openai.com/research/new-model" }], + status: { state: "error", acceptedCount: 1 }, + }); + }); + + it("keeps OpenAI items but reports error when some sub-sitemaps fail", async () => { + vi.useFakeTimers(); + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation((url) => { + if (String(url).includes("/research/")) { + return Promise.resolve( + new Response( + `https://openai.com/research/new-model2026-06-02`, + { status: 200 }, + ), + ); + } + return Promise.resolve(new Response("", { status: 503 })); + }), + ); + + const resultPromise = fetchSiteContent("openai", emptyState()); + await vi.runAllTimersAsync(); + + await expect(resultPromise).resolves.toMatchObject({ + newItems: [{ url: "https://openai.com/research/new-model" }], + status: { id: "web-openai", state: "error", acceptedCount: 1 }, + }); + }); +}); diff --git a/src/arxiv.ts b/src/arxiv.ts index ab55059..9e16554 100644 --- a/src/arxiv.ts +++ b/src/arxiv.ts @@ -5,6 +5,8 @@ * sorted by submission date, filtered to last 48h. */ +import { createSourceStatus, type SourceStatus } from "./source-status.ts"; + // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- @@ -24,6 +26,7 @@ export interface ArxivPaper { export interface ArxivData { papers: ArxivPaper[]; fetchSuccess: boolean; + status: SourceStatus; } // --------------------------------------------------------------------------- @@ -36,8 +39,9 @@ const API_URL = "https://export.arxiv.org/api/query"; /** ArXiv categories to search. */ const CATEGORIES = ["cs.AI", "cs.CL", "cs.LG"]; -/** Delay between requests (ArXiv asks for 3s). */ +/** Delay before retrying a rate-limited request. */ const REQUEST_DELAY_MS = 3000; +const MAX_ATTEMPTS = 3; // --------------------------------------------------------------------------- // XML helpers (lightweight, no dependency) @@ -104,46 +108,99 @@ async function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -export async function fetchArxivData(): Promise { - const seen = new Map(); +function retryDelayMs(resp: Response, attempt: number): number { + const retryAfter = resp.headers.get("retry-after"); + if (retryAfter !== null) { + const seconds = Number(retryAfter); + if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000; + const dateMs = Date.parse(retryAfter); + if (!Number.isNaN(dateMs)) return Math.max(0, dateMs - Date.now()); + } + return REQUEST_DELAY_MS * 2 ** attempt; +} - for (let i = 0; i < CATEGORIES.length; i++) { - const cat = CATEGORIES[i]!; - if (i > 0) await sleep(REQUEST_DELAY_MS); +function isAtomFeed(xml: string): boolean { + return /]/.test(xml); +} +export async function fetchArxivData(): Promise { + const seen = new Map(); + const params = new URLSearchParams({ + search_query: CATEGORIES.map((cat) => `cat:${cat}`).join(" OR "), + sortBy: "submittedDate", + sortOrder: "descending", + max_results: String(ARXIV_MAX_RESULTS), + }); + let xml = ""; + + for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { try { - const params = new URLSearchParams({ - search_query: `cat:${cat}`, - sortBy: "submittedDate", - sortOrder: "descending", - max_results: String(ARXIV_MAX_RESULTS), - }); - const resp = await fetch(`${API_URL}?${params}`, { headers: { "User-Agent": "ai-topic-radar/1.0" }, }); if (!resp.ok) { - console.error(` [arxiv] ${cat}: HTTP ${resp.status}`); - continue; - } - - const xml = await resp.text(); - - // Split into entries - const entryBlocks = xml.split("").slice(1); - for (const block of entryBlocks) { - const paper = parseEntry("" + block); - if (paper && !seen.has(paper.id)) { - seen.set(paper.id, paper); + const error = `HTTP ${resp.status}`; + console.error(` [arxiv] ${error}`); + if (resp.status === 429 && attempt < MAX_ATTEMPTS - 1) { + await sleep(retryDelayMs(resp, attempt)); + continue; } + return { + papers: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "arxiv", + label: "ArXiv", + fetchedCount: 0, + acceptedCount: 0, + error, + }), + }; } - console.log(` [arxiv] ${cat}: ${entryBlocks.length} papers`); + xml = await resp.text(); + if (!isAtomFeed(xml)) { + const error = "unexpected Atom feed shape"; + console.error(` [arxiv] ${error}`); + return { + papers: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "arxiv", + label: "ArXiv", + fetchedCount: 0, + acceptedCount: 0, + error, + }), + }; + } + break; } catch (err) { - console.error(` [arxiv] ${cat}: ${err}`); + const error = String(err); + console.error(` [arxiv] fetch failed: ${error}`); + return { + papers: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "arxiv", + label: "ArXiv", + fetchedCount: 0, + acceptedCount: 0, + error, + }), + }; + } + } + + const entryBlocks = xml.split("").slice(1); + for (const block of entryBlocks) { + const paper = parseEntry("" + block); + if (paper && !seen.has(paper.id)) { + seen.set(paper.id, paper); } } + console.log(` [arxiv] ${entryBlocks.length} papers fetched`); // Filter to last 48h (ArXiv has a ~1-day publishing delay, so 24h would miss today's batch) const cutoff = Date.now() - 48 * 60 * 60 * 1000; @@ -153,5 +210,14 @@ export async function fetchArxivData(): Promise { .slice(0, ARXIV_MAX_RESULTS); console.log(` [arxiv] ${papers.length} papers (from ${seen.size} unique)`); - return { papers, fetchSuccess: papers.length > 0 }; + return { + papers, + fetchSuccess: true, + status: createSourceStatus({ + id: "arxiv", + label: "ArXiv", + fetchedCount: entryBlocks.length, + acceptedCount: papers.length, + }), + }; } diff --git a/src/china-sources.ts b/src/china-sources.ts index d39afcf..4fdd036 100644 --- a/src/china-sources.ts +++ b/src/china-sources.ts @@ -8,6 +8,7 @@ import { fetchInfoqCnData, type InfoqCnData } from "./infoq-cn.ts"; import { fetchGiteeData, type GiteeData } from "./gitee.ts"; import { fetchOschinaData, type OschinaData } from "./oschina.ts"; import { fetchJuejinData, type JuejinData } from "./juejin.ts"; +import { createSourceStatus } from "./source-status.ts"; export interface ChinaSourcesData { kr36: Kr36Data; @@ -19,13 +20,7 @@ export interface ChinaSourcesData { /** Check if any Chinese source has data. */ export function hasChinaSourcesData(data: ChinaSourcesData): boolean { - return ( - data.kr36.fetchSuccess || - data.infoqCn.fetchSuccess || - data.gitee.fetchSuccess || - data.oschina.fetchSuccess || - data.juejin.fetchSuccess - ); + return countChinaSourcesItems(data) > 0; } /** Total items across all Chinese sources. */ @@ -41,11 +36,71 @@ export function countChinaSourcesItems(data: ChinaSourcesData): number { export async function fetchChinaSourcesData(): Promise { const [kr36, infoqCn, gitee, oschina, juejin] = await Promise.all([ - fetchKr36Data().catch((): Kr36Data => ({ articles: [], fetchSuccess: false })), - fetchInfoqCnData().catch((): InfoqCnData => ({ articles: [], fetchSuccess: false })), - fetchGiteeData().catch((): GiteeData => ({ projects: [], fetchSuccess: false })), - fetchOschinaData().catch((): OschinaData => ({ news: [], fetchSuccess: false })), - fetchJuejinData().catch((): JuejinData => ({ articles: [], fetchSuccess: false })), + fetchKr36Data().catch( + (): Kr36Data => ({ + articles: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "kr36", + label: "36kr", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }), + ), + fetchInfoqCnData().catch( + (): InfoqCnData => ({ + articles: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "infoq-cn", + label: "InfoQ 中国", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }), + ), + fetchGiteeData().catch( + (): GiteeData => ({ + projects: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "gitee", + label: "Gitee", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }), + ), + fetchOschinaData().catch( + (): OschinaData => ({ + news: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "oschina", + label: "开源中国", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }), + ), + fetchJuejinData().catch( + (): JuejinData => ({ + articles: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "juejin", + label: "掘金", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }), + ), ]); return { kr36, infoqCn, gitee, oschina, juejin }; diff --git a/src/generate-manifest.ts b/src/generate-manifest.ts index 27ee137..2adaf86 100644 --- a/src/generate-manifest.ts +++ b/src/generate-manifest.ts @@ -186,7 +186,7 @@ async function main(): Promise { generateSearchIndex(entries); } -function generateSearchIndex(entries: DateEntry[]): void { +export function generateSearchIndex(entries: DateEntry[]): void { interface SearchTopic { date: string; title: string; @@ -206,16 +206,19 @@ function generateSearchIndex(entries: DateEntry[]): void { if (fs.existsSync(poolPath)) { try { const pool = JSON.parse(fs.readFileSync(poolPath, "utf-8")); - if (Array.isArray(pool.topics)) { - for (const t of pool.topics) { - topics.push({ - date, - title: t.topic ?? t.title ?? "", - score: t.score ?? 0, - category: t.category ?? "", - source: (t.evidence?.[0] ?? "").slice(0, 80), - }); - } + const poolTopics = Array.isArray(pool.candidates) + ? pool.candidates + : Array.isArray(pool.topics) + ? pool.topics + : []; + for (const t of poolTopics) { + topics.push({ + date, + title: t.topic ?? t.title ?? "", + score: t.score ?? 0, + category: t.category ?? "", + source: (t.source ?? t.evidence?.[0] ?? "").slice(0, 80), + }); } } catch { // skip corrupt pool files diff --git a/src/gitee.ts b/src/gitee.ts index 1005113..f597f60 100644 --- a/src/gitee.ts +++ b/src/gitee.ts @@ -2,6 +2,8 @@ * Gitee popular AI projects fetched via REST API v5. */ +import { createSourceStatus, type SourceStatus } from "./source-status.ts"; + // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- @@ -22,6 +24,7 @@ export interface GiteeProject { export interface GiteeData { projects: GiteeProject[]; fetchSuccess: boolean; + status: SourceStatus; } // --------------------------------------------------------------------------- @@ -29,38 +32,8 @@ export interface GiteeData { // --------------------------------------------------------------------------- const MAX_PROJECTS = 30; -const API_URL = "https://gitee.com/api/v5/projects"; - -/** AI-related keywords for filtering project names/descriptions. */ -const AI_KEYWORDS = [ - "ai", - "人工智能", - "大模型", - "llm", - "gpt", - "chatgpt", - "机器学习", - "深度学习", - "deep-learning", - "machine-learning", - "aigc", - "rag", - "agent", - "langchain", - "transformer", - "diffusion", - "nlp", - "自然语言处理", - "computer-vision", - "计算机视觉", - "向量", - "vector", - "embedding", - "神经网络", - "neural", - "pytorch", - "tensorflow", -]; +const API_URL = "https://gitee.com/api/v5/search/repositories"; +const SEARCH_KEYWORDS = ["ai", "llm", "大模型", "agent", "rag"]; // --------------------------------------------------------------------------- // API types @@ -83,9 +56,19 @@ interface GiteeApiProject { // Helpers // --------------------------------------------------------------------------- -function isAiRelated(name: string, description: string): boolean { - const text = `${name} ${description}`.toLowerCase(); - return AI_KEYWORDS.some((kw) => text.includes(kw)); +function safeError(err: unknown): string { + return String(err).replace(/access_token=[^&\s]+/g, "access_token=REDACTED"); +} + +function result(projects: GiteeProject[], fetchedCount: number, error?: string): GiteeData { + const status = createSourceStatus({ + id: "gitee", + label: "Gitee", + fetchedCount, + acceptedCount: projects.length, + error, + }); + return { projects, fetchSuccess: status.state !== "error", status }; } // --------------------------------------------------------------------------- @@ -94,63 +77,71 @@ function isAiRelated(name: string, description: string): boolean { export async function fetchGiteeData(): Promise { const seen = new Map(); - - try { - const token = process.env["GITEE_TOKEN"]; - const params = new URLSearchParams({ - sort: "stars", - per_page: "100", - order: "desc", - page: "1", - }); - if (token) params.set("access_token", token); - - const headers: Record = { - "User-Agent": "ai-topic-radar/1.0", - Accept: "application/json", - }; - - const resp = await fetch(`${API_URL}?${params}`, { headers }); - - if (!resp.ok) { - console.error(` [gitee] API returned HTTP ${resp.status}`); - return { projects: [], fetchSuccess: false }; - } - - const raw = (await resp.json()) as GiteeApiProject[]; - if (!Array.isArray(raw)) { - console.error(` [gitee] unexpected response shape`); - return { projects: [], fetchSuccess: false }; - } - - for (const p of raw) { - const desc = p.description ?? ""; - if (!isAiRelated(p.name, desc)) continue; - - if (!seen.has(p.id)) { - seen.set(p.id, { - id: p.id, - name: p.name, - fullName: p.full_name, - url: p.html_url, - description: desc, - language: p.language ?? "", - stars: p.stargazers_count, - forks: p.forks_count, - updatedAt: p.updated_at, - namespace: p.namespace.path, - }); + const errors: string[] = []; + let fetchedCount = 0; + + const token = process.env["GITEE_TOKEN"]; + const headers: Record = { + "User-Agent": "ai-topic-radar/1.0", + Accept: "application/json", + }; + + for (const keyword of SEARCH_KEYWORDS) { + try { + const params = new URLSearchParams({ + q: keyword, + sort: "stars_count", + order: "desc", + per_page: "100", + page: "1", + }); + if (token) params.set("access_token", token); + + const resp = await fetch(`${API_URL}?${params}`, { headers }); + + if (!resp.ok) { + console.error(` [gitee] API returned HTTP ${resp.status}`); + errors.push(`${keyword}: HTTP ${resp.status}`); + continue; } - if (seen.size >= MAX_PROJECTS) break; + const raw = (await resp.json()) as GiteeApiProject[]; + if (!Array.isArray(raw)) { + console.error(` [gitee] unexpected response shape`); + errors.push(`${keyword}: unexpected response shape`); + continue; + } + fetchedCount += raw.length; + + for (const p of raw) { + const desc = p.description ?? ""; + if (!seen.has(p.id)) { + seen.set(p.id, { + id: p.id, + name: p.name, + fullName: p.full_name, + url: p.html_url, + description: desc, + language: p.language ?? "", + stars: p.stargazers_count, + forks: p.forks_count, + updatedAt: p.updated_at, + namespace: p.namespace.path, + }); + } + } + } catch (err) { + const safeMsg = safeError(err); + console.error(` [gitee] fetch failed: ${safeMsg}`); + errors.push(`${keyword}: ${safeMsg}`); } - - const projects = [...seen.values()].sort((a, b) => b.stars - a.stars).slice(0, MAX_PROJECTS); - console.log(` [gitee] ${projects.length} AI projects (from ${raw.length} total)`); - return { projects, fetchSuccess: projects.length > 0 }; - } catch (err) { - const safeMsg = String(err).replace(/access_token=[^&\s]+/g, "access_token=REDACTED"); - console.error(` [gitee] fetch failed: ${safeMsg}`); - return { projects: [], fetchSuccess: false }; } + + const projects = [...seen.values()].sort((a, b) => b.stars - a.stars).slice(0, MAX_PROJECTS); + console.log(` [gitee] ${projects.length} AI projects (from ${fetchedCount} results)`); + return result( + projects, + fetchedCount, + errors.length > 0 ? `partial failure: ${errors.join("; ")}` : undefined, + ); } diff --git a/src/hf.ts b/src/hf.ts index 0e41ff7..f709d30 100644 --- a/src/hf.ts +++ b/src/hf.ts @@ -5,6 +5,8 @@ * HF Hub API, returning a mapped subset of fields. */ +import { createSourceStatus, type SourceStatus } from "./source-status.ts"; + // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- @@ -23,6 +25,7 @@ export interface HfModel { export interface HfData { models: HfModel[]; fetchSuccess: boolean; + status: SourceStatus; } // --------------------------------------------------------------------------- @@ -47,6 +50,17 @@ interface HfApiModel { lastModified?: string; } +function result(models: HfModel[], fetchedCount: number, error?: string): HfData { + const status = createSourceStatus({ + id: "hf", + label: "Hugging Face", + fetchedCount, + acceptedCount: models.length, + error, + }); + return { models, fetchSuccess: status.state !== "error", status }; +} + // --------------------------------------------------------------------------- // Fetch // --------------------------------------------------------------------------- @@ -67,10 +81,14 @@ export async function fetchHfData(): Promise { if (!resp.ok) { console.error(` [hf] HTTP ${resp.status}`); - return { models: [], fetchSuccess: false }; + return result([], 0, `HTTP ${resp.status}`); } const raw = (await resp.json()) as HfApiModel[]; + if (!Array.isArray(raw)) { + console.error(` [hf] unexpected response shape`); + return result([], 0, "unexpected response shape"); + } const models: HfModel[] = raw.map((m) => ({ id: m.id, @@ -84,9 +102,9 @@ export async function fetchHfData(): Promise { })); console.log(` [hf] ${models.length} trending models`); - return { models, fetchSuccess: models.length > 0 }; + return result(models, raw.length); } catch (err) { console.error(` [hf] fetch failed: ${err}`); - return { models: [], fetchSuccess: false }; + return result([], 0, String(err)); } } diff --git a/src/hn.ts b/src/hn.ts index 98abbb9..70a06a8 100644 --- a/src/hn.ts +++ b/src/hn.ts @@ -2,6 +2,8 @@ * Hacker News AI stories fetched via the Algolia HN Search API. */ +import { createSourceStatus, type SourceStatus } from "./source-status.ts"; + // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- @@ -20,6 +22,7 @@ export interface HnStory { export interface HnData { stories: HnStory[]; fetchSuccess: boolean; + status: SourceStatus; } // --------------------------------------------------------------------------- @@ -49,6 +52,17 @@ interface AlgoliaResponse { hits: AlgoliaHit[]; } +function result(stories: HnStory[], fetchedCount: number, errors: string[] = []): HnData { + const status = createSourceStatus({ + id: "hn", + label: "Hacker News", + fetchedCount, + acceptedCount: stories.length, + error: errors.length > 0 ? `partial failure: ${errors.join("; ")}` : undefined, + }); + return { stories, fetchSuccess: status.state !== "error", status }; +} + // --------------------------------------------------------------------------- // Fetch // --------------------------------------------------------------------------- @@ -56,6 +70,8 @@ interface AlgoliaResponse { export async function fetchHnData(): Promise { const since = Math.floor((Date.now() - 24 * 60 * 60 * 1000) / 1000); const seen = new Map(); + const errors: string[] = []; + let fetchedCount = 0; try { await Promise.all( @@ -72,10 +88,17 @@ export async function fetchHnData(): Promise { }); if (!resp.ok) { console.error(` [hn] "${q}": HTTP ${resp.status}`); + errors.push(`"${q}": HTTP ${resp.status}`); return; } const data = (await resp.json()) as AlgoliaResponse; - for (const hit of data.hits ?? []) { + if (!data || !Array.isArray(data.hits)) { + console.error(` [hn] "${q}": unexpected response shape`); + errors.push(`"${q}": unexpected response shape`); + return; + } + fetchedCount += data.hits.length; + for (const hit of data.hits) { if (!seen.has(hit.objectID)) { const hnUrl = `https://news.ycombinator.com/item?id=${hit.objectID}`; seen.set(hit.objectID, { @@ -92,6 +115,7 @@ export async function fetchHnData(): Promise { } } catch (err) { console.error(` [hn] "${q}": ${err}`); + errors.push(`"${q}": ${err}`); } }), ); @@ -99,9 +123,9 @@ export async function fetchHnData(): Promise { const stories = [...seen.values()].sort((a, b) => b.points - a.points).slice(0, HN_TOP_STORIES); console.log(` [hn] ${stories.length} stories (from ${seen.size} unique)`); - return { stories, fetchSuccess: stories.length > 0 }; + return result(stories, fetchedCount, errors); } catch (err) { console.error(` [hn] fetch failed: ${err}`); - return { stories: [], fetchSuccess: false }; + return result([], fetchedCount, [String(err)]); } } diff --git a/src/index.ts b/src/index.ts index aeb8bdc..99baac0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -55,6 +55,7 @@ import { toCstDateStr, toUtcStr } from "./date.ts"; import { type Lang, MSG, ISSUE_LABELS, CLI_ISSUE_TITLE, OPENCLAW_ISSUE_TITLE } from "./i18n.ts"; import { buildTopicRadar, saveTopicRadar } from "./topic-radar.ts"; import { getReportLangs, shouldSaveSourceReports } from "./options.ts"; +import { createSourceStatus } from "./source-status.ts"; // --------------------------------------------------------------------------- // Repo config — loaded from config.yml, falls back to built-in defaults @@ -142,11 +143,31 @@ async function fetchAllData( isFirstRun: false, newItems: [], totalDiscovered: 0, + status: createSourceStatus({ + id: "web-anthropic", + label: "Anthropic (Claude)", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), }; }), fetchSiteContent("openai", webState).catch((err): WebFetchResult => { console.error(` [web/openai] fetch failed: ${err}`); - return { site: "openai", siteName: "OpenAI", isFirstRun: false, newItems: [], totalDiscovered: 0 }; + return { + site: "openai", + siteName: "OpenAI", + isFirstRun: false, + newItems: [], + totalDiscovered: 0, + status: createSourceStatus({ + id: "web-openai", + label: "OpenAI", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }; }), fetchSiteContent("deepmind", webState).catch((err): WebFetchResult => { console.error(` [web/deepmind] fetch failed: ${err}`); @@ -156,6 +177,13 @@ async function fetchAllData( isFirstRun: false, newItems: [], totalDiscovered: 0, + status: createSourceStatus({ + id: "web-deepmind", + label: "Google DeepMind", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), }; }), ]), @@ -164,21 +192,126 @@ async function fetchAllData( trendingRepos: [], searchRepos: [], trendingFetchSuccess: false, + status: createSourceStatus({ + id: "github-trending", + label: "GitHub Trending HTML", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }), + ), + fetchHnData().catch( + (): HnData => ({ + stories: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "hn", + label: "Hacker News", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }), + ), + fetchPhData().catch( + (): PhData => ({ + products: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "product-hunt", + label: "Product Hunt", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }), + ), + fetchArxivData().catch( + (): ArxivData => ({ + papers: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "arxiv", + label: "ArXiv", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }), + ), + fetchHfData().catch( + (): HfData => ({ + models: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "hf", + label: "Hugging Face", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), }), ), - fetchHnData().catch((): HnData => ({ stories: [], fetchSuccess: false })), - fetchPhData().catch((): PhData => ({ products: [], fetchSuccess: false })), - fetchArxivData().catch((): ArxivData => ({ papers: [], fetchSuccess: false })), - fetchHfData().catch((): HfData => ({ models: [], fetchSuccess: false })), fetchDevtoData().catch((): DevtoData => ({ articles: [], fetchSuccess: false })), fetchLobstersData().catch((): LobstersData => ({ stories: [], fetchSuccess: false })), fetchChinaSourcesData().catch( (): ChinaSourcesData => ({ - kr36: { articles: [], fetchSuccess: false }, - infoqCn: { articles: [], fetchSuccess: false }, - gitee: { projects: [], fetchSuccess: false }, - oschina: { news: [], fetchSuccess: false }, - juejin: { articles: [], fetchSuccess: false }, + kr36: { + articles: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "kr36", + label: "36kr", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }, + infoqCn: { + articles: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "infoq-cn", + label: "InfoQ 中国", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }, + gitee: { + projects: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "gitee", + label: "Gitee", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }, + oschina: { + news: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "oschina", + label: "开源中国", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }, + juejin: { + articles: [], + fetchSuccess: false, + status: createSourceStatus({ + id: "juejin", + label: "掘金", + fetchedCount: 0, + acceptedCount: 0, + error: "fetch failed", + }), + }, }), ), ]); @@ -449,6 +582,8 @@ async function main(): Promise { phData, arxivData, hfData, + devtoData, + lobstersData, webResults, chinaSourcesData, dateStr, diff --git a/src/infoq-cn.ts b/src/infoq-cn.ts index 72f9f95..e500351 100644 --- a/src/infoq-cn.ts +++ b/src/infoq-cn.ts @@ -3,6 +3,7 @@ */ import { FETCH_TIMEOUT_MS } from "./rss-utils.ts"; +import { createSourceStatus, type SourceStatus } from "./source-status.ts"; // --------------------------------------------------------------------------- // Types @@ -21,6 +22,7 @@ export interface InfoqCnArticle { export interface InfoqCnData { articles: InfoqCnArticle[]; fetchSuccess: boolean; + status: SourceStatus; } // --------------------------------------------------------------------------- @@ -54,16 +56,20 @@ const AI_KEYWORDS = [ interface InfoqArticleItem { id: number; uuid: string; - title: string; - url: string; - description: string; - author: { name: string; nickname: string } | null; + title?: string; + article_title?: string; + url?: string; + description?: string; + article_summary?: string; + author?: { name?: string; nickname?: string } | Array<{ name?: string; nickname?: string }> | null; publish_time: number; - topics: string[]; + topics?: string[]; + topic?: Array<{ name?: string }>; } interface InfoqApiResponse { data: InfoqArticleItem[]; + code?: number; err_msg?: string; } @@ -76,6 +82,17 @@ function isAiRelated(title: string, description: string, topics: string[]): bool return AI_KEYWORDS.some((kw) => text.includes(kw.toLowerCase())); } +function result(articles: InfoqCnArticle[], fetchedCount: number, error?: string): InfoqCnData { + const status = createSourceStatus({ + id: "infoq-cn", + label: "InfoQ 中国", + fetchedCount, + acceptedCount: articles.length, + error, + }); + return { articles, fetchSuccess: status.state !== "error", status }; +} + // --------------------------------------------------------------------------- // Fetch // --------------------------------------------------------------------------- @@ -100,23 +117,28 @@ export async function fetchInfoqCnData(): Promise { if (!resp.ok) { console.error(` [infoq-cn] API returned HTTP ${resp.status}`); - return { articles: [], fetchSuccess: false }; + return result([], 0, `HTTP ${resp.status}`); } const raw = (await resp.json()) as InfoqApiResponse; if (!raw || !Array.isArray(raw.data)) { console.error(` [infoq-cn] unexpected response shape`); - return { articles: [], fetchSuccess: false }; + return result([], 0, "unexpected response shape"); } - if (raw.err_msg) { - console.error(` [infoq-cn] API error: ${raw.err_msg}`); - return { articles: [], fetchSuccess: false }; + if ((raw.code !== undefined && raw.code !== 0) || raw.err_msg) { + const error = raw.err_msg || `API code ${raw.code}`; + console.error(` [infoq-cn] API error: ${error}`); + return result([], raw.data.length, error); } for (const item of raw.data ?? []) { - const title = item.title; - const description = item.description ?? ""; - const topics = item.topics ?? []; + const title = item.article_title ?? item.title ?? ""; + const description = item.article_summary ?? item.description ?? ""; + const topics = + item.topic?.map(({ name }) => name).filter((name): name is string => Boolean(name)) ?? + item.topics ?? + []; + const author = Array.isArray(item.author) ? item.author[0] : item.author; if (!isAiRelated(title, description, topics)) continue; @@ -129,7 +151,7 @@ export async function fetchInfoqCnData(): Promise { title, url, summary: description.slice(0, 500), - author: item.author?.nickname || item.author?.name || "InfoQ", + author: author?.nickname || author?.name || "InfoQ", publishTime: item.publish_time ? new Date(item.publish_time * 1000).toISOString() : new Date().toISOString(), @@ -142,9 +164,9 @@ export async function fetchInfoqCnData(): Promise { const articles = [...seen.values()].slice(0, MAX_ARTICLES); console.log(` [infoq-cn] ${articles.length} AI articles (from ${seen.size} unique)`); - return { articles, fetchSuccess: articles.length > 0 }; + return result(articles, raw.data.length); } catch (err) { console.error(` [infoq-cn] fetch failed: ${err}`); - return { articles: [], fetchSuccess: false }; + return result([], 0, String(err)); } } diff --git a/src/juejin.ts b/src/juejin.ts index 0917ecd..fe53c90 100644 --- a/src/juejin.ts +++ b/src/juejin.ts @@ -3,6 +3,7 @@ */ import { FETCH_TIMEOUT_MS } from "./rss-utils.ts"; +import { createSourceStatus, type SourceStatus } from "./source-status.ts"; // --------------------------------------------------------------------------- // Types @@ -23,6 +24,7 @@ export interface JuejinArticle { export interface JuejinData { articles: JuejinArticle[]; fetchSuccess: boolean; + status: SourceStatus; } // --------------------------------------------------------------------------- @@ -55,9 +57,21 @@ interface JuejinArticleInfo { interface JuejinApiResponse { data: JuejinArticleInfo[]; + err_no: number; err_msg?: string; } +function result(articles: JuejinArticle[], fetchedCount: number, error?: string): JuejinData { + const status = createSourceStatus({ + id: "juejin", + label: "掘金", + fetchedCount, + acceptedCount: articles.length, + error, + }); + return { articles, fetchSuccess: status.state !== "error", status }; +} + // --------------------------------------------------------------------------- // Fetch // --------------------------------------------------------------------------- @@ -87,17 +101,18 @@ export async function fetchJuejinData(): Promise { if (!resp.ok) { console.error(` [juejin] API returned HTTP ${resp.status}`); - return { articles: [], fetchSuccess: false }; + return result([], 0, `HTTP ${resp.status}`); } const raw = (await resp.json()) as JuejinApiResponse; if (!raw || !Array.isArray(raw.data)) { console.error(` [juejin] unexpected response shape`); - return { articles: [], fetchSuccess: false }; + return result([], 0, "unexpected response shape"); } - if (raw.err_msg) { - console.error(` [juejin] API error: ${raw.err_msg}`); - return { articles: [], fetchSuccess: false }; + if (raw.err_no !== 0) { + const error = raw.err_msg || `API err_no ${raw.err_no}`; + console.error(` [juejin] API error: ${error}`); + return result([], 0, error); } for (const item of raw.data ?? []) { @@ -124,9 +139,9 @@ export async function fetchJuejinData(): Promise { const articles = [...seen.values()].sort((a, b) => b.diggCount - a.diggCount).slice(0, MAX_ARTICLES); console.log(` [juejin] ${articles.length} AI articles`); - return { articles, fetchSuccess: articles.length > 0 }; + return result(articles, raw.data.length); } catch (err) { console.error(` [juejin] fetch failed: ${err}`); - return { articles: [], fetchSuccess: false }; + return result([], 0, String(err)); } } diff --git a/src/kr36.ts b/src/kr36.ts index 53c7252..5c2d427 100644 --- a/src/kr36.ts +++ b/src/kr36.ts @@ -3,6 +3,7 @@ */ import { extractTag, extractCdata, stripHtml, FETCH_TIMEOUT_MS } from "./rss-utils.ts"; +import { createSourceStatus, type SourceStatus } from "./source-status.ts"; // --------------------------------------------------------------------------- // Types @@ -20,6 +21,7 @@ export interface Kr36Article { export interface Kr36Data { articles: Kr36Article[]; fetchSuccess: boolean; + status: SourceStatus; } // --------------------------------------------------------------------------- @@ -61,6 +63,21 @@ function isAiRelated(title: string, summary: string): boolean { return AI_KEYWORDS.some((kw) => text.includes(kw.toLowerCase())); } +function result(articles: Kr36Article[], fetchedCount: number, error?: string): Kr36Data { + const status = createSourceStatus({ + id: "kr36", + label: "36kr", + fetchedCount, + acceptedCount: articles.length, + error, + }); + return { articles, fetchSuccess: status.state !== "error", status }; +} + +function isRssFeed(xml: string): boolean { + return /<(rss|feed)[\s>]/i.test(xml); +} + // --------------------------------------------------------------------------- // Fetch via RSS // --------------------------------------------------------------------------- @@ -81,10 +98,14 @@ export async function fetchKr36Data(): Promise { const cl = resp.headers.get("content-length"); if (cl && Number(cl) > 5 * 1024 * 1024) { console.error(` [kr36] RSS response too large: ${cl} bytes`); - return { articles: [], fetchSuccess: false }; + return result([], 0, `RSS response too large: ${cl} bytes`); } const xml = await resp.text(); + if (!isRssFeed(xml)) { + console.error(` [kr36] unexpected RSS response shape`); + return result([], 0, "unexpected RSS response shape"); + } const itemBlocks = xml.split("").slice(1); for (const block of itemBlocks) { @@ -112,13 +133,13 @@ export async function fetchKr36Data(): Promise { } console.log(` [kr36] ${articles.length} AI articles from RSS`); - return { articles, fetchSuccess: articles.length > 0 }; + return result(articles, itemBlocks.length); } console.log(` [kr36] RSS returned ${resp.status}, no fallback available`); + return result([], 0, `HTTP ${resp.status}`); } catch (err) { console.error(` [kr36] RSS fetch failed: ${err}`); + return result([], 0, String(err)); } - - return { articles: [], fetchSuccess: false }; } diff --git a/src/oschina.ts b/src/oschina.ts index 99eaef8..f8ad125 100644 --- a/src/oschina.ts +++ b/src/oschina.ts @@ -3,6 +3,7 @@ */ import { extractTag, extractCdata, stripHtml, FETCH_TIMEOUT_MS } from "./rss-utils.ts"; +import { createSourceStatus, type SourceStatus } from "./source-status.ts"; // --------------------------------------------------------------------------- // Types @@ -20,6 +21,7 @@ export interface OschinaNews { export interface OschinaData { news: OschinaNews[]; fetchSuccess: boolean; + status: SourceStatus; } // --------------------------------------------------------------------------- @@ -61,6 +63,21 @@ function isAiRelated(title: string, body: string): boolean { return AI_KEYWORDS.some((kw) => text.includes(kw.toLowerCase())); } +function result(news: OschinaNews[], fetchedCount: number, error?: string): OschinaData { + const status = createSourceStatus({ + id: "oschina", + label: "开源中国", + fetchedCount, + acceptedCount: news.length, + error, + }); + return { news, fetchSuccess: status.state !== "error", status }; +} + +function isRssFeed(xml: string): boolean { + return /<(rss|feed)[\s>]/i.test(xml); +} + // --------------------------------------------------------------------------- // Fetch // --------------------------------------------------------------------------- @@ -79,16 +96,20 @@ export async function fetchOschinaData(): Promise { if (!resp.ok) { console.error(` [oschina] RSS returned HTTP ${resp.status}`); - return { news: [], fetchSuccess: false }; + return result([], 0, `HTTP ${resp.status}`); } const cl = resp.headers.get("content-length"); if (cl && Number(cl) > 5 * 1024 * 1024) { console.error(` [oschina] RSS response too large: ${cl} bytes`); - return { news: [], fetchSuccess: false }; + return result([], 0, `RSS response too large: ${cl} bytes`); } const xml = await resp.text(); + if (!isRssFeed(xml)) { + console.error(` [oschina] unexpected RSS response shape`); + return result([], 0, "unexpected RSS response shape"); + } const itemBlocks = xml.split("").slice(1); for (const block of itemBlocks) { @@ -116,9 +137,9 @@ export async function fetchOschinaData(): Promise { } console.log(` [oschina] ${articles.length} AI news articles`); - return { news: articles, fetchSuccess: articles.length > 0 }; + return result(articles, itemBlocks.length); } catch (err) { console.error(` [oschina] fetch failed: ${err}`); - return { news: [], fetchSuccess: false }; + return result([], 0, String(err)); } } diff --git a/src/ph.ts b/src/ph.ts index 44934f4..adc5dfc 100644 --- a/src/ph.ts +++ b/src/ph.ts @@ -5,6 +5,8 @@ * then filter locally for AI-related topics. */ +import { createSourceStatus, type SourceStatus } from "./source-status.ts"; + // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- @@ -24,6 +26,7 @@ export interface PhProduct { export interface PhData { products: PhProduct[]; fetchSuccess: boolean; + status: SourceStatus; } // --------------------------------------------------------------------------- @@ -100,6 +103,18 @@ interface PhResponse { errors?: Array<{ message: string }>; } +function result(products: PhProduct[], fetchedCount: number, error?: string, skipped?: string): PhData { + const status = createSourceStatus({ + id: "product-hunt", + label: "Product Hunt", + fetchedCount, + acceptedCount: products.length, + error, + skipped, + }); + return { products, fetchSuccess: status.state !== "error" && status.state !== "skipped", status }; +} + // --------------------------------------------------------------------------- // Fetch // --------------------------------------------------------------------------- @@ -108,7 +123,7 @@ export async function fetchPhData(): Promise { const token = process.env["PRODUCTHUNT_TOKEN"] ?? ""; if (!token) { console.log(" [ph] PRODUCTHUNT_TOKEN not set — skipping."); - return { products: [], fetchSuccess: false }; + return result([], 0, undefined, "PRODUCTHUNT_TOKEN not set"); } // Fetch yesterday's products (they've had a full day to accumulate votes) @@ -136,18 +151,24 @@ export async function fetchPhData(): Promise { if (!resp.ok) { console.error(` [ph] HTTP ${resp.status}`); - return { products: [], fetchSuccess: false }; + return result([], 0, `HTTP ${resp.status}`); } const json = (await resp.json()) as PhResponse; if (json.errors?.length) { console.error(` [ph] API errors: ${json.errors.map((e) => e.message).join("; ")}`); - return { products: [], fetchSuccess: false }; + return result([], 0, json.errors.map((e) => e.message).join("; ")); + } + + const edges = json.data?.posts?.edges; + if (!Array.isArray(edges)) { + console.error(` [ph] unexpected response shape`); + return result([], 0, "unexpected response shape"); } const allProducts: PhProduct[] = []; - for (const edge of json.data?.posts?.edges ?? []) { + for (const edge of edges) { const node = edge.node; const topicSlugs = node.topics?.edges?.map((e) => e.node.slug) ?? []; const topicNames = node.topics?.edges?.map((e) => e.node.name) ?? []; @@ -171,10 +192,10 @@ export async function fetchPhData(): Promise { const products = allProducts.sort((a, b) => b.votesCount - a.votesCount).slice(0, PH_TOP_PRODUCTS); - console.log(` [ph] ${products.length} AI products (from ${json.data?.posts?.edges?.length ?? 0} total)`); - return { products, fetchSuccess: products.length > 0 }; + console.log(` [ph] ${products.length} AI products (from ${edges.length} total)`); + return result(products, edges.length); } catch (err) { console.error(` [ph] fetch failed: ${err}`); - return { products: [], fetchSuccess: false }; + return result([], 0, String(err)); } } diff --git a/src/report-savers.ts b/src/report-savers.ts index 3790f41..c78c5ed 100644 --- a/src/report-savers.ts +++ b/src/report-savers.ts @@ -153,7 +153,7 @@ export async function saveHnReport( footer: string, lang: Lang = "zh", ): Promise { - if (!hnData.fetchSuccess) { + if (hnData.stories.length === 0) { console.log(` [hn/${lang}] No data available, skipping report.`); return; } @@ -200,7 +200,7 @@ export async function savePhReport( footer: string, lang: Lang = "zh", ): Promise { - if (!phData.fetchSuccess) { + if (phData.products.length === 0) { console.log(` [ph/${lang}] No data available, skipping report.`); return; } @@ -247,7 +247,7 @@ export async function saveArxivReport( footer: string, lang: Lang = "zh", ): Promise { - if (!arxivData.fetchSuccess) { + if (arxivData.papers.length === 0) { console.log(` [arxiv/${lang}] No data available, skipping report.`); return; } @@ -294,7 +294,7 @@ export async function saveHfReport( footer: string, lang: Lang = "zh", ): Promise { - if (!hfData.fetchSuccess) { + if (hfData.models.length === 0) { console.log(` [hf/${lang}] No data available, skipping report.`); return; } diff --git a/src/source-status.ts b/src/source-status.ts new file mode 100644 index 0000000..0a0613c --- /dev/null +++ b/src/source-status.ts @@ -0,0 +1,36 @@ +export type SourceState = "ok" | "empty" | "error" | "skipped"; + +export interface SourceStatus { + id: string; + label: string; + state: SourceState; + fetchedCount: number; + acceptedCount: number; + detail?: string; +} + +interface CreateSourceStatusInput { + id: string; + label: string; + fetchedCount: number; + acceptedCount: number; + error?: string; + skipped?: string; +} + +export function createSourceStatus(input: CreateSourceStatusInput): SourceStatus { + const { id, label, fetchedCount, acceptedCount } = input; + if (input.error) { + return { id, label, state: "error", fetchedCount, acceptedCount, detail: input.error }; + } + if (input.skipped) { + return { id, label, state: "skipped", fetchedCount, acceptedCount, detail: input.skipped }; + } + return { + id, + label, + state: acceptedCount > 0 ? "ok" : "empty", + fetchedCount, + acceptedCount, + }; +} diff --git a/src/topic-radar.ts b/src/topic-radar.ts index f88a9b4..9c5ed2b 100644 --- a/src/topic-radar.ts +++ b/src/topic-radar.ts @@ -12,6 +12,9 @@ import type { ArxivData } from "./arxiv.ts"; import type { HfData } from "./hf.ts"; import type { WebFetchResult } from "./web.ts"; import type { ChinaSourcesData } from "./china-sources.ts"; +import type { SourceStatus } from "./source-status.ts"; +import type { DevtoData } from "./devto.ts"; +import type { LobstersData } from "./lobsters.ts"; import fs from "node:fs"; import path from "node:path"; @@ -66,6 +69,8 @@ export interface TopicRadarInput { phData: PhData; arxivData: ArxivData; hfData: HfData; + devtoData?: DevtoData; + lobstersData?: LobstersData; webResults: WebFetchResult[]; chinaSourcesData?: ChinaSourcesData; dateStr: string; @@ -77,6 +82,8 @@ export interface TopicRadarResult { generatedAt: string; date: string; candidates: TopicCandidate[]; + sourceStatuses: SourceStatus[]; + notices: string[]; warnings: string[]; } @@ -168,7 +175,7 @@ function heatScore(topic: RawTopic): number { case "community": return clamp(strongestSignal / 12, 30); case "github": - return clamp(strongestSignal / 15, 30); + return clamp(Math.log10(strongestSignal + 1) * 5, 24); case "model": return clamp(strongestSignal / 20, 30); case "official": @@ -311,6 +318,75 @@ function scoreTopic(topic: RawTopic, now: Date): TopicCandidate { }; } +function candidateKey(candidate: TopicCandidate): string { + return candidate.url || `${candidate.source}:${candidate.title}`; +} + +function dedupeCandidates(candidates: TopicCandidate[]): TopicCandidate[] { + const byUrl = new Map(); + for (const candidate of candidates) { + const key = candidateKey(candidate); + const existing = byUrl.get(key); + if (!existing || candidate.score > existing.score) byUrl.set(key, candidate); + } + return [...byUrl.values()]; +} + +type SourceGroup = "repo" | "domestic" | "research" | "product" | "official" | "community"; + +function sourceGroup(candidate: TopicCandidate): SourceGroup { + if ( + candidate.source === "GitHub Trending" || + candidate.source.startsWith("GitHub Search") || + candidate.source === "Gitee" + ) { + return "repo"; + } + if (["36kr", "InfoQ 中国", "开源中国", "掘金"].includes(candidate.source)) return "domestic"; + if (candidate.source === "ArXiv" || candidate.source === "Hugging Face") return "research"; + if (candidate.source === "Product Hunt") return "product"; + if (["OpenAI", "Anthropic (Claude)", "Google DeepMind"].includes(candidate.source)) return "official"; + return "community"; +} + +function selectCandidates(scored: TopicCandidate[]): TopicCandidate[] { + const eligible = dedupeCandidates(scored) + .filter((candidate) => candidate.score >= 50) + .sort((a, b) => b.score - a.score); + const selected: TopicCandidate[] = []; + const used = new Set(); + const counts = new Map(); + const groups: Array<{ id: SourceGroup; base: number; max: number }> = [ + { id: "repo", base: 15, max: 20 }, + { id: "domestic", base: 8, max: 12 }, + { id: "research", base: 8, max: 10 }, + { id: "product", base: 6, max: 10 }, + { id: "official", base: 5, max: 8 }, + { id: "community", base: 5, max: 8 }, + ]; + const add = (candidate: TopicCandidate): void => { + if (selected.length >= 60) return; + const key = candidateKey(candidate); + if (used.has(key)) return; + const group = sourceGroup(candidate); + const max = groups.find((item) => item.id === group)?.max ?? 60; + if ((counts.get(group) ?? 0) >= max) return; + used.add(key); + counts.set(group, (counts.get(group) ?? 0) + 1); + selected.push(candidate); + }; + + for (const group of groups) { + for (const candidate of eligible) { + if (sourceGroup(candidate) !== group.id) continue; + if ((counts.get(group.id) ?? 0) >= group.base) break; + add(candidate); + } + } + for (const candidate of eligible) add(candidate); + return selected; +} + function collectRawTopics(input: TopicRadarInput): RawTopic[] { const topics: RawTopic[] = []; @@ -385,6 +461,30 @@ function collectRawTopics(input: TopicRadarInput): RawTopic[] { tags: [model.pipelineTag, ...model.tags].filter(Boolean), }); } + for (const article of input.devtoData?.articles ?? []) { + topics.push({ + title: article.title, + url: article.url, + source: "Dev.to", + sourceType: "community", + summary: article.description, + publishedAt: article.publishedAt, + heatSignals: [article.positiveReactionsCount, article.commentsCount], + tags: ["devto", ...article.tags], + }); + } + for (const story of input.lobstersData?.stories ?? []) { + topics.push({ + title: story.title, + url: story.url, + source: "Lobste.rs", + sourceType: "community", + summary: `Comments: ${story.commentCount} by ${story.author}`, + publishedAt: story.publishedAt, + heatSignals: [story.score, story.commentCount], + tags: ["lobsters", ...story.tags], + }); + } for (const result of input.webResults) { for (const item of result.newItems) { topics.push({ @@ -468,49 +568,157 @@ function collectRawTopics(input: TopicRadarInput): RawTopic[] { return topics; } -function collectWarnings(input: TopicRadarInput): string[] { - const warnings: string[] = []; - if (!input.trendingData.trendingFetchSuccess) { - warnings.push( - "GitHub Trending HTML 获取失败；可检查 GitHub 页面结构或网络环境，GitHub Search 结果仍可使用。", - ); +function appendSourceStatus( + notices: string[], + warnings: string[], + status: SourceStatus | undefined, + fetchSuccess: boolean, + emptyNotice: string, + errorWarning: string, +): void { + if (!status) { + if (!fetchSuccess) warnings.push(errorWarning); + return; } - if (!input.hnData.fetchSuccess) warnings.push("Hacker News 获取失败；可检查 hn.algolia.com 是否可访问。"); - if (!input.phData.fetchSuccess) { - const hint = process.env["PRODUCTHUNT_TOKEN"] - ? "Product Hunt 无可用数据；可检查 GraphQL API 响应和 AI topic 过滤条件。" - : "Product Hunt 已跳过；配置 PRODUCTHUNT_TOKEN 后可启用产品榜单信号。"; - warnings.push(hint); + if (status.state === "empty") notices.push(emptyNotice); + if (status.state === "error") warnings.push(errorWarning); + if (status.state === "skipped") { + notices.push(`${status.label} 已跳过${status.detail ? `；${status.detail}` : "。"}`); } - if (!input.arxivData.fetchSuccess) warnings.push("ArXiv 获取失败；可检查 export.arxiv.org 网络或重试。"); - if (!input.hfData.fetchSuccess) - warnings.push("Hugging Face 获取失败；可检查 huggingface.co API 是否可访问。"); - if (!input.webResults.some((result) => result.newItems.length > 0)) { - warnings.push("官方内容源今日没有检测到新内容；首次运行后这是正常情况。"); +} + +function collectStatusMessages(input: TopicRadarInput): Pick { + const notices: string[] = []; + const warnings: string[] = []; + appendSourceStatus( + notices, + warnings, + input.trendingData.status, + input.trendingData.trendingFetchSuccess, + "GitHub Trending HTML 暂无条目；抓取成功。", + "GitHub Trending HTML 获取失败；可检查 GitHub 页面结构或网络环境，GitHub Search 结果仍可使用。", + ); + appendSourceStatus( + notices, + warnings, + input.hnData.status, + input.hnData.fetchSuccess, + "Hacker News 暂无符合时间窗口的新内容；抓取成功。", + "Hacker News 获取失败；可检查 hn.algolia.com 是否可访问。", + ); + appendSourceStatus( + notices, + warnings, + input.phData.status, + input.phData.fetchSuccess, + "Product Hunt 暂无符合条件的新产品；抓取成功。", + "Product Hunt 获取失败；可检查 GraphQL API 响应和 AI topic 过滤条件。", + ); + appendSourceStatus( + notices, + warnings, + input.arxivData.status, + input.arxivData.fetchSuccess, + "ArXiv 暂无符合时间窗口的新论文；抓取成功。", + "ArXiv 获取失败；可检查 export.arxiv.org 网络或重试。", + ); + appendSourceStatus( + notices, + warnings, + input.hfData.status, + input.hfData.fetchSuccess, + "Hugging Face 暂无热门模型条目；抓取成功。", + "Hugging Face 获取失败；可检查 huggingface.co API 是否可访问。", + ); + for (const result of input.webResults) { + if (result.status.state === "error") { + warnings.push(`${result.siteName} 官方内容源获取失败；可检查 sitemap 或网络环境。`); + } + } + if ( + input.webResults.every((result) => result.status.state !== "error") && + !input.webResults.some((result) => result.newItems.length > 0) + ) { + notices.push("官方内容源今日没有检测到新内容；首次运行后这是正常情况。"); } if (input.chinaSourcesData) { const cn = input.chinaSourcesData; - if (!cn.kr36.fetchSuccess) warnings.push("36kr 获取失败；可检查网络或 RSS 源是否可用。"); - if (!cn.infoqCn.fetchSuccess) warnings.push("InfoQ 中国获取失败；可检查 infoq.cn API 是否可用。"); - if (!cn.gitee.fetchSuccess) warnings.push("Gitee 获取失败；可检查 gitee.com API 是否可访问。"); - if (!cn.oschina.fetchSuccess) warnings.push("开源中国获取失败；可检查 oschina.net RSS 是否可用。"); - if (!cn.juejin.fetchSuccess) warnings.push("掘金获取失败；可检查 juejin.com API 是否可用。"); + appendSourceStatus( + notices, + warnings, + cn.kr36.status, + cn.kr36.fetchSuccess, + "36kr 暂无符合条件的新内容；抓取成功。", + "36kr 获取失败；可检查网络或 RSS 源是否可用。", + ); + appendSourceStatus( + notices, + warnings, + cn.infoqCn.status, + cn.infoqCn.fetchSuccess, + "InfoQ 中国暂无符合条件的新内容；抓取成功。", + "InfoQ 中国获取失败；可检查 infoq.cn API 是否可用。", + ); + appendSourceStatus( + notices, + warnings, + cn.gitee.status, + cn.gitee.fetchSuccess, + "Gitee 暂无符合条件的新项目；抓取成功。", + "Gitee 获取失败；可检查 gitee.com API 是否可访问。", + ); + appendSourceStatus( + notices, + warnings, + cn.oschina.status, + cn.oschina.fetchSuccess, + "开源中国暂无符合条件的新内容；抓取成功。", + "开源中国获取失败；可检查 oschina.net RSS 是否可用。", + ); + appendSourceStatus( + notices, + warnings, + cn.juejin.status, + cn.juejin.fetchSuccess, + "掘金暂无符合条件的新内容；抓取成功。", + "掘金获取失败；可检查 juejin.com API 是否可用。", + ); + } + return { notices, warnings }; +} + +function collectSourceStatuses(input: TopicRadarInput): SourceStatus[] { + const statuses = [ + input.trendingData.status, + input.hnData.status, + input.phData.status, + input.arxivData.status, + input.hfData.status, + ...input.webResults.map((result) => result.status), + ]; + if (input.chinaSourcesData) { + statuses.push( + input.chinaSourcesData.kr36.status, + input.chinaSourcesData.infoqCn.status, + input.chinaSourcesData.gitee.status, + input.chinaSourcesData.oschina.status, + input.chinaSourcesData.juejin.status, + ); } - return warnings; + return statuses; } export function buildTopicRadar(input: TopicRadarInput): TopicRadarResult { const now = input.now ?? new Date(); - const candidates = collectRawTopics(input) - .map((topic) => scoreTopic(topic, now)) - .sort((a, b) => b.score - a.score) - .slice(0, 60); + const statusMessages = collectStatusMessages(input); + const candidates = selectCandidates(collectRawTopics(input).map((topic) => scoreTopic(topic, now))); return { generatedAt: input.utcStr, date: input.dateStr, candidates, - warnings: collectWarnings(input), + sourceStatuses: collectSourceStatuses(input), + ...statusMessages, }; } @@ -531,6 +739,8 @@ export function buildTopicRadarMarkdown(result: TopicRadarResult): string { const deepDive = result.candidates.filter((item) => item.action === "深挖").slice(0, 10); const pool = result.candidates.filter((item) => item.action === "入池").slice(0, 15); const watch = result.candidates.filter((item) => item.action === "观察").slice(0, 15); + const watchSection = + watch.length === 0 ? "暂无 50–64 分观察项。这不代表数据源采集失败。\n" : tableRows(watch); const categorySections = TOPIC_CATEGORIES.map((category) => { const items = result.candidates.filter((item) => item.category === category).slice(0, 5); @@ -539,8 +749,12 @@ export function buildTopicRadarMarkdown(result: TopicRadarResult): string { const warningSection = result.warnings.length === 0 - ? "暂无失败或跳过的数据源。\n" + ? "暂无需要修复的数据源。\n" : result.warnings.map((warning) => `- ${warning}`).join("\n") + "\n"; + const noticeSection = + result.notices.length === 0 + ? "暂无普通状态提示。\n" + : result.notices.map((notice) => `- ${notice}`).join("\n") + "\n"; return [ `# AI 热点选题池 ${result.date}`, @@ -560,9 +774,12 @@ export function buildTopicRadarMarkdown(result: TopicRadarResult): string { categorySections, "## 观察项", "", - tableRows(watch), + watchSection, "", - "## 数据源状态与修复提示", + "## 数据源普通状态提示", + "", + noticeSection, + "## 数据源修复提示", "", warningSection, ].join("\n"); @@ -618,8 +835,12 @@ export function buildTopicRadarHtml(result: TopicRadarResult): string { const watch = result.candidates.filter((item) => item.action === "观察").slice(0, 15); const warnings = result.warnings.length === 0 - ? `

暂无失败或跳过的数据源。

` + ? `

暂无需要修复的数据源。

` : `
    ${result.warnings.map((warning) => `
  • ${escapeHtml(warning)}
    • `).join("")}
`; + const notices = + result.notices.length === 0 + ? `

暂无普通状态提示。

` + : `
    ${result.notices.map((notice) => `
  • ${escapeHtml(notice)}
    • `).join("")}
`; const categorySections = TOPIC_CATEGORIES.map((category) => { const items = result.candidates.filter((item) => item.category === category).slice(0, 5); @@ -831,11 +1052,16 @@ export function buildTopicRadarHtml(result: TopicRadarResult): string {

观察项

-
${renderTopicCards(watch)}
+
${watch.length === 0 ? '

暂无 50–64 分观察项。这不代表数据源采集失败。

' : renderTopicCards(watch)}
+
+ +
+

数据源普通状态提示

+ ${notices}
-
-

数据源状态与修复提示

+
+

数据源修复提示

${warnings}
diff --git a/src/trending.ts b/src/trending.ts index 21047cc..a9674af 100644 --- a/src/trending.ts +++ b/src/trending.ts @@ -2,6 +2,8 @@ * GitHub trending and AI topic search data fetching. */ +import { createSourceStatus, type SourceStatus } from "./source-status.ts"; + // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- @@ -30,6 +32,7 @@ export interface TrendingData { trendingRepos: TrendingRepo[]; searchRepos: SearchRepo[]; trendingFetchSuccess: boolean; + status: SourceStatus; } // --------------------------------------------------------------------------- @@ -201,5 +204,16 @@ export async function fetchTrendingData(): Promise { searchAiRepos(sevenDaysAgo), ]); - return { trendingRepos, searchRepos, trendingFetchSuccess: success }; + return { + trendingRepos, + searchRepos, + trendingFetchSuccess: success, + status: createSourceStatus({ + id: "github-trending", + label: "GitHub Trending HTML", + fetchedCount: trendingRepos.length, + acceptedCount: trendingRepos.length, + error: success ? undefined : "fetch or parse failed", + }), + }; } diff --git a/src/web.ts b/src/web.ts index 84c285f..4d97257 100644 --- a/src/web.ts +++ b/src/web.ts @@ -13,6 +13,7 @@ import fs from "node:fs"; import path from "node:path"; import { sleep } from "./date.ts"; +import { createSourceStatus, type SourceStatus } from "./source-status.ts"; // --------------------------------------------------------------------------- // Types @@ -46,6 +47,7 @@ export interface WebFetchResult { newItems: WebPageItem[]; /** Total URLs discovered in sitemap (for context in the report) */ totalDiscovered: number; + status: SourceStatus; } // --------------------------------------------------------------------------- @@ -146,6 +148,10 @@ export function isSitemapIndex(xml: string): boolean { return /]/.test(xml); } +function isUrlSet(xml: string): boolean { + return /]/.test(xml); +} + // --------------------------------------------------------------------------- // HTML content extraction // --------------------------------------------------------------------------- @@ -208,9 +214,10 @@ export function titleFromUrl(url: string): string { async function discoverUrls( site: "anthropic" | "openai" | "deepmind", -): Promise> { +): Promise<{ urls: Array<{ loc: string; lastmod?: string }>; errors: string[] }> { const cfg = SITE_CONFIGS[site]; const results: Array<{ loc: string; lastmod?: string }> = []; + const errors: string[] = []; if (cfg.subSitemapNames && cfg.subSitemapTemplate) { // Sitemap index: fetch each named sub-sitemap @@ -218,18 +225,22 @@ async function discoverUrls( const subUrl = cfg.subSitemapTemplate.replace("{name}", name); try { const xml = await httpGet(subUrl); + if (!isUrlSet(xml)) { + throw new Error(isSitemapIndex(xml) ? "unexpected sitemap index" : "unexpected sitemap response"); + } results.push(...parseSitemapUrls(xml)); await sleep(100); } catch (err) { console.error(` [web/${site}] sub-sitemap "${name}" failed: ${err}`); + errors.push(`${name}: ${err}`); } } } else { // Single sitemap const xml = await httpGet(cfg.sitemapUrl); - const all = isSitemapIndex(xml) - ? [] // unexpected; skip rather than recurse - : parseSitemapUrls(xml); + if (isSitemapIndex(xml)) throw new Error("unexpected sitemap index"); + if (!isUrlSet(xml)) throw new Error("unexpected sitemap response"); + const all = parseSitemapUrls(xml); const prefixes = cfg.prefixes ?? []; results.push( @@ -243,7 +254,7 @@ async function discoverUrls( ); } - return results; + return { urls: results, errors }; } // --------------------------------------------------------------------------- @@ -260,9 +271,18 @@ export function emptyState(): WebState { }; } +export function normalizeWebState(state: Partial): WebState { + const empty = emptyState(); + return { + anthropic: state.anthropic ?? empty.anthropic, + openai: state.openai ?? empty.openai, + deepmind: state.deepmind ?? empty.deepmind, + }; +} + export function loadWebState(): WebState { try { - return JSON.parse(fs.readFileSync(STATE_FILE, "utf-8")) as WebState; + return normalizeWebState(JSON.parse(fs.readFileSync(STATE_FILE, "utf-8")) as Partial); } catch { return emptyState(); } @@ -282,11 +302,13 @@ export async function fetchSiteContent( state: WebState, ): Promise { const cfg = SITE_CONFIGS[site]; - const siteState = state[site]; + const normalizedState = normalizeWebState(state); + Object.assign(state, normalizedState); + const siteState = normalizedState[site]; const isFirstRun = Object.keys(siteState.seenUrls).length === 0; console.log(` [web/${site}] Discovering URLs from sitemap...`); - const allDiscovered = await discoverUrls(site); + const { urls: allDiscovered, errors } = await discoverUrls(site); console.log(` [web/${site}] Discovered ${allDiscovered.length} URLs`); // Newest first @@ -318,6 +340,7 @@ export async function fetchSiteContent( // Build items — either from full page fetches or from sitemap metadata only const items: WebPageItem[] = []; + const failedContentUrls = new Set(); if (cfg.metadataOnly) { for (const { loc, lastmod } of toFetch) { items.push({ @@ -344,14 +367,17 @@ export async function fetchSiteContent( }); } catch (err) { console.error(` [web/${site}] Failed to fetch ${loc}: ${err}`); + errors.push(`${loc}: ${err}`); + failedContentUrls.add(loc); } await sleep(FETCH_DELAY_MS); } } - // Mark ALL discovered URLs as seen (not just fetched ones) - // This ensures future runs are truly incremental + // Mark discovered URLs as seen unless a requested content fetch failed. + // Capped first-run URLs still advance so future runs remain incremental. for (const { loc, lastmod } of allDiscovered) { + if (failedContentUrls.has(loc)) continue; siteState.seenUrls[loc] = lastmod ?? "seen"; } siteState.lastChecked = new Date().toISOString(); @@ -362,5 +388,12 @@ export async function fetchSiteContent( isFirstRun, newItems: items, totalDiscovered: allDiscovered.length, + status: createSourceStatus({ + id: `web-${site}`, + label: cfg.name, + fetchedCount: allDiscovered.length, + acceptedCount: items.length, + error: errors.length > 0 ? `partial failure: ${errors.join("; ")}` : undefined, + }), }; }