diff --git a/.gitignore b/.gitignore index 60bfcfe..186248b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ team.yaml tokens.yaml registry.yaml feishu.yaml +notion.yaml +google.yaml github.token *.local.* diff --git a/README.md b/README.md index 9bae3b0..7099aeb 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,17 @@ If your team keeps human-written docs (goals, decisions, notes) in a Lark/Feishu > China Feishu (`open.feishu.cn`) and international Lark (`open.larksuite.com`) are isolated platforms — create the app on whichever one your team uses. Create a **new** wiki later? Repeat step 3 for it, or the brain won't see it. +### Other doc sources (Notion · Google Docs) + +Doc mirroring is **provider-pluggable** — the same sync engine (`server/docsync.mjs`) backs every source, so each one is just a small adapter (`core/.mjs` + `server/docs.mjs`). Two more ship today, both gated by their own `*.yaml` (leave it out → that layer stays off) and both following the same "share with the bot, then it mirrors" model: + +- **Notion** — create an *internal integration* at (read-only); **share** the pages/databases with it (page → ••• → *Connections*); copy `notion.example.yaml` → `notion.yaml`, fill `api_token`, restart. Pages land under `notion//…`. +- **Google Docs** — create a *service account* (enable the Drive API), download its JSON key, and **share** the docs/folders with the service account's email (read-only); copy `google.example.yaml` → `google.yaml`, point it at the key, restart. Docs land under `google//…`. (Auth is a self-signed JWT — no extra SDK.) + +Unshared pages/docs stay invisible (the share is the real access gate); sub-pages and folder contents inherit. After a poll cycle everything is searchable via `grep`. + +> Confluence can follow the same adapter shape — contributions welcome. + --- ## Changelog diff --git a/README.zh-CN.md b/README.zh-CN.md index b1f28db..0f5ed10 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -173,6 +173,17 @@ npm run sync -- --once # 收一次（或 `npm run sync` 让它在后台持续 > 国内飞书（`open.feishu.cn`）与国际版 Lark（`open.larksuite.com`）数据隔离——在你团队实际所在的平台建应用。以后每**新建**一个知识库，都要对它重复第 3 步，否则大脑看不见。 +### 其它文档源（Notion · Google Docs） + +文档镜像是**可插拔多源**的——所有源共用同一套同步引擎（`server/docsync.mjs`），每个源只是一个小 adapter（`core/<源>.mjs` + `server/<源>docs.mjs`）。现已多支持两个源，各由自己的 `*.yaml` 把关（不建即不启用），都遵循"把内容 share 给机器人、它就镜像"的模型： + +- **Notion** —— 在 建一个 *internal integration*（只读）；把页面/数据库 **share** 给它（页面 → ••• → *Connections*）；复制 `notion.example.yaml` → `notion.yaml`，填 `api_token`，重启。页面落到 `notion//…`。 +- **Google Docs** —— 建一个 *service account*（启用 Drive API），下载 JSON key，把文档/文件夹 **share** 给该 service account 的邮箱（只读）；复制 `google.example.yaml` → `google.yaml`，指向 key，重启。文档落到 `google//…`。（认证用自签 JWT，不引入额外 SDK。） + +没 share 的页面/文档看不到（share 才是真正的授权闸）；子页面、文件夹内容继承。一轮后即可 `grep` 搜到。 + +> Confluence 可按同一 adapter 范式接入——欢迎贡献。 + --- ## 更新日志 diff --git a/core/feishu.mjs b/core/feishu.mjs index f2870ca..09bbc6d 100644 --- a/core/feishu.mjs +++ b/core/feishu.mjs @@ -4,6 +4,9 @@ import { readFileSync, existsSync } from "node:fs"; import { parse } from "yaml"; import * as lark from "@larksuiteoapi/node-sdk"; +import { sleep, withRetry } from "./retry.mjs"; + +export { sleep, withRetry }; // feishu.yaml（服务器级、gitignore、启动加载 restart 生效，同 registry/tokens 一族）。 // 缺文件 / 解析失败 / 没配齐 app 凭证 → 返回 null = 文档层不启用（与 GITHUB_TOKEN 缺省同款行为）。 @@ -20,22 +23,6 @@ export function loadFeishu(path) { }; } -export const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); - -// 重试退避：429（限流）与 5xx（网关偶发）值得等一等再试；4xx 业务错误（权限/不存在）重试无意义直接抛。 -// status 取不到（网络层断连）也按可重试算。 -const RETRYABLE = new Set([429, 500, 502, 503, 504]); -export async function withRetry(fn, { delays = [1000, 3000, 10000], sleepFn = sleep } = {}) { - for (let i = 0; ; i++) { - try { return await fn(); } - catch (e) { - const status = e?.response?.status ?? 0; - if (i >= delays.length || (status !== 0 && !RETRYABLE.has(status))) throw e; - await sleepFn(delays[i]); - } - } -} - // 统一请求口：client.request 自动注入 tenant_access_token；外面包一层重试。 export function makeReq({ app_id, app_secret }, retryOpts = {}) { const client = new lark.Client({ diff --git a/core/google.mjs b/core/google.mjs new file mode 100644 index 0000000..43cc510 --- /dev/null +++ b/core/google.mjs @@ -0,0 +1,91 @@ +// Google Docs 只读镜像（Drive + export，原生 fetch + node:crypto 自签 JWT，无 googleapis SDK）。 +// 凭证 = 一个 service account；把文档/文件夹 share 给它的邮箱才可见（同 Notion 的"share 给 integration"）。 +// 正文走 Drive export 到 text/plain：一次拿全文，免解析 Docs 结构。增量靠 Drive 的 modifiedTime（已是 RFC3339 ISO）。 +import { readFileSync, existsSync } from "node:fs"; +import { createSign } from "node:crypto"; +import { parse } from "yaml"; +import { sleep, withRetry } from "./retry.mjs"; + +export { sleep, withRetry }; + +const DRIVE = "https://www.googleapis.com/drive/v3"; +const TOKEN_URI = "https://oauth2.googleapis.com/token"; +const SCOPE = "https://www.googleapis.com/auth/drive.readonly"; + +// google.yaml（服务器级、gitignore、启动加载 restart 生效）。缺文件/解析失败/没配齐凭证 → null = Google 层不启用。 +// 凭证两种给法：① key_file 指向下载的 service-account JSON；② 直接内联 client_email + private_key。 +export function loadGoogle(path) { + if (!existsSync(path)) return null; + let cfg; + try { cfg = parse(readFileSync(path, "utf8")) || {}; } catch { return null; } + let client_email = cfg.client_email, private_key = cfg.private_key; + if (cfg.key_file) { + try { const k = JSON.parse(readFileSync(cfg.key_file, "utf8")); client_email = k.client_email; private_key = k.private_key; } + catch { return null; } + } + if (!client_email || !private_key) return null; + return { + client_email: String(client_email), + private_key: String(private_key), + poll_hours: Number(cfg.poll_hours) || 4, + workspace: String(cfg.workspace || "google"), // 真相库 google/ 下的目录名（自己起） + }; +} + +// service-account JWT → assertion（RS256 签名走 node:crypto，免 SDK）。 +function signJwt({ client_email, private_key }) { + const enc = (o) => Buffer.from(JSON.stringify(o)).toString("base64url"); + const iat = Math.floor(Date.now() / 1000); + const head = { alg: "RS256", typ: "JWT" }; + const claim = { iss: client_email, scope: SCOPE, aud: TOKEN_URI, iat, exp: iat + 3600 }; + const body = `${enc(head)}.${enc(claim)}`; + const sig = createSign("RSA-SHA256").update(body).end().sign(private_key).toString("base64url"); + return `${body}.${sig}`; +} + +// 用 JWT 换 access_token（OAuth2 jwt-bearer 流）。 +export async function fetchToken(cfg) { + const r = await fetch(TOKEN_URI, { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer", assertion: signJwt(cfg) }), + }); + if (!r.ok) throw Object.assign(new Error(`Google token ${r.status}: ${(await r.text()).slice(0, 120)}`), { status: r.status }); + return r.json(); // { access_token, expires_in, token_type } +} + +// 认证请求口：缓存 access_token（~1h），到期前 60s 刷新；req(method, path, {raw}) → json 或纯文本。 +// path 以 / 开头时相对 Drive v3；完整 http 原样用。非 2xx 抛带 .status 的错误（供 withRetry 判重试）。 +export function makeReq(cfg, retryOpts = {}) { + let tok = null, exp = 0; + const auth = async () => { + if (tok && Date.now() < exp - 60000) return tok; + const t = await fetchToken(cfg); + tok = t.access_token; exp = Date.now() + (Number(t.expires_in) || 3600) * 1000; + return tok; + }; + return (method, path, { raw = false } = {}) => withRetry(async () => { + const token = await auth(); + const r = await fetch(path.startsWith("http") ? path : DRIVE + path, { method, headers: { authorization: `Bearer ${token}` } }); + if (!r.ok) throw Object.assign(new Error(`Google ${r.status} on ${path}: ${(await r.text()).slice(0, 120)}`), { status: r.status }); + return raw ? r.text() : r.json(); + }, retryOpts); +} + +// 列出 service account 可见的全部 Google 文档（分页）。归一为 [{id,name,modifiedTime,webViewLink}]。 +export async function listDocs(req) { + const out = []; + const q = "mimeType='application/vnd.google-apps.document' and trashed=false"; + const fields = "nextPageToken,files(id,name,modifiedTime,webViewLink)"; + let pageToken = ""; + do { + const qs = new URLSearchParams({ q, fields, pageSize: "100", ...(pageToken ? { pageToken } : {}) }); + const r = await req("GET", `/files?${qs}`); + out.push(...(r?.files || [])); + pageToken = r?.nextPageToken || ""; + } while (pageToken); + return out; +} + +// 导出某文档为纯文本（Drive export，一次拿全文，免解析 Docs 结构）。 +export const exportDocText = (req, id) => req("GET", `/files/${id}/export?mimeType=text/plain`, { raw: true }); diff --git a/core/notion.mjs b/core/notion.mjs new file mode 100644 index 0000000..0232172 --- /dev/null +++ b/core/notion.mjs @@ -0,0 +1,99 @@ +// Notion 只读 API（REST, 原生 fetch）——单向镜像团队 Notion 页面进真相库，供 grep/read。 +// 与 github.mjs 同款：原生 fetch、无 SDK（不给客户端/服务端添依赖）；错误带 .status；429/5xx 退避重试。 +// 凭证 = 一个 internal integration token；只有被 share 给该 integration 的页面才可见（见 notion.example.yaml）。 +import { readFileSync, existsSync } from "node:fs"; +import { parse } from "yaml"; +import { sleep, withRetry } from "./retry.mjs"; + +export { sleep, withRetry }; + +const API = "https://api.notion.com/v1"; +const VERSION = "2022-06-28"; // Notion-Version 请求头，固定一个验证过的版本，别跟着平台默认漂 + +// notion.yaml（服务器级、gitignore、启动加载 restart 生效，同 feishu/registry/tokens 一族）。 +// 缺文件 / 解析失败 / 没配 api_token → 返回 null = Notion 文档层不启用（与 loadFeishu 同款行为）。 +export function loadNotion(path) { + if (!existsSync(path)) return null; + let cfg; + try { cfg = parse(readFileSync(path, "utf8")) || {}; } catch { return null; } + if (!cfg.api_token) return null; + return { + api_token: String(cfg.api_token), + poll_hours: Number(cfg.poll_hours) || 4, + workspace: String(cfg.workspace || "notion"), // 真相库 notion/ 下的目录名（Notion API 不给工作区名，自己起） + }; +} + +// 统一请求口：注入 token + Notion-Version；非 2xx 抛带 .status 的错误（供 withRetry 判重试，同 github.mjs）。 +// GET 不带 body（分页 cursor 走 query）；POST/search 带 JSON body。 +export function makeReq({ api_token }, retryOpts = {}) { + const headers = { + authorization: `Bearer ${api_token}`, + "notion-version": VERSION, + "content-type": "application/json", + "user-agent": "team-brain", + }; + return (method, path, body) => withRetry(async () => { + const r = await fetch(API + path, { method, headers, body: body ? JSON.stringify(body) : undefined }); + if (!r.ok) throw Object.assign(new Error(`Notion ${r.status} on ${path}: ${(await r.text()).slice(0, 120)}`), { status: r.status }); + return r.json(); + }, retryOpts); +} + +// POST /search 分页（cursor 在 body）：列出 integration 可见的全部 page。 +async function searchAll(req, body) { + const out = []; + let cursor; + do { + const r = await req("POST", "/search", { ...body, page_size: 100, ...(cursor ? { start_cursor: cursor } : {}) }); + out.push(...(r?.results || [])); + cursor = r?.has_more ? (r?.next_cursor || null) : null; + } while (cursor); + return out; +} +export const searchPages = (req) => searchAll(req, { filter: { value: "page", property: "object" } }); + +// GET /blocks/{id}/children 分页（cursor 在 query）。 +async function childrenAll(req, blockId) { + const out = []; + let cursor; + do { + const qs = `?page_size=100${cursor ? `&start_cursor=${encodeURIComponent(cursor)}` : ""}`; + const r = await req("GET", `/blocks/${blockId}/children${qs}`); + out.push(...(r?.results || [])); + cursor = r?.has_more ? (r?.next_cursor || null) : null; + } while (cursor); + return out; +} + +// 从 page 对象取标题：properties 里 type==="title" 的那个，拼 plain_text；取不到兜底 untitled。 +export function pageTitle(page) { + const props = page?.properties || {}; + for (const k of Object.keys(props)) { + if (props[k]?.type === "title") return (props[k].title || []).map((t) => t.plain_text || "").join("").trim() || "untitled"; + } + return "untitled"; +} + +// 一个块的纯文本：任何块类型下的 rich_text 拼接（标题/段落/列表/引用… 字段名都叫 rich_text）。 +const blockText = (b) => { + const rich = b?.[b.type]?.rich_text; + return Array.isArray(rich) ? rich.map((x) => x.plain_text || "").join("") : ""; +}; + +// 递归取 page/block 的纯文本正文：children 翻页 → 各块拼文本 → 有子块下钻（限深防御）。pace 由调用方按限流给。 +export async function pageBlocksText(req, blockId, { sleepFn = sleep, pace = 0, depth = 0 } = {}) { + if (depth > 8) return ""; + const blocks = await childrenAll(req, blockId); + const parts = []; + for (const b of blocks) { + const line = blockText(b); + if (line) parts.push(line); + if (b.has_children) { + if (pace) await sleepFn(pace); + const sub = await pageBlocksText(req, b.id, { sleepFn, pace, depth: depth + 1 }); + if (sub) parts.push(sub); + } + } + return parts.join("\n"); +} diff --git a/core/retry.mjs b/core/retry.mjs new file mode 100644 index 0000000..93e080c --- /dev/null +++ b/core/retry.mjs @@ -0,0 +1,17 @@ +// 共享重试退避：429（限流）与 5xx（网关偶发）值得等一等再试；4xx 业务错误（权限/不存在）重试无意义直接抛。 +// status 取不到（网络层断连）也按可重试算。错误形状两种都认：原生 fetch 抛的 e.status / SDK 抛的 e.response.status。 +// 文档源各 provider（feishu/notion/google）共用——退避策略只此一处，改一处全改。 +export const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +const RETRYABLE = new Set([429, 500, 502, 503, 504]); + +export async function withRetry(fn, { delays = [1000, 3000, 10000], sleepFn = sleep } = {}) { + for (let i = 0; ; i++) { + try { return await fn(); } + catch (e) { + const status = e?.status ?? e?.response?.status ?? 0; + if (i >= delays.length || (status !== 0 && !RETRYABLE.has(status))) throw e; + await sleepFn(delays[i]); + } + } +} diff --git a/core/safe.mjs b/core/safe.mjs index 28db8d4..0e6833a 100644 --- a/core/safe.mjs +++ b/core/safe.mjs @@ -10,6 +10,11 @@ export function safeSegment(s, label = "segment") { return v; } +// 文档源标题/库名要进文件名：去掉路径分隔与控制符、压空白、截短到 80；空兜底 untitled。 +// 之后仍过 safeSegment（穿越红线，别绕开）。文档镜像各 provider（feishu/notion/google）共用——名字消毒只此一处。 +export const saniName = (s, label = "name") => + safeSegment((String(s || "").replace(/[/\\\0]/g, "-").replace(/\s+/g, " ").trim() || "untitled").slice(0, 80), label); + // 多段相对路径锁在 root 内：解析成绝对路径后必须仍以 root 为前缀（或恰为 root）。 // 给只读查询（ls/log/grep 的 path 收窄）用——客户端给的 rel 直接落进 root 子树前必过。 export function safeRelPath(root, rel, label = "path") { diff --git a/google.example.yaml b/google.example.yaml new file mode 100644 index 0000000..8a3798e --- /dev/null +++ b/google.example.yaml @@ -0,0 +1,28 @@ +# 复制成 google.yaml（服务器级、已 gitignore——含 service account 私钥）。 +# 不建该文件 / 不填凭证 → Google 文档层不启用（与 feishu.yaml / notion.yaml 同款"缺省即关"）。 +# +# 准备（一次性）： +# 1. 在 Google Cloud Console 建一个项目，启用 **Google Drive API**。 +# 2. 建一个 **service account**，给它生成一把 JSON key 并下载。 +# 3. 把要镜像的 Google 文档（或整个文件夹）**share 给该 service account 的邮箱** +# （xxx@xxx.iam.gserviceaccount.com），只读权限即可。 +# —— 这步才是真正的授权闸：没 share 的文档 service account 看不到；文件夹内文档随文件夹继承。 +# 4. 填下面的凭证，重启 server。启动 120s 后首跑，一轮后文档落到 +# google//<标题>--.md，grep/read 即可搜（正文已脱敏）。 +# +# 凭证两种给法（二选一）： +# ① key_file：指向下载的 service-account JSON（把那个 .json 也放到 server 上、别进仓）。 +# ② 内联 client_email + private_key（private_key 是多行 PEM，用 YAML 块标量 |）。 +# +# 正本永远在 Google Docs（人在那读写），这棵子树只是可重建的只读检索镜像——要改去 Google Docs 改。 + +key_file: /var/lib/team-brain/google-sa.json +# 或者内联： +# client_email: team-brain@your-project.iam.gserviceaccount.com +# private_key: | +# -----BEGIN PRIVATE KEY----- +# ... +# -----END PRIVATE KEY----- + +poll_hours: 4 # 轮询间隔（小时），可选，默认 4 +workspace: google # 真相库 google/ 下的目录名（自己起一个），可选 diff --git a/mcp/server.mjs b/mcp/server.mjs index f879f61..04bf391 100644 --- a/mcp/server.mjs +++ b/mcp/server.mjs @@ -45,12 +45,12 @@ async function api(path, params) { } class HttpError extends Error {} -const INSTRUCTIONS = `团队大脑（team-brain）：全队 Claude Code/Codex 的 session 汇成一个 git 真相库（每条 session 一份【脱敏全文对话】），再叠 GitHub/GitLab/Gitea 代码现状 + 飞书文档镜像。 +const INSTRUCTIONS = `团队大脑（team-brain）：全队 Claude Code/Codex 的 session 汇成一个 git 真相库（每条 session 一份【脱敏全文对话】），再叠 GitHub/GitLab/Gitea 代码现状 + 人写文档镜像（飞书 wiki / Notion / Google Docs）。 回答「X 做到哪了 / 当初怎么定的 / 谁在搞 Y / 最近有啥进展」这类跨人跨项目的问题时，先用这些工具查证，再据实答（带依据）。 == 心智模型：把真相库当一个只读文件夹来逛 == 结构：spaces//sessions//-.md（全文可读对话）+ 同名 .jsonl（原始结构）； -　　　feishu/<知识库>/<标题>--.md（飞书文档镜像：战略/PRD/客户笔记等人写文档，单向同步、grep 可中）。 +　　　feishu/·notion/·google/<库>/<标题>--.md（人写文档镜像：飞书 wiki / Notion / Google Docs，战略/PRD/客户笔记等，单向同步、grep 可中）。 所有工具用统一坐标【真相库相对 path】串起来：grep/find/ls 给你 path，read 拿 path 深挖——就像在本地翻一个项目目录。 == 6 个只读原语 + 1 个出网 == @@ -71,7 +71,7 @@ const INSTRUCTIONS = `团队大脑（team-brain）：全队 Claude Code/Codex == 三种料源（合着用才答得全）== · session(.md) = 进展前沿：最新、含思考过程、可能还没 push →（grep/find 找，read 深挖） · GitHub/GitLab/Gitea = 代码现状：分支/PR·MR/commit →（read_github）。看具体文件得 read_github 给路径现拉。 -· 飞书文档镜像(feishu/) = 目标·决策（人写的战略/PRD/笔记）：grep 全文可中、read 读全文；正本在飞书（frontmatter 带 url），要改/评论引导去飞书。 +· 人写文档镜像(feishu/·notion/·google/) = 目标·决策（战略/PRD/笔记，来自飞书 wiki / Notion / Google Docs）：grep 全文可中、read 读全文；正本在源平台（frontmatter 带 url），要改/评论引导去源平台。 == 套路：先低成本定位，再深挖 1-2 条 == · "X 做到哪了 / 怎么定的" → grep 定位 → read 看细节 →（涉代码再 read_github） @@ -79,7 +79,7 @@ const INSTRUCTIONS = `团队大脑（team-brain）：全队 Claude Code/Codex · "某人某段时间做了什么" → sessions(author=…, since=…, until=…)（按工作时间，别用 log）→ read 深挖 · "最近真相库收进了啥" → log（入库时间线，可加 since/space）→ read 深挖 · "项目里有什么 / 哪些分支 / 哪条 session" → ls 摸结构，或 find 按名找 -· "目标是什么 / 文档里怎么写的" → grep 定位（命中 feishu/ 的就是人写文档）→ read；ls feishu 看有哪些知识库/文档 +· "目标是什么 / 文档里怎么写的" → grep 定位（命中 feishu/·notion/·google/ 的就是人写文档）→ read；ls feishu|notion|google 看有哪些库/文档 == space 模型 == 团队登记的代码仓（跨人合并）：github__owner__repo / gitlab__host__owner__repo / gitea__host__owner__repo（gitlab/gitea 带 host 区分自建实例）； @@ -96,7 +96,7 @@ server.registerTool( title: "正则全文搜（精确定位首选）", description: "用 git grep 在真相库做正则全文搜（带上下文行）：支持【或 融资|finance】、标识符、代码符号、词边界。" + - "默认搜全部 .md——session 全文对话 + feishu/ 飞书文档镜像（人写的战略/PRD/笔记）都在内；raw=true 才连 .jsonl 原始结构一起搜（更全更吵）。可按 space 收窄。" + + "默认搜全部 .md——session 全文对话 + feishu/·notion/·google/ 人写文档镜像（飞书/Notion/Google Docs：战略/PRD/笔记）都在内；raw=true 才连 .jsonl 原始结构一起搜（更全更吵）。可按 space 收窄。" + "返回 path:line: 形式——把 path 抄给 read 即可深挖。有具体词就先用它定位，是第一选择。", inputSchema: { q: z.string().describe("正则/关键词；善用『或』a|b 一次搜多个同义词、用词边界提精度，如 ontology、融资|finance"), @@ -162,7 +162,7 @@ server.registerTool( description: "列结构，用于先搞清「有哪些 space / 这个项目对应哪个 space / 某 space 有哪些分支、几条 session」。" + "不给 path → 列所有 space（github__owner__repo 团队仓 / local__<人> 个人桶）；" + - "给 path（如 spaces//sessions）→ 往里看；path=feishu → 看有哪些飞书知识库/文档（人写的战略/PRD/笔记镜像）。" + + "给 path（如 spaces//sessions）→ 往里看；path=feishu|notion|google → 看有哪些人写文档库/文档（飞书 wiki / Notion / Google Docs 镜像）。" + "目录附子项计数。不确定项目对应哪个 space 时，先 ls 再 grep/find 收窄。", inputSchema: { path: z.string().optional().describe("相对真相库根的路径；不给则列 spaces 顶层"), @@ -326,4 +326,4 @@ server.registerTool( ); await server.connect(new StdioServerTransport()); -console.error("team-brain MCP (0.6) 已启动：grep + find + read + ls + sessions + stats + log + read_github（含 feishu/ 文档镜像）→", cfg.server_url); +console.error("team-brain MCP (0.6) 已启动：grep + find + read + ls + sessions + stats + log + read_github（含 feishu/·notion/·google/ 文档镜像）→", cfg.server_url); diff --git a/notion.example.yaml b/notion.example.yaml new file mode 100644 index 0000000..eb640e4 --- /dev/null +++ b/notion.example.yaml @@ -0,0 +1,16 @@ +# 复制成 notion.yaml（服务器级、已 gitignore——含 integration token）。 +# 不建该文件 / 不填 api_token → Notion 文档层不启用（与 feishu.yaml 同款"缺省即关"）。 +# +# 准备（一次性）： +# 1. 到 https://www.notion.so/my-integrations 建一个 internal integration， +# 拿 "Internal Integration Secret"（只需 Read content 权限）。 +# 2. 把要镜像的页面/数据库 share 给该 integration： +# 目标页面右上 ··· → Connections（连接）→ 选你建的 integration。 +# —— 这步才是真正的授权闸：没 share 的页面 integration 看不到；子页面随父页面继承。 +# 3. 填下面的 api_token，重启 server。启动 90s 后首跑，一轮后页面落到 +# notion//<标题>--.md，grep/read 即可搜（正文已脱敏）。 +# +# 正本永远在 Notion（人在那读写），这棵子树只是可重建的只读检索镜像——要改去 Notion 改。 +api_token: secret_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +poll_hours: 4 # 轮询间隔（小时），可选，默认 4 +workspace: notion # 真相库 notion/ 下的目录名（Notion API 不提供工作区名，自己起一个，可选） diff --git a/server/docsync.mjs b/server/docsync.mjs new file mode 100644 index 0000000..1cc3a6c --- /dev/null +++ b/server/docsync.mjs @@ -0,0 +1,90 @@ +// 通用文档单向镜像引擎：把任一文档源（飞书 / Notion / …）的 collection→doc 树对账进真相库 +// //（frontmatter + 脱敏正文），agent 用现有 grep/find/read 即可搜。 +// 【provider 无关】的部分全在这里：增量（按 edited 指纹跳过未变）、孤儿/死库 prune、零库保护、一轮一个 commit。 +// 每个源只实现一个 provider adapter（见 feishudocs.mjs / notiondocs.mjs），把"这个源长什么样"喂进来。 +import { join } from "node:path"; +import { readFileSync, writeFileSync, mkdirSync, readdirSync, existsSync, rmSync } from "node:fs"; +import { fm } from "../core/card.mjs"; +import { log } from "../core/log.mjs"; +import { commit } from "./gitstore.mjs"; + +export const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// provider 接口（全部 provider 无关逻辑都靠这几个钩子参数化）： +// subtree 真相库下子树名（"feishu" / "notion"） +// label 日志前缀（"feishu-docs" / "notion-docs"） +// listCollections(req) → [collection]，源里的"库"（飞书=wiki 知识库，Notion=合成单库）；空/抛错都触发零库保护 +// collectionDirOf(collection) → dirName，子树下目录名（可 throw → 跳过该库，不影响其它） +// walkDocs(req, collection) → [item]，库里全部文档；item.node 是文档对象，item 其余字段透传给 renderDoc（如 titlePath） +// fileNameOf(node) → fileName（.md；可 throw → 跳过该文档，路径红线在这兜） +// editTimeOf(node) → isoString，增量指纹（与已落盘 frontmatter 的 edited 比；空串 → 每轮都重拉） +// fetchBody(req, node, { sleepFn, pace }) → body 正文（可 throw → 保留旧镜像、计一次 error） +// renderDoc({ collection, node, body, now, ...item }) → cardText（buildCard + redactAgent，调用方负责脱敏） +// commitMessage(written, pruned) → string +export async function syncDocs(TRUTH, req, provider, opts = {}) { + const { commitFn = commit, pace = 250, sleepFn = sleep, now = () => new Date().toISOString() } = opts; + const root = join(TRUTH, provider.subtree); + + let collections; + try { collections = await provider.listCollections(req); } + catch (e) { log.warn(`[${provider.label}] 列库失败 → 本轮跳过，不清理`, { err: e.message }); return { collections: 0, written: 0, pruned: 0, skipped: 0, errors: 1 }; } + // 一个库都看不到（多半凭证/授权坏了，不是文档真没了）→ 整轮跳过且不删，护住已有镜像。 + if (!collections.length) { + log.warn(`[${provider.label}] 一个库都看不到（授权/凭证/scope 坏了？）→ 本轮跳过，不清理`); + return { collections: 0, written: 0, pruned: 0, skipped: 0, errors: 1 }; + } + + let written = 0, pruned = 0, skipped = 0, errors = 0; + const liveDirs = new Set(); + for (const collection of collections) { + let dirName; + try { dirName = provider.collectionDirOf(collection); } + catch (e) { errors++; log.warn(`[${provider.label}] 库名/ID 不合法，跳过`, { err: e.message }); continue; } + liveDirs.add(dirName); + const dir = join(root, dirName); + mkdirSync(dir, { recursive: true }); + + let docs; + try { docs = await provider.walkDocs(req, collection); } + catch (e) { errors++; log.warn(`[${provider.label}] 遍历库失败，跳过（不清理）`, { err: e.message }); continue; } + + const expect = new Set(); + for (const item of docs) { + const { node } = item; + let fname; + try { fname = provider.fileNameOf(node); } + catch (e) { errors++; log.warn(`[${provider.label}] 文档名不合法，跳过`, { err: e.message }); continue; } + expect.add(fname); + const fpath = join(dir, fname); + const editedISO = provider.editTimeOf(node); + // 增量：指纹未变就不拉正文、不重写（飞书 raw_content 限流，Notion 也按 request 计费）。 + if (editedISO && existsSync(fpath) && fm(readFileSync(fpath, "utf8").slice(0, 2048), "edited") === editedISO) { skipped++; continue; } + + let body; + try { body = await provider.fetchBody(req, node, { sleepFn, pace }); } + catch (e) { errors++; log.warn(`[${provider.label}] 读正文失败（保留旧镜像）`, { err: e.message }); continue; } + writeFileSync(fpath, provider.renderDoc({ ...item, collection, body, now: now() })); + written++; + } + // 该库遍历成功 → 树里已不存在的文档，删掉镜像（在源被删/移走）。 + for (const f of readdirSync(dir)) { + if (f.endsWith(".md") && !expect.has(f)) { rmSync(join(dir, f)); pruned++; } + } + } + // 整个库从可见列表消失（被删/取消授权）→ 镜像目录一并清（列表非空才走到这，有零库保护兜底）。 + if (existsSync(root)) { + for (const d of readdirSync(root, { withFileTypes: true })) { + if (d.isDirectory() && !liveDirs.has(d.name)) { rmSync(join(root, d.name), { recursive: true }); pruned++; } + } + } + + let sha = null; + if (written || pruned) { + sha = await commitFn(TRUTH, { + name: "team-brain-bot", email: "bot@team-brain", + message: provider.commitMessage(written, pruned), + paths: [provider.subtree], + }); + } + return { collections: collections.length, written, pruned, skipped, errors, commit: sha }; +} diff --git a/server/feishudocs.mjs b/server/feishudocs.mjs index 0c161d7..72ea531 100644 --- a/server/feishudocs.mjs +++ b/server/feishudocs.mjs @@ -3,24 +3,17 @@ // 飞书是文档的【权威库】（人在飞书读写），这棵子树只是可重建的检索镜像—— // agent 用现有 grep/find/read 就能搜到文档（tenant token 原生搜索恒 0，自建索引是必须，见 FEISHU_RESEARCH §10）。 // 增量靠节点的 obj_edit_time：没变就不拉正文（raw_content 限流 5/s，全量重拉既慢又撞限流）。 -import { join } from "node:path"; -import { readFileSync, writeFileSync, mkdirSync, readdirSync, existsSync, rmSync } from "node:fs"; -import { buildCard, fm } from "../core/card.mjs"; +// provider 无关的对账/增量/prune/commit/安全边界都在 ./docsync.mjs；这里只描述"飞书长什么样"。 +import { buildCard } from "../core/card.mjs"; import { redactAgent } from "../core/redact.mjs"; -import { safeSegment } from "../core/safe.mjs"; -import { log } from "../core/log.mjs"; -import { commit } from "./gitstore.mjs"; -import { listWikiSpaces, walkWikiNodes, docxRawContent, sleep } from "../core/feishu.mjs"; +import { safeSegment, saniName } from "../core/safe.mjs"; +import { listWikiSpaces, walkWikiNodes, docxRawContent } from "../core/feishu.mjs"; +import { syncDocs } from "./docsync.mjs"; -// 飞书标题/库名要进文件名：去掉路径分隔与控制符、压空白、截短；空标题兜底。 -// 之后仍过 safeSegment（穿越红线，别绕开）。 -const saniName = (s) => - safeSegment((String(s || "").replace(/[/\\\0]/g, "-").replace(/\s+/g, " ").trim() || "untitled").slice(0, 80), "feishu-name"); +const fileNameOf = (node) => `${saniName(node.title, "feishu-name")}--${safeSegment(node.node_token, "node_token")}.md`; +const spaceDirOf = (sp) => `${saniName(sp.name, "feishu-name")}__${safeSegment(sp.space_id, "space_id")}`; -const fileNameOf = (node) => `${saniName(node.title)}--${safeSegment(node.node_token, "node_token")}.md`; -const spaceDirOf = (sp) => `${saniName(sp.name)}__${safeSegment(sp.space_id, "space_id")}`; - -// obj_edit_time 是 unix 秒（字符串）→ ISO；取不到给空（frontmatter 空值自动略过）。 +// obj_edit_time 是 unix 秒（字符串）→ ISO；取不到给空（frontmatter 空值自动略过；空 edited → 每轮都重拉）。 const isoOf = (t) => { const n = Number(t); return n ? new Date(n * 1000).toISOString() : ""; }; // 一篇文档卡：frontmatter（身份/位置/时间）+ 脱敏正文。edited 同时是增量对账的指纹。 @@ -41,70 +34,28 @@ export function renderDoc({ space, node, titlePath, body, wikiBase = "", now = " }, redactAgent(body)); } -// 全量对账一轮：写有变化的、清理已消失的、其余跳过。一轮一个 commit。 -// 安全边界：① 只有【该库遍历成功】才清它的孤儿（API 抖一下不能误删整库镜像）； -// ② 知识库列表为空直接整轮跳过（多半是授权/凭证坏了，不是文档真没了）。 -export async function syncFeishuDocs(TRUTH, req, opts = {}) { - const { wikiBase = "", commitFn = commit, pace = 250, sleepFn = sleep, now = () => new Date().toISOString() } = opts; - const root = join(TRUTH, "feishu"); - const spaces = await listWikiSpaces(req); - if (!spaces.length) { - log.warn("[feishu-docs] 一个知识库都看不到（应用没被加进任何知识库，或凭证/scope 坏了）→ 本轮跳过，不清理"); - return { spaces: 0, written: 0, pruned: 0, skipped: 0, errors: 1 }; - } - - let written = 0, pruned = 0, skipped = 0, errors = 0; - const liveDirs = new Set(); - for (const sp of spaces) { - let dirName; - try { dirName = spaceDirOf(sp); } catch (e) { errors++; log.warn("[feishu-docs] 知识库名/ID 不合法，跳过", { space: sp?.space_id, err: e.message }); continue; } - liveDirs.add(dirName); - const dir = join(root, dirName); - mkdirSync(dir, { recursive: true }); - - let nodes; - try { nodes = await walkWikiNodes(req, sp.space_id); } - catch (e) { errors++; log.warn("[feishu-docs] 遍历知识库失败，跳过（不清理）", { space: sp.name, err: e.message }); continue; } - - const expect = new Set(); - for (const { node, titlePath } of nodes) { - let fname; - try { fname = fileNameOf(node); } catch (e) { errors++; log.warn("[feishu-docs] 节点名不合法，跳过", { title: node?.title, err: e.message }); continue; } - expect.add(fname); - const fpath = join(dir, fname); - const editedISO = isoOf(node.obj_edit_time); - if (existsSync(fpath) && fm(readFileSync(fpath, "utf8").slice(0, 2048), "edited") === editedISO) { skipped++; continue; } - - let body; - if (node.obj_type === "docx") { - try { await sleepFn(pace); body = await docxRawContent(req, node.obj_token); } - catch (e) { errors++; log.warn("[feishu-docs] 读正文失败（保留旧镜像）", { title: node.title, err: e.message }); continue; } - } else { - const url = wikiBase ? `${wikiBase}/wiki/${node.node_token}` : `node_token=${node.node_token}`; - body = `（${node.obj_type} 类型，正文不入索引——去飞书看：${url}）`; - } - writeFileSync(fpath, renderDoc({ space: sp, node, titlePath, body, wikiBase, now: now() })); - written++; - } - // 该库遍历成功 → 树里已不存在的节点，删掉镜像（文档在飞书被删/移走） - for (const f of readdirSync(dir)) { - if (f.endsWith(".md") && !expect.has(f)) { rmSync(join(dir, f)); pruned++; } - } - } - // 整个知识库从可见列表消失（被删/取消授权）→ 镜像目录一并清（列表非空才走到这，有零库保护） - if (existsSync(root)) { - for (const d of readdirSync(root, { withFileTypes: true })) { - if (d.isDirectory() && !liveDirs.has(d.name)) { rmSync(join(root, d.name), { recursive: true }); pruned++; } - } - } +// 飞书 provider adapter：把"飞书长什么样"喂给通用引擎。 +function feishuProvider({ wikiBase = "" } = {}) { + return { + subtree: "feishu", + label: "feishu-docs", + listCollections: (req) => listWikiSpaces(req), + collectionDirOf: (sp) => spaceDirOf(sp), + walkDocs: (req, sp) => walkWikiNodes(req, sp.space_id), // → [{ node, titlePath }]，titlePath 透传给 renderDoc + fileNameOf, + editTimeOf: (node) => isoOf(node.obj_edit_time), + fetchBody: async (req, node, { sleepFn, pace }) => { + if (node.obj_type === "docx") { await sleepFn(pace); return docxRawContent(req, node.obj_token); } + // 非 docx（表格/多维表…）正文不入索引，落一张指针卡指回飞书。 + const url = wikiBase ? `${wikiBase}/wiki/${node.node_token}` : `node_token=${node.node_token}`; + return `（${node.obj_type} 类型，正文不入索引——去飞书看：${url}）`; + }, + renderDoc: ({ collection, node, titlePath, body, now }) => renderDoc({ space: collection, node, titlePath, body, wikiBase, now }), + commitMessage: (written, pruned) => `feishu-docs: 同步 ${written} 篇` + (pruned ? `，清理 ${pruned}` : ""), + }; +} - let sha = null; - if (written || pruned) { - sha = await commitFn(TRUTH, { - name: "team-brain-bot", email: "bot@team-brain", - message: `feishu-docs: 同步 ${written} 篇` + (pruned ? `，清理 ${pruned}` : ""), - paths: ["feishu"], - }); - } - return { spaces: spaces.length, written, pruned, skipped, errors, commit: sha }; +// 全量对账一轮（飞书）。返回 { collections, written, pruned, skipped, errors, commit }。 +export async function syncFeishuDocs(TRUTH, req, opts = {}) { + return syncDocs(TRUTH, req, feishuProvider({ wikiBase: opts.wikiBase }), opts); } diff --git a/server/googledocs.mjs b/server/googledocs.mjs new file mode 100644 index 0000000..e5e20e6 --- /dev/null +++ b/server/googledocs.mjs @@ -0,0 +1,55 @@ +// Google Docs 文档单向镜像：把 service account 可见的文档拉进真相库 google//<标题>--.md。 +// Google Drive 没有"知识库"层级（文档平铺、靠把文档/文件夹 share 给 service account 邮箱授权）→ 用单个 +// 合成 collection（workspace）装可见的全部文档。增量靠 Drive 的 modifiedTime（本就是 ISO，直接当 edited 指纹）。 +// provider 无关的对账/增量/prune/commit 都在 ./docsync.mjs；这里只描述"Google Docs 长什么样"。 +import { buildCard } from "../core/card.mjs"; +import { redactAgent } from "../core/redact.mjs"; +import { safeSegment, saniName } from "../core/safe.mjs"; +import { listDocs, exportDocText } from "../core/google.mjs"; +import { syncDocs } from "./docsync.mjs"; + +const fileNameOf = (node) => `${saniName(node.title, "google-name")}--${safeSegment(node.id, "doc_id")}.md`; + +// 一篇文档卡：frontmatter（身份/时间/链接）+ 脱敏正文。edited 同时是增量对账的指纹（modifiedTime 已是 ISO）。 +export function renderDoc({ collection, node, body, now = "" }) { + return buildCard({ + type: "google-doc", + title: node.title || "untitled", + workspace: collection?.name || "", + doc_id: node.id, + edited: node.modifiedTime || "", + synced: now, + url: node.url || "", + }, redactAgent(body)); +} + +// Google provider adapter：把"Google Docs 长什么样"喂给通用引擎。 +function googleProvider({ workspace = "google" } = {}) { + return { + subtree: "google", + label: "google-docs", + // 单合成库：一次 list 把可见文档全捞回来挂在 collection 上（walkDocs 直接用，不重复请求）。 + // 可见文档为空 → 返回 []，触发引擎的零库保护（跳过、不删，护住已有镜像）。 + listCollections: async (req) => { + const docs = await listDocs(req); + return docs.length ? [{ id: "workspace", name: workspace, docs }] : []; + }, + collectionDirOf: (c) => saniName(c.name, "google-name"), + walkDocs: (_req, c) => c.docs.map((d) => ({ + node: { id: d.id, title: d.name, modifiedTime: d.modifiedTime, url: d.webViewLink || `https://docs.google.com/document/d/${d.id}` }, + })), + fileNameOf, + editTimeOf: (node) => node.modifiedTime || "", + fetchBody: async (req, node, { sleepFn, pace }) => { + if (pace) await sleepFn(pace); // 进每篇正文前歇一下，避开 Drive 限流 + return exportDocText(req, node.id); + }, + renderDoc, + commitMessage: (written, pruned) => `google-docs: 同步 ${written} 篇` + (pruned ? `，清理 ${pruned}` : ""), + }; +} + +// 全量对账一轮（Google Docs）。安全边界（零库保护 / 抓取失败不清理 / 路径红线）都在通用引擎里。 +export async function syncGoogleDocs(TRUTH, req, opts = {}) { + return syncDocs(TRUTH, req, googleProvider({ workspace: opts.workspace }), opts); +} diff --git a/server/notiondocs.mjs b/server/notiondocs.mjs new file mode 100644 index 0000000..c339f4b --- /dev/null +++ b/server/notiondocs.mjs @@ -0,0 +1,55 @@ +// Notion 文档单向镜像：把 integration 可见的页面拉进真相库 notion//<标题>--.md。 +// Notion 没有"知识库"层级（页面平铺、靠把页面 share 给 integration 来授权）→ 用单个合成 collection（workspace） +// 装可见的全部页面。增量靠 page.last_edited_time（本就是 ISO，直接当 edited 指纹）。 +// provider 无关的对账/增量/prune/commit 都在 ./docsync.mjs；这里只描述"Notion 长什么样"。 +import { buildCard } from "../core/card.mjs"; +import { redactAgent } from "../core/redact.mjs"; +import { safeSegment, saniName } from "../core/safe.mjs"; +import { searchPages, pageTitle, pageBlocksText } from "../core/notion.mjs"; +import { syncDocs } from "./docsync.mjs"; + +const fileNameOf = (node) => `${saniName(node.title, "notion-name")}--${safeSegment(node.id, "page_id")}.md`; + +// 一篇文档卡：frontmatter（身份/时间/链接）+ 脱敏正文。edited 同时是增量对账的指纹（last_edited_time 已是 ISO）。 +export function renderDoc({ collection, node, body, now = "" }) { + return buildCard({ + type: "notion-doc", + title: node.title || "untitled", + workspace: collection?.name || "", + page_id: node.id, + edited: node.last_edited_time || "", + synced: now, + url: node.url || "", // Notion API 直接给可点 url + }, redactAgent(body)); +} + +// Notion provider adapter：把"Notion 长什么样"喂给通用引擎。 +function notionProvider({ workspace = "notion" } = {}) { + return { + subtree: "notion", + label: "notion-docs", + // 单合成库：一次 search 把可见页面全捞回来、挂在 collection 上（walkDocs 直接用，不重复请求）。 + // 可见页面为空 → 返回 []，触发引擎的零库保护（跳过、不删，护住已有镜像）。 + listCollections: async (req) => { + const pages = await searchPages(req); + return pages.length ? [{ id: "workspace", name: workspace, pages }] : []; + }, + collectionDirOf: (c) => saniName(c.name, "notion-name"), + walkDocs: (_req, c) => c.pages.map((p) => ({ + node: { id: p.id, title: pageTitle(p), last_edited_time: p.last_edited_time, url: p.url }, + })), + fileNameOf, + editTimeOf: (node) => node.last_edited_time || "", + fetchBody: async (req, node, { sleepFn, pace }) => { + if (pace) await sleepFn(pace); // 进每页正文前歇一下，避开 Notion ~3 req/s 限流 + return pageBlocksText(req, node.id, { sleepFn, pace }); + }, + renderDoc, + commitMessage: (written, pruned) => `notion-docs: 同步 ${written} 篇` + (pruned ? `，清理 ${pruned}` : ""), + }; +} + +// 全量对账一轮（Notion）。安全边界（零库保护 / 抓取失败不清理 / 路径红线）都在通用引擎里。 +export async function syncNotionDocs(TRUTH, req, opts = {}) { + return syncDocs(TRUTH, req, notionProvider({ workspace: opts.workspace }), opts); +} diff --git a/server/server.mjs b/server/server.mjs index fa11f91..4530f4d 100644 --- a/server/server.mjs +++ b/server/server.mjs @@ -17,11 +17,15 @@ import { log } from "../core/log.mjs"; import { CLIENT_VERSION } from "../core/version.mjs"; // 服务器自身版本 == 打进 /client.tgz 的版本（启动重打包）→ 权威「最新客户端版本」 import { safeSegment, safeRelPath } from "../core/safe.mjs"; import { loadFeishu, makeReq } from "../core/feishu.mjs"; +import { loadNotion, makeReq as makeNotionReq } from "../core/notion.mjs"; +import { loadGoogle, makeReq as makeGoogleReq } from "../core/google.mjs"; import { initTruth } from "./gitstore.mjs"; import { ingest } from "./ingest.mjs"; import { refreshAll, enumAndRegisterOrgRepos } from "./codestate.mjs"; import { readSpaceMeta } from "./space.mjs"; import { syncFeishuDocs } from "./feishudocs.mjs"; +import { syncNotionDocs } from "./notiondocs.mjs"; +import { syncGoogleDocs } from "./googledocs.mjs"; import { grepTruth, findTruth, lsTruth, logTruth, sessionsTruth, statsTruth, frontmatterOf, spaceStatsTruth } from "./query.mjs"; import { canonicalizePath, canonicalSpaceKey } from "../core/identity.mjs"; @@ -59,6 +63,8 @@ const GITHUB_TOKEN = process.env.GITHUB_TOKEN || (process.env.GITHUB_TOKEN_FILE && existsSync(process.env.GITHUB_TOKEN_FILE) ? readFileSync(process.env.GITHUB_TOKEN_FILE, "utf8").trim() : ""); const FEISHU = loadFeishu(process.env.FEISHU_FILE || join(ROOT, "feishu.yaml")); // 文档层（飞书）凭证：缺则不启用 +const NOTION = loadNotion(process.env.NOTION_FILE || join(ROOT, "notion.yaml")); // 文档层（Notion）凭证：缺则不启用 +const GOOGLE = loadGoogle(process.env.GOOGLE_FILE || join(ROOT, "google.yaml")); // 文档层（Google Docs）凭证：缺则不启用 initTruth(TRUTH); // 「问一句」理解层（可选）：复用服务器上已装的 codex（或任意 agent CLI）当引擎。 @@ -73,7 +79,7 @@ const ASK = { max: Number(process.env.ASK_MAX_CONCURRENT) || 2, // 并发上限，挡住成本/滥用 }; let askInflight = 0; -const ASK_HINT = "你在团队大脑「真相库」（一个 git 仓）当前目录里回答问题。库里有：spaces//sessions/**/*.md（全队脱敏对话 transcript）、feishu/**/*.md（飞书文档镜像）、spaces//code-state.md（代码状态）。请用 grep/read 翻这些 .md 文件，给出综合答复，并在末尾列出你引用的文件 path。不要读 .jsonl（用 .md）。问题：\n\n"; +const ASK_HINT = "你在团队大脑「真相库」（一个 git 仓）当前目录里回答问题。库里有：spaces//sessions/**/*.md（全队脱敏对话 transcript）、feishu/·notion/·google/**/*.md（人写文档镜像：飞书 wiki / Notion / Google Docs）、spaces//code-state.md（代码状态）。请用 grep/read 翻这些 .md 文件，给出综合答复，并在末尾列出你引用的文件 path。不要读 .jsonl（用 .md）。问题：\n\n"; // 从 codex 输出里抽最终答复：--json 模式解 JSONL 取末条 agent 消息；否则直接回 stdout。 const extractAnswer = (out) => { const s = String(out || "").trim(); @@ -294,7 +300,7 @@ async function handle(req, res, u) { // --- 能力位（Web GUI 据此决定显隐可选功能）--- if (req.method === "GET" && u.pathname === "/capabilities") { if (!authMember(req)) return json(res, 401, { error: "invalid token" }); - return json(res, 200, { ask: ASK.enabled, feishu: !!FEISHU }); + return json(res, 200, { ask: ASK.enabled, feishu: !!FEISHU, notion: !!NOTION, google: !!GOOGLE }); } // --- 「问一句」：spawn 服务器上的 codex 现查真相库 → 综合答复（可选，ASK_ENABLED 才开）--- @@ -500,7 +506,7 @@ log.info(ASK.enabled ? "[ask] 「问一句」已启用" : "[ask] 「问一句」 // 本地静态开发模式：NO_POLL=1 跳过两个后台轮询（不出网、不改写 TRUTH）→ 拿静态 fixtures 开发 web GUI const NO_POLL = process.env.NO_POLL === "1"; -if (NO_POLL) log.info("[no-poll] 后台轮询已关（NO_POLL=1）：code-state / feishu 同步跳过，TRUTH 保持静态"); +if (NO_POLL) log.info("[no-poll] 后台轮询已关（NO_POLL=1）：code-state / feishu / notion / google 同步跳过，TRUTH 保持静态"); // code-state 4h 轮询（registry 有任一 provider 登记、或配了全局 GITHUB_TOKEN 就启用） if (!NO_POLL && (GITHUB_TOKEN || hasAnyRemote(registry))) { @@ -518,15 +524,23 @@ if (!NO_POLL && (GITHUB_TOKEN || hasAnyRemote(registry))) { log.info("[code-state] 未配 registry 也无 GITHUB_TOKEN → code-state 轮询 / read_github 暂不启用"); } -// 飞书文档镜像轮询（配了 feishu.yaml 就启用）：单向拉 wiki 正文进 feishu/ 子树，grep/read 即可搜 -if (!NO_POLL && FEISHU) { - const freq = makeReq(FEISHU); - const ftick = () => syncFeishuDocs(TRUTH, freq, { wikiBase: FEISHU.wiki_base }) - .then((r) => log.info("[feishu-docs] 同步完成", { spaces: r.spaces, written: r.written, pruned: r.pruned, skipped: r.skipped, errors: r.errors })) - .catch((e) => log.error("[feishu-docs] 失败", { err: e.message })); - setTimeout(ftick, 60_000); // 启动 60s 后首跑（错开 code-state 的 30s） - setInterval(ftick, FEISHU.poll_hours * 3600 * 1000); - log.info("[feishu-docs] 轮询已开", { hours: FEISHU.poll_hours }); -} else { - log.info("[feishu-docs] 未配 feishu.yaml → 文档层暂不启用"); +// 文档源镜像轮询：配了对应 *.yaml 就启用，单向拉正文进 <子树>/，grep/read 即可搜。 +// 各源同一形态（增量对账引擎在 docsync.mjs），用一张表驱动——加新源只加一行，错峰首跑避免一起出网。 +const DOC_SOURCES = [ + { label: "feishu", file: "feishu.yaml", cfg: FEISHU, makeReq, sync: syncFeishuDocs, opts: (c) => ({ wikiBase: c.wiki_base }), offset: 60_000 }, + { label: "notion", file: "notion.yaml", cfg: NOTION, makeReq: makeNotionReq, sync: syncNotionDocs, opts: (c) => ({ workspace: c.workspace }), offset: 90_000 }, + { label: "google", file: "google.yaml", cfg: GOOGLE, makeReq: makeGoogleReq, sync: syncGoogleDocs, opts: (c) => ({ workspace: c.workspace }), offset: 120_000 }, +]; +for (const s of DOC_SOURCES) { + if (!NO_POLL && s.cfg) { + const req = s.makeReq(s.cfg); + const tick = () => s.sync(TRUTH, req, s.opts(s.cfg)) + .then((r) => log.info(`[${s.label}-docs] 同步完成`, { collections: r.collections, written: r.written, pruned: r.pruned, skipped: r.skipped, errors: r.errors })) + .catch((e) => log.error(`[${s.label}-docs] 失败`, { err: e.message })); + setTimeout(tick, s.offset); // 启动后错峰首跑（code-state 30s 之后，各源 60/90/120s） + setInterval(tick, s.cfg.poll_hours * 3600 * 1000); + log.info(`[${s.label}-docs] 轮询已开`, { hours: s.cfg.poll_hours }); + } else { + log.info(`[${s.label}-docs] 未配 ${s.file} → 文档层暂不启用`); + } } diff --git a/test/google.test.mjs b/test/google.test.mjs new file mode 100644 index 0000000..28cc8a0 --- /dev/null +++ b/test/google.test.mjs @@ -0,0 +1,173 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync, readFileSync, existsSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { loadGoogle, withRetry, listDocs } from "../core/google.mjs"; +import { syncGoogleDocs, renderDoc } from "../server/googledocs.mjs"; +import { fm } from "../core/card.mjs"; + +// ---------- 测试用假 Google：world 驱动（Drive 文件列表 / 各文档导出文本），可数 export 调用 ---------- +const doc = (id, name, { modified = "2026-06-01T00:00:00.000Z" } = {}) => + ({ id, name, modifiedTime: modified, webViewLink: "https://docs.google.com/document/d/" + id }); + +const fakeReq = (world) => async (method, path, { raw = false } = {}) => { + if (method === "GET" && path.startsWith("/files?")) { + if (world.failList) throw Object.assign(new Error("list boom"), { status: 500 }); + return { files: world.docs, nextPageToken: "" }; + } + const m = path.match(/^\/files\/([^/?]+)\/export/); + if (method === "GET" && m && raw) { + world.exportCalls = (world.exportCalls || 0) + 1; + return world.content[m[1]] ?? ""; + } + throw new Error("unexpected: " + method + " " + path + " raw=" + raw); +}; + +const mkWorld = () => ({ + docs: [ + doc("docA", "PRD 主文档", { modified: "2026-06-01T00:00:00.000Z" }), + doc("docB", "客户笔记", { modified: "2026-06-02T00:00:00.000Z" }), + ], + content: { docA: "正文A，含密钥 ghp_" + "A".repeat(36), docB: "正文C" }, + exportCalls: 0, +}); + +const syncOpts = (commits) => ({ + pace: 0, sleepFn: async () => {}, now: () => "2026-06-10T00:00:00.000Z", workspace: "Team GDrive", + commitFn: async (_dir, info) => { commits.push(info); return "sha-test"; }, +}); + +// ---------- loadGoogle ---------- +test("loadGoogle：缺文件 / 缺凭证 → null；内联凭证与 key_file 两条路都通；默认值生效", () => { + const dir = mkdtempSync(join(tmpdir(), "gg-cfg-")); + assert.equal(loadGoogle(join(dir, "nope.yaml")), null); + writeFileSync(join(dir, "bad.yaml"), "poll_hours: 6\n"); // 没凭证 + assert.equal(loadGoogle(join(dir, "bad.yaml")), null); + + // ① 内联 + writeFileSync(join(dir, "inline.yaml"), "client_email: sa@x.iam.gserviceaccount.com\nprivate_key: KEYDATA\n"); + const c1 = loadGoogle(join(dir, "inline.yaml")); + assert.equal(c1.client_email, "sa@x.iam.gserviceaccount.com"); + assert.equal(c1.poll_hours, 4); // 默认 4h + assert.equal(c1.workspace, "google"); // 默认目录名 + + // ② key_file 指向 service-account JSON + writeFileSync(join(dir, "sa.json"), JSON.stringify({ client_email: "sa2@x.iam.gserviceaccount.com", private_key: "PEMDATA" })); + writeFileSync(join(dir, "ext.yaml"), `key_file: ${join(dir, "sa.json")}\nworkspace: KB\n`); + const c2 = loadGoogle(join(dir, "ext.yaml")); + assert.equal(c2.client_email, "sa2@x.iam.gserviceaccount.com"); + assert.equal(c2.private_key, "PEMDATA"); + assert.equal(c2.workspace, "KB"); +}); + +// ---------- withRetry ---------- +test("withRetry：429/5xx 退避重试后成功；4xx 业务错误不重试", async () => { + let n = 0; + const flaky = async () => { if (++n < 3) throw Object.assign(new Error("503"), { status: 503 }); return "ok"; }; + assert.equal(await withRetry(flaky, { delays: [0, 0, 0], sleepFn: async () => {} }), "ok"); + assert.equal(n, 3); + let tries = 0; + const denied = async () => { tries++; throw Object.assign(new Error("404"), { status: 404 }); }; + await assert.rejects(() => withRetry(denied, { delays: [0, 0], sleepFn: async () => {} }), /404/); + assert.equal(tries, 1); +}); + +// ---------- listDocs 分页 ---------- +test("listDocs：翻页拼全（nextPageToken）", async () => { + const pages = [ + { files: [doc("d1", "A")], nextPageToken: "p2" }, + { files: [doc("d2", "B")], nextPageToken: "" }, + ]; + let i = 0; + const req = async (method, path) => { + assert.equal(method, "GET"); + assert.match(path, /^\/files\?/); + if (i === 1) assert.match(path, /pageToken=p2/); // 第二页带上一页游标 + return pages[i++]; + }; + const out = await listDocs(req); + assert.deepEqual(out.map((d) => d.id), ["d1", "d2"]); +}); + +// ---------- renderDoc ---------- +test("renderDoc：frontmatter 齐全 + 正文脱敏 + url", () => { + const txt = renderDoc({ + collection: { id: "workspace", name: "Team GDrive" }, + node: { id: "docA", title: "PRD 主文档", modifiedTime: "2026-06-01T00:00:00.000Z", url: "https://docs.google.com/document/d/docA" }, + body: "密钥 ghp_" + "A".repeat(36), + now: "2026-06-10T00:00:00.000Z", + }); + assert.equal(fm(txt, "type"), "google-doc"); + assert.equal(fm(txt, "edited"), "2026-06-01T00:00:00.000Z"); + assert.equal(fm(txt, "workspace"), "Team GDrive"); + assert.equal(fm(txt, "doc_id"), "docA"); + assert.equal(fm(txt, "url"), "https://docs.google.com/document/d/docA"); + assert.match(txt, /\[REDACTED_GH\]/); + assert.doesNotMatch(txt, /ghp_A/); +}); + +// ---------- sync 端到端 ---------- +test("sync：首轮全写、export 拉正文脱敏、一轮一个 commit", async () => { + const TRUTH = mkdtempSync(join(tmpdir(), "gg-truth-")); + const world = mkWorld(); const commits = []; + const r = await syncGoogleDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.deepEqual([r.written, r.pruned, r.skipped, r.errors], [2, 0, 0, 0]); + const dir = join(TRUTH, "google", "Team GDrive"); + assert.match(readFileSync(join(dir, "PRD 主文档--docA.md"), "utf8"), /\[REDACTED_GH\]/); + assert.match(readFileSync(join(dir, "客户笔记--docB.md"), "utf8"), /正文C/); + assert.equal(world.exportCalls, 2); + assert.equal(commits.length, 1); + assert.match(commits[0].message, /同步 2 篇/); + assert.deepEqual(commits[0].paths, ["google"]); + rmSync(TRUTH, { recursive: true, force: true }); +}); + +test("sync 增量：modifiedTime 没变 → 不导出、不重写、不 commit；改一篇只重拉一篇", async () => { + const TRUTH = mkdtempSync(join(tmpdir(), "gg-truth-")); + const world = mkWorld(); const commits = []; + await syncGoogleDocs(TRUTH, fakeReq(world), syncOpts(commits)); + const r2 = await syncGoogleDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.deepEqual([r2.written, r2.skipped], [0, 2]); + assert.equal(world.exportCalls, 2); // 第二轮零次 export + assert.equal(commits.length, 1); + + world.docs[1] = doc("docB", "客户笔记", { modified: "2026-06-09T00:00:00.000Z" }); + world.content.docB = "正文C v2"; + const r3 = await syncGoogleDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.deepEqual([r3.written, r3.skipped], [1, 1]); + assert.match(readFileSync(join(TRUTH, "google", "Team GDrive", "客户笔记--docB.md"), "utf8"), /正文C v2/); + rmSync(TRUTH, { recursive: true, force: true }); +}); + +test("sync 孤儿清理 + 安全边界 + 路径红线", async () => { + const TRUTH = mkdtempSync(join(tmpdir(), "gg-truth-")); + const world = mkWorld(); const commits = []; + await syncGoogleDocs(TRUTH, fakeReq(world), syncOpts(commits)); + const live = join(TRUTH, "google", "Team GDrive", "PRD 主文档--docA.md"); + + // 孤儿清理：docB 没了 → 镜像删掉 + world.docs = [doc("docA", "PRD 主文档")]; + const r1 = await syncGoogleDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.equal(r1.pruned, 1); + assert.ok(!existsSync(join(TRUTH, "google", "Team GDrive", "客户笔记--docB.md"))); + + // list 失败 → 不清理 + world.failList = true; + const r2 = await syncGoogleDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.equal(r2.errors, 1); + assert.ok(existsSync(live)); + + // 可见文档为空 → 整轮跳过且不删 + world.failList = false; world.docs = []; + const r3 = await syncGoogleDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.equal(r3.errors, 1); + assert.ok(existsSync(live)); + + // 路径红线：doc id 含分隔符 → 跳过不落盘，其余照写 + world.docs = [doc("docA", "PRD 主文档"), doc("../evil", "恶意")]; + const r4 = await syncGoogleDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.ok(r4.errors >= 1); + assert.ok(!existsSync(join(TRUTH, "google", "Team GDrive", "恶意--../evil.md"))); + rmSync(TRUTH, { recursive: true, force: true }); +}); diff --git a/test/notion.test.mjs b/test/notion.test.mjs new file mode 100644 index 0000000..fa19555 --- /dev/null +++ b/test/notion.test.mjs @@ -0,0 +1,199 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync, readFileSync, existsSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { loadNotion, withRetry, searchPages, pageTitle, pageBlocksText } from "../core/notion.mjs"; +import { syncNotionDocs, renderDoc } from "../server/notiondocs.mjs"; +import { fm } from "../core/card.mjs"; + +// ---------- 测试用假 Notion：world 驱动（pages / 每页 block 树），可数 children 调用 ---------- +const page = (id, title, { edited = "2026-06-01T00:00:00.000Z", url = "https://www.notion.so/" + id } = {}) => ({ + object: "page", id, url, last_edited_time: edited, + properties: { Name: { type: "title", title: [{ plain_text: title }] } }, +}); +const para = (id, text, { has_children = false } = {}) => + ({ id, type: "paragraph", has_children, paragraph: { rich_text: [{ plain_text: text }] } }); + +const fakeReq = (world) => async (method, path) => { + if (method === "POST" && path === "/search") { + if (world.failSearch) throw Object.assign(new Error("search boom"), { status: 500 }); + return { results: world.pages, has_more: false }; + } + const m = path.match(/^\/blocks\/([^/?]+)\/children/); + if (method === "GET" && m) { + world.blockCalls = (world.blockCalls || 0) + 1; + return { results: world.blocks[m[1]] || [], has_more: false }; + } + throw new Error("unexpected: " + method + " " + path); +}; + +const mkWorld = () => ({ + pages: [ + page("pageA", "PRD 主文档", { edited: "2026-06-01T00:00:00.000Z" }), + page("pageB", "客户笔记", { edited: "2026-06-02T00:00:00.000Z" }), + ], + blocks: { + pageA: [para("a1", "正文A，含密钥 ghp_" + "A".repeat(36))], + pageB: [para("b1", "正文C")], + }, + blockCalls: 0, +}); + +const syncOpts = (commits) => ({ + pace: 0, sleepFn: async () => {}, now: () => "2026-06-10T00:00:00.000Z", workspace: "Team Notion", + commitFn: async (_dir, info) => { commits.push(info); return "sha-test"; }, +}); + +// ---------- loadNotion ---------- +test("loadNotion：缺文件 / 缺 token → null；配齐 → 默认值生效", () => { + const dir = mkdtempSync(join(tmpdir(), "nt-cfg-")); + assert.equal(loadNotion(join(dir, "nope.yaml")), null); + writeFileSync(join(dir, "bad.yaml"), "poll_hours: 6\n"); // 没 api_token + assert.equal(loadNotion(join(dir, "bad.yaml")), null); + writeFileSync(join(dir, "ok.yaml"), "api_token: secret_x\n"); + const c = loadNotion(join(dir, "ok.yaml")); + assert.equal(c.poll_hours, 4); // 默认 4h + assert.equal(c.workspace, "notion"); // 默认目录名 + writeFileSync(join(dir, "ok2.yaml"), "api_token: secret_x\npoll_hours: 6\nworkspace: KB\n"); + assert.deepEqual( + (({ poll_hours, workspace }) => ({ poll_hours, workspace }))(loadNotion(join(dir, "ok2.yaml"))), + { poll_hours: 6, workspace: "KB" }); +}); + +// ---------- withRetry ---------- +test("withRetry：429/5xx 退避重试后成功；4xx 业务错误不重试", async () => { + let n = 0; + const flaky = async () => { if (++n < 3) throw Object.assign(new Error("429"), { status: 429 }); return "ok"; }; + assert.equal(await withRetry(flaky, { delays: [0, 0, 0], sleepFn: async () => {} }), "ok"); + assert.equal(n, 3); + let tries = 0; + const denied = async () => { tries++; throw Object.assign(new Error("403"), { status: 403 }); }; + await assert.rejects(() => withRetry(denied, { delays: [0, 0], sleepFn: async () => {} }), /403/); + assert.equal(tries, 1); // 权限错误重试无意义 +}); + +// ---------- pageTitle / searchPages 分页 ---------- +test("pageTitle：取 type=title 的属性拼 plain_text；缺则 untitled", () => { + assert.equal(pageTitle(page("p", "我的页")), "我的页"); + assert.equal(pageTitle({ properties: { N: { type: "rich_text", rich_text: [] } } }), "untitled"); + assert.equal(pageTitle({}), "untitled"); +}); + +test("searchPages：翻页拼全（cursor 在 body）", async () => { + const pages = [ + { results: [page("p1", "A")], has_more: true, next_cursor: "c2" }, + { results: [page("p2", "B")], has_more: false }, + ]; + let i = 0; + const req = async (method, path, body) => { + assert.equal(method, "POST"); assert.equal(path, "/search"); + if (i === 1) assert.equal(body.start_cursor, "c2"); // 第二页带上一页游标 + return pages[i++]; + }; + const out = await searchPages(req); + assert.deepEqual(out.map((p) => p.id), ["p1", "p2"]); +}); + +// ---------- pageBlocksText：递归 + 子块下钻 ---------- +test("pageBlocksText：拼各块文本、has_children 下钻", async () => { + const world = { + blocks: { + root: [para("x1", "第一段"), para("x2", "带子块", { has_children: true })], + x2: [para("x3", "子段")], + }, + }; + const txt = await pageBlocksText(fakeReq(world), "root", { pace: 0, sleepFn: async () => {} }); + assert.equal(txt, "第一段\n带子块\n子段"); +}); + +// ---------- renderDoc ---------- +test("renderDoc：frontmatter 齐全 + 正文脱敏 + url", () => { + const txt = renderDoc({ + collection: { id: "workspace", name: "Team Notion" }, + node: { id: "pageA", title: "PRD 主文档", last_edited_time: "2026-06-01T00:00:00.000Z", url: "https://www.notion.so/pageA" }, + body: "密钥 ghp_" + "A".repeat(36), + now: "2026-06-10T00:00:00.000Z", + }); + assert.equal(fm(txt, "type"), "notion-doc"); + assert.equal(fm(txt, "edited"), "2026-06-01T00:00:00.000Z"); + assert.equal(fm(txt, "workspace"), "Team Notion"); + assert.equal(fm(txt, "page_id"), "pageA"); + assert.equal(fm(txt, "url"), "https://www.notion.so/pageA"); + assert.match(txt, /\[REDACTED_GH\]/); // 派生必脱敏 + assert.doesNotMatch(txt, /ghp_A/); +}); + +// ---------- sync 端到端 ---------- +test("sync：首轮全写、拉正文脱敏、一轮一个 commit", async () => { + const TRUTH = mkdtempSync(join(tmpdir(), "nt-truth-")); + const world = mkWorld(); const commits = []; + const r = await syncNotionDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.deepEqual([r.written, r.pruned, r.skipped, r.errors], [2, 0, 0, 0]); + const dir = join(TRUTH, "notion", "Team Notion"); + assert.match(readFileSync(join(dir, "PRD 主文档--pageA.md"), "utf8"), /\[REDACTED_GH\]/); + assert.match(readFileSync(join(dir, "客户笔记--pageB.md"), "utf8"), /正文C/); + assert.equal(world.blockCalls, 2); // 两页各拉一次正文 + assert.equal(commits.length, 1); + assert.match(commits[0].message, /同步 2 篇/); + assert.deepEqual(commits[0].paths, ["notion"]); + rmSync(TRUTH, { recursive: true, force: true }); +}); + +test("sync 增量：last_edited_time 没变 → 不拉正文、不重写、不 commit；改一篇只重拉一篇", async () => { + const TRUTH = mkdtempSync(join(tmpdir(), "nt-truth-")); + const world = mkWorld(); const commits = []; + await syncNotionDocs(TRUTH, fakeReq(world), syncOpts(commits)); + const r2 = await syncNotionDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.deepEqual([r2.written, r2.skipped], [0, 2]); + assert.equal(world.blockCalls, 2); // 第二轮零次 children + assert.equal(commits.length, 1); + + world.pages[1] = page("pageB", "客户笔记", { edited: "2026-06-09T00:00:00.000Z" }); + world.blocks.pageB = [para("b1", "正文C v2")]; + const r3 = await syncNotionDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.deepEqual([r3.written, r3.skipped], [1, 1]); + assert.match(readFileSync(join(TRUTH, "notion", "Team Notion", "客户笔记--pageB.md"), "utf8"), /正文C v2/); + rmSync(TRUTH, { recursive: true, force: true }); +}); + +test("sync 孤儿清理：页面消失 → 镜像删掉；commit 带清理数", async () => { + const TRUTH = mkdtempSync(join(tmpdir(), "nt-truth-")); + const world = mkWorld(); const commits = []; + await syncNotionDocs(TRUTH, fakeReq(world), syncOpts(commits)); + world.pages = [page("pageA", "PRD 主文档")]; // pageB 没了 + const r = await syncNotionDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.equal(r.pruned, 1); + assert.ok(!existsSync(join(TRUTH, "notion", "Team Notion", "客户笔记--pageB.md"))); + assert.match(commits.at(-1).message, /清理 1/); + rmSync(TRUTH, { recursive: true, force: true }); +}); + +test("sync 安全边界：search 失败不清理；可见页面为空整轮跳过", async () => { + const TRUTH = mkdtempSync(join(tmpdir(), "nt-truth-")); + const world = mkWorld(); const commits = []; + await syncNotionDocs(TRUTH, fakeReq(world), syncOpts(commits)); + const live = join(TRUTH, "notion", "Team Notion", "PRD 主文档--pageA.md"); + // ① search 挂了 → 已有镜像原样保留 + world.failSearch = true; + const r1 = await syncNotionDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.equal(r1.errors, 1); + assert.ok(existsSync(live)); + // ② 一个页面都看不到（多半凭证/授权坏了）→ 跳过且不删 + world.failSearch = false; world.pages = []; + const r2 = await syncNotionDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.equal(r2.errors, 1); + assert.ok(existsSync(live)); + rmSync(TRUTH, { recursive: true, force: true }); +}); + +test("sync 路径红线：page id 含分隔符 → 该页跳过不落盘，其余照写", async () => { + const TRUTH = mkdtempSync(join(tmpdir(), "nt-truth-")); + const world = mkWorld(); const commits = []; + world.pages.push(page("../evil", "恶意")); + const r = await syncNotionDocs(TRUTH, fakeReq(world), syncOpts(commits)); + assert.ok(r.errors >= 1); + assert.equal(r.written, 2); // 其余 2 篇照常 + assert.ok(!existsSync(join(TRUTH, "notion", "Team Notion", "恶意--../evil.md"))); + rmSync(TRUTH, { recursive: true, force: true }); +}); diff --git a/web/app.js b/web/app.js index 6cf57cd..d6598a9 100644 --- a/web/app.js +++ b/web/app.js @@ -16,7 +16,7 @@ const I18N = { en: { // 静态（index.html） "doc.title": "team-brain · truth layer", - "doc.desc": "The truth layer of the team project brain — browse the whole team's sessions, Feishu docs, code state and activity.", + "doc.desc": "The truth layer of the team project brain — browse the whole team's sessions, team docs, code state and activity.", "nav.menu": "Menu", "brand.tag": "truth layer", "search.ph": "grep the team truth repo… ( / to focus)", @@ -69,7 +69,7 @@ const I18N = { "ov.attentionHead": "⚠️ Needs attention · unpushed", "ov.allRepos": "All repos →", "ov.recentDocs": "Recent docs", - "ov.feishuDocs": "Feishu docs →", + "ov.feishuDocs": "Docs →", "load.activity": "Loading activity", "load.scanCode": "Scanning code state", "load.docs": "Loading docs", @@ -138,7 +138,7 @@ const I18N = { // 搜索 "search.crumb": "Search", "search.title": "Search the truth repo", - "search.sub": "git grep full-text across all team sessions (redacted transcripts) and Feishu docs. Regex supported.", + "search.sub": "git grep full-text across all team sessions (redacted transcripts) and team docs. Regex supported.", "search.qPh": "regex / keyword…", "search.raw": "incl. raw jsonl", "search.btn": "Search", @@ -156,8 +156,8 @@ const I18N = { "act.linkTitle": "Copy web link (share with people, click to open)", "act.code": "Code", "act.codeTitle": (br) => `Open on GitHub${br ? ` (${br})` : ""}`, - "act.feishu": "Feishu", - "act.feishuTitle": "Open the original in Feishu", + "act.feishu": "Original", + "act.feishuTitle": "Open the original (Feishu / Notion / Google Docs)", "act.aside": "Aside", "act.asideTitle": (n) => `Show / hide assistant asides (${n})`, "act.raw": "Raw", @@ -167,11 +167,11 @@ const I18N = { "act.copyFail": "Copy failed", "read.agentMsg": (path) => `Read and explain this record with team-brain, as context for the discussion that follows: ${path}`, "meta.detail": "Details", - "meta.feishuOriginal": "Feishu original ↗", + "meta.feishuOriginal": "Original ↗", // 文档 - "docs.noMirror": "No Feishu doc mirror yet.", - "docs.noMirrorDesc": "Once the server has feishu.yaml configured and the app added to the wiki, docs auto-mirror in every 4h.", - "docs.title": "Feishu docs", + "docs.noMirror": "No doc mirror yet.", + "docs.noMirrorDesc": "Configure a doc source on the server (feishu.yaml / notion.yaml / google.yaml) and docs auto-mirror in roughly every 4h.", + "docs.title": "Team docs", "docs.wikis": "Wikis / folders", "docs.docs": "Docs", "docs.items": (n) => `${n} items`, @@ -221,7 +221,7 @@ const I18N = { }, zh: { "doc.title": "team-brain · 真相层", - "doc.desc": "团队项目大脑的真相层 —— 浏览全队 session、飞书文档、代码状态与活动流。", + "doc.desc": "团队项目大脑的真相层 —— 浏览全队 session、团队文档、代码状态与活动流。", "nav.menu": "菜单", "brand.tag": "真相层", "search.ph": "grep 全队真相库… ( / 聚焦)", @@ -270,7 +270,7 @@ const I18N = { "ov.attentionHead": "⚠️ 待关注 · 未推进度", "ov.allRepos": "全部仓库 →", "ov.recentDocs": "最近文档", - "ov.feishuDocs": "飞书文档 →", + "ov.feishuDocs": "文档 →", "load.activity": "加载活动", "load.scanCode": "扫描代码状态", "load.docs": "加载文档", @@ -331,7 +331,7 @@ const I18N = { "tag.draft": "个人草稿", "search.crumb": "搜索", "search.title": "搜索真相库", - "search.sub": "git grep 全文检索全队 session（脱敏 transcript）与飞书文档。支持正则。", + "search.sub": "git grep 全文检索全队 session（脱敏 transcript）与团队文档。支持正则。", "search.qPh": "正则 / 关键词…", "search.raw": "连原文 jsonl", "search.btn": "搜索", @@ -348,8 +348,8 @@ const I18N = { "act.linkTitle": "复制网页链接（分享给人，点开即看）", "act.code": "代码", "act.codeTitle": (br) => `在 GitHub 打开${br ? `（${br}）` : ""}`, - "act.feishu": "飞书", - "act.feishuTitle": "在飞书打开原文", + "act.feishu": "原文档", + "act.feishuTitle": "在源平台打开原文（飞书 / Notion / Google Docs）", "act.aside": "旁白", "act.asideTitle": (n) => `显示 / 隐藏过程旁白（${n} 条）`, "act.raw": "原文", @@ -359,10 +359,10 @@ const I18N = { "act.copyFail": "复制失败", "read.agentMsg": (path) => `用 team-brain 读取并讲解这条记录，作为接下来讨论的上下文：${path}`, "meta.detail": "详情", - "meta.feishuOriginal": "飞书原文 ↗", - "docs.noMirror": "还没有飞书文档镜像。", - "docs.noMirrorDesc": "服务器配了 feishu.yaml 并把应用加进知识库后，每 4h 自动镜像进来。", - "docs.title": "飞书文档", + "meta.feishuOriginal": "原文 ↗", + "docs.noMirror": "还没有文档镜像。", + "docs.noMirrorDesc": "服务器配好任一文档源（feishu.yaml / notion.yaml / google.yaml）后，每约 4h 自动镜像进来。", + "docs.title": "团队文档", "docs.wikis": "知识库 / 目录", "docs.docs": "文档", "docs.items": (n) => `${n} 项`, @@ -783,16 +783,18 @@ async function viewOverview() { `; + // 文档源拉取共用一次扇出（统计格子 + 下方「最近文档」两处复用，避免对每个源重复 /find） + const docFilesP = fetchDocFiles(500); + // 统计 + 活动（快）；待关注计数由下方扫描回填 Promise.all([ getSpaces(), api("/sessions", { limit: 1 }).catch(() => ({ total: 0 })), api("/log", { limit: 8 }).catch(() => ({ commits: [] })), - api("/ls", { path: "feishu" }).catch(() => ({ entries: [] })), - ]).then(([spaces, ses, lg, feishu]) => { + docFilesP, + ]).then(([spaces, ses, lg, docFiles]) => { const active = spaces.filter((e) => isRepoSpace(e.name) && e.active).length; - const docs = (feishu.entries || []).reduce((a, e) => a + (e.children || 0), 0); - setStat(0, active); setStat(1, ses.total ?? 0); setStat(2, docs); + setStat(0, active); setStat(1, ses.total ?? 0); setStat(2, docFiles.length); const act = $("#ov-activity"); if (act) act.innerHTML = (lg.commits || []).length ? timelineHtml(lg.commits) : emptyNote(t("ov.noActivity")); }); @@ -818,7 +820,7 @@ async function viewOverview() { (async () => { const box = $("#ov-docs"); if (!box) return; let files = []; - try { files = (await api("/find", { path: "feishu", name: "*.md", meta: 1, limit: 200 })).files || []; } catch {} + try { files = await docFilesP; } catch {} if (!files.length) { box.innerHTML = emptyNote(t("ov.noMirror")); return; } const docs = files.map((f) => ({ path: f.path, ...(f.meta || {}) })) .sort((a, b) => String(b.edited || "").localeCompare(String(a.edited || ""))).slice(0, 5); @@ -1286,7 +1288,7 @@ async function viewRead(q) { ? actBtn("external", t("act.feishu"), { tag: "a", title: t("act.feishuTitle"), attrs: `href="${esc(meta.url)}" target="_blank" rel="noopener"` }) : ""; const crumbParts = [crumbHome()]; if (spaceKey) crumbParts.push(`${esc(spaceLabel(spaceKey).name)}`); - else if (parts[0] === "feishu") crumbParts.push(`${esc(t("nav.docs"))}`); + else if (DOC_ROOTS.includes(parts[0])) crumbParts.push(`${esc(t("nav.docs"))}`); crumbParts.push(`${esc(parts[parts.length - 1])}`); main.innerHTML = `
@@ -1340,27 +1342,46 @@ function metaCard(meta) { }).join(""); } -/* ============================================================ 文档（飞书） ============================================================ */ +/* ============================================================ 文档（飞书 / Notion / Google Docs） ============================================================ */ +const DOC_ROOTS = ["feishu", "notion", "google"]; // 人写文档镜像子树（与服务端 provider.subtree 对齐） + +// 跨所有文档源拉 .md（含 frontmatter meta），供总览统计 / 最近文档用。某源不存在则跳过。 +async function fetchDocFiles(limit = 200) { + const lists = await Promise.all(DOC_ROOTS.map((r) => + api("/find", { path: r, name: "*.md", meta: 1, limit }).catch(() => ({ files: [] })))); + return lists.flatMap((l) => l.files || []); +} + async function viewDocs(q) { - const path = q.path || "feishu"; + const path = q.path || ""; // 空 = 文档源总览（列出存在的 feishu/notion/google 子树） main.innerHTML = loading(t("load.docs")); - let entries; - try { entries = (await api("/ls", { path })).entries || []; } - catch (e) { - if (e.code === 401) { main.innerHTML = errView(e); return; } + const noMirror = () => { main.innerHTML = `
${crumb(crumbHome(), `${esc(t("nav.docs"))}`)}
${esc(t("docs.noMirror"))}

${esc(t("docs.noMirrorDesc"))}

`; - highlightSidebar(); return; + highlightSidebar(); + }; + let entries; + try { + if (path) entries = (await api("/ls", { path })).entries || []; + else { // 总览：ls 真相库根，只留存在的文档源子树 + const root = (await api("/ls", { path: "" }).catch(() => ({ entries: [] }))).entries || []; + entries = root.filter((e) => e.type === "dir" && DOC_ROOTS.includes(e.name)); + } + } catch (e) { + if (e.code === 401) { main.innerHTML = errView(e); return; } + noMirror(); return; } - const rel = path.replace(/^feishu\/?/, ""); + if (!path && !entries.length) { noMirror(); return; } + const segs = path ? path.split("/") : []; + const join = (a, b) => (a ? a.replace(/\/$/, "") + "/" + b : b); // path 为空时不带前导斜杠 const dirs = entries.filter((e) => e.type === "dir"); const files = entries.filter((e) => e.type === "file" && e.name.endsWith(".md")); - const segCrumbs = rel ? rel.split("/").map((seg, i, arr) => { const p = "feishu/" + arr.slice(0, i + 1).join("/"); return i === arr.length - 1 ? `${esc(kbName(seg))}` : `${esc(kbName(seg))}`; }) : []; + const segCrumbs = segs.map((seg, i) => { const p = segs.slice(0, i + 1).join("/"); return i === segs.length - 1 ? `${esc(kbName(seg))}` : `${esc(kbName(seg))}`; }); main.innerHTML = `
${crumb(crumbHome(), `${esc(t("nav.docs"))}`, ...segCrumbs)} -

${rel ? esc(kbName(rel.split("/").pop())) : esc(t("docs.title"))}

- ${dirs.length ? `

${esc(t("docs.wikis"))}

${dirs.map((d) => `
📚 ${esc(kbName(d.name))}
${esc(t("docs.items", d.children ?? 0))}
`).join("")}
` : ""} - ${files.length ? `

${esc(t("docs.docs"))}

${files.map((f) => `
📄 ${esc(docName(f.name))}
`).join("")}
` : ""} +

${segs.length ? esc(kbName(segs[segs.length - 1])) : esc(t("docs.title"))}

+ ${dirs.length ? `

${esc(t("docs.wikis"))}

${dirs.map((d) => `
📚 ${esc(kbName(d.name))}
${esc(t("docs.items", d.children ?? 0))}
`).join("")}
` : ""} + ${files.length ? `

${esc(t("docs.docs"))}

${files.map((f) => `
📄 ${esc(docName(f.name))}
`).join("")}
` : ""} ${!dirs.length && !files.length ? emptyNote(t("docs.empty")) : ""}
`; highlightSidebar();