diff --git a/src/knowledge/retriever/index.ts b/src/knowledge/retriever/index.ts index c6ce2a0..4e25dad 100644 --- a/src/knowledge/retriever/index.ts +++ b/src/knowledge/retriever/index.ts @@ -448,6 +448,34 @@ function readOptionalStringField(source: Record, key: string): return undefined; } +function readConfluenceAuth( + auth: Config['knowledge']['sources'][number]['auth'] +): { email: string; apiToken: string } | null { + if (!auth || typeof auth !== 'object') { + return null; + } + + const authRecord = auth as Record; + const email = readOptionalStringField(authRecord, 'email'); + const apiToken = readOptionalStringField(authRecord, 'apiToken'); + if (!email || !apiToken) { + return null; + } + + return { email, apiToken }; +} + +function readLegacyApiToken( + auth: Config['knowledge']['sources'][number]['auth'] +): string | undefined { + if (!auth || typeof auth !== 'object') { + return undefined; + } + + const authRecord = auth as Record; + return readOptionalStringField(authRecord, 'apiToken'); +} + function normalizeConfiguredSources( sources: Config['knowledge']['sources'], baseDir: string @@ -471,8 +499,8 @@ function normalizeConfiguredSources( } case 'confluence': { - const auth = source.auth; - if (!source.baseUrl || !source.spaceKey || !auth?.email || !auth?.apiToken) { + const auth = readConfluenceAuth(source.auth); + if (!source.baseUrl || !source.spaceKey || !auth) { console.warn('Skipping incomplete Confluence knowledge source.'); continue; } @@ -517,7 +545,7 @@ function normalizeConfiguredSources( const sourceRecord = source as Record; const notionApiKey = readOptionalStringField(sourceRecord, 'apiKey') || - source.auth?.apiToken || + readLegacyApiToken(source.auth) || process.env.RUNBOOK_NOTION_API_KEY || process.env.NOTION_API_KEY; @@ -547,7 +575,7 @@ function normalizeConfiguredSources( path: source.path || '', token: explicitToken || - source.auth?.apiToken || + readLegacyApiToken(source.auth) || process.env.RUNBOOK_GITHUB_TOKEN || process.env.GITHUB_TOKEN, }); diff --git a/src/knowledge/sources/__tests__/notion.test.ts b/src/knowledge/sources/__tests__/notion.test.ts new file mode 100644 index 0000000..eccf005 --- /dev/null +++ b/src/knowledge/sources/__tests__/notion.test.ts @@ -0,0 +1,177 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { loadFromNotion } from '../notion'; +import type { NotionSourceConfig } from '../../types'; + +function asFetchResponse(body: unknown, ok: boolean = true, status: number = 200): Response { + return { + ok, + status, + text: async () => (typeof body === 'string' ? body : JSON.stringify(body)), + json: async () => body, + } as Response; +} + +const notionConfig: NotionSourceConfig = { + type: 'notion', + databaseId: 'db_123', + apiKey: 'secret_notion_key', +}; + +describe('Notion knowledge source', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('loads database pages and converts blocks to markdown documents', async () => { + const fetchMock = vi.fn().mockImplementation(async (url: string) => { + if (url.includes('/databases/db_123/query')) { + return asFetchResponse({ + results: [ + { + id: 'page_1', + url: 'https://www.notion.so/page_1', + created_time: '2026-01-01T00:00:00.000Z', + last_edited_time: '2026-01-02T00:00:00.000Z', + properties: { + Name: { + type: 'title', + title: [{ plain_text: 'Checkout API Runbook' }], + }, + Type: { + type: 'select', + select: { name: 'runbook' }, + }, + Services: { + type: 'multi_select', + multi_select: [{ name: 'checkout' }, { name: 'payments' }], + }, + }, + }, + ], + has_more: false, + next_cursor: null, + }); + } + + if (url.includes('/blocks/page_1/children')) { + return asFetchResponse({ + results: [ + { + id: 'blk_1', + type: 'heading_1', + heading_1: { + rich_text: [{ plain_text: 'Mitigation Steps' }], + }, + has_children: false, + }, + { + id: 'blk_2', + type: 'paragraph', + paragraph: { + rich_text: [{ plain_text: 'Restart workers and flush stale connections.' }], + }, + has_children: false, + }, + ], + has_more: false, + next_cursor: null, + }); + } + + return asFetchResponse('not found', false, 404); + }); + + vi.stubGlobal('fetch', fetchMock); + + const documents = await loadFromNotion(notionConfig); + expect(documents.length).toBe(1); + expect(documents[0].title).toBe('Checkout API Runbook'); + expect(documents[0].services).toEqual(['checkout', 'payments']); + expect(documents[0].type).toBe('runbook'); + expect(documents[0].content).toContain('# Mitigation Steps'); + expect(documents[0].content).toContain('Restart workers'); + expect(documents[0].chunks.length).toBeGreaterThan(0); + }); + + it('respects the since option and skips older pages', async () => { + const fetchMock = vi.fn().mockImplementation(async (url: string) => { + if (url.includes('/databases/db_123/query')) { + return asFetchResponse({ + results: [ + { + id: 'old_page', + url: 'https://www.notion.so/old_page', + created_time: '2026-01-01T00:00:00.000Z', + last_edited_time: '2026-01-01T00:00:00.000Z', + properties: { + Name: { + type: 'title', + title: [{ plain_text: 'Old Runbook' }], + }, + }, + }, + ], + has_more: false, + next_cursor: null, + }); + } + return asFetchResponse({ + results: [], + has_more: false, + next_cursor: null, + }); + }); + + vi.stubGlobal('fetch', fetchMock); + + const documents = await loadFromNotion(notionConfig, { + since: '2026-01-02T00:00:00.000Z', + }); + expect(documents).toEqual([]); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('uses fallback title when title property is missing', async () => { + const fetchMock = vi.fn().mockImplementation(async (url: string) => { + if (url.includes('/databases/db_123/query')) { + return asFetchResponse({ + results: [ + { + id: 'page_no_title', + url: 'https://www.notion.so/page_no_title', + created_time: '2026-01-01T00:00:00.000Z', + last_edited_time: '2026-01-03T00:00:00.000Z', + properties: {}, + }, + ], + has_more: false, + next_cursor: null, + }); + } + if (url.includes('/blocks/page_no_title/children')) { + return asFetchResponse({ + results: [ + { + id: 'blk_x', + type: 'paragraph', + paragraph: { + rich_text: [{ plain_text: 'General recovery guidance.' }], + }, + has_children: false, + }, + ], + has_more: false, + next_cursor: null, + }); + } + return asFetchResponse('not found', false, 404); + }); + + vi.stubGlobal('fetch', fetchMock); + + const documents = await loadFromNotion(notionConfig); + expect(documents.length).toBe(1); + expect(documents[0].title).toBe('Untitled'); + expect(documents[0].type).toBe('runbook'); + }); +}); diff --git a/src/knowledge/sources/index.ts b/src/knowledge/sources/index.ts index 5a71f28..e0bd59d 100644 --- a/src/knowledge/sources/index.ts +++ b/src/knowledge/sources/index.ts @@ -7,6 +7,7 @@ import { loadFromFilesystem } from './filesystem'; import { loadFromConfluence } from './confluence'; import { loadFromGoogleDrive } from './google-drive'; +import { loadFromNotion } from './notion'; import { loadFromGitHub } from './github'; import type { KnowledgeDocument, KnowledgeSourceConfig } from '../types'; @@ -28,13 +29,15 @@ export async function loadFromSource( case 'confluence': return loadFromConfluence(config, options); + case 'notion': + return loadFromNotion(config, options); + case 'google_drive': return loadFromGoogleDrive(config, options); case 'github': return loadFromGitHub(config, options); - case 'notion': case 'api': console.warn(`Source type '${config.type}' is not yet implemented`); return []; @@ -48,4 +51,5 @@ export async function loadFromSource( export { loadFromFilesystem } from './filesystem'; export { loadFromConfluence } from './confluence'; export { loadFromGoogleDrive } from './google-drive'; +export { loadFromNotion } from './notion'; export { loadFromGitHub } from './github'; diff --git a/src/knowledge/sources/notion.ts b/src/knowledge/sources/notion.ts new file mode 100644 index 0000000..8fa044d --- /dev/null +++ b/src/knowledge/sources/notion.ts @@ -0,0 +1,511 @@ +/** + * Notion Knowledge Source + * + * Loads runbooks and knowledge docs from a Notion database. + * Fetches page properties and page block content, then normalizes to markdown. + */ + +import type { + KnowledgeChunk, + KnowledgeDocument, + KnowledgeType, + NotionSourceConfig, +} from '../types'; +import type { LoadOptions } from './index'; + +const NOTION_API_BASE = 'https://api.notion.com/v1'; +const NOTION_VERSION = '2022-06-28'; + +interface NotionQueryResponse { + results: NotionPage[]; + has_more: boolean; + next_cursor: string | null; +} + +interface NotionPage { + id: string; + url: string; + created_time: string; + last_edited_time: string; + properties: Record; +} + +type NotionProperty = + | { + type: 'title'; + title?: Array<{ plain_text?: string }>; + } + | { + type: 'rich_text'; + rich_text?: Array<{ plain_text?: string }>; + } + | { + type: 'select'; + select?: { name?: string }; + } + | { + type: 'multi_select'; + multi_select?: Array<{ name?: string }>; + } + | { + type: 'status'; + status?: { name?: string }; + } + | { + type: 'people'; + people?: Array<{ name?: string }>; + } + | { + type: 'relation'; + relation?: Array<{ id?: string }>; + } + | { + type: string; + [key: string]: unknown; + }; + +interface NotionBlocksResponse { + results: NotionBlock[]; + has_more: boolean; + next_cursor: string | null; +} + +interface NotionBlock { + id: string; + type: string; + has_children?: boolean; + [key: string]: unknown; +} + +interface NotionRichTextItem { + plain_text?: string; +} + +export async function loadFromNotion( + config: NotionSourceConfig, + options: LoadOptions = {} +): Promise { + const headers = buildHeaders(config.apiKey); + const pages = await queryDatabasePages(config, headers); + const documents: KnowledgeDocument[] = []; + const sinceDate = options.since ? new Date(options.since) : null; + + for (const page of pages) { + if (sinceDate) { + const edited = new Date(page.last_edited_time); + if (!Number.isNaN(edited.getTime()) && edited <= sinceDate) { + continue; + } + } + + const document = await processPage(page, config, headers); + if (document) { + documents.push(document); + } + } + + return documents; +} + +function buildHeaders(apiKey: string): Record { + return { + Authorization: `Bearer ${apiKey}`, + 'Notion-Version': NOTION_VERSION, + Accept: 'application/json', + 'Content-Type': 'application/json', + }; +} + +async function queryDatabasePages( + config: NotionSourceConfig, + headers: Record +): Promise { + const pages: NotionPage[] = []; + let cursor: string | null = null; + + do { + const body: Record = { + page_size: 100, + }; + if (config.filter) { + body.filter = config.filter; + } + if (cursor) { + body.start_cursor = cursor; + } + + const response = await fetch(`${NOTION_API_BASE}/databases/${config.databaseId}/query`, { + method: 'POST', + headers, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const text = await response.text(); + throw new Error(`Notion database query failed (${response.status}): ${text}`); + } + + const payload = (await response.json()) as NotionQueryResponse; + pages.push(...payload.results); + cursor = payload.has_more ? payload.next_cursor : null; + } while (cursor); + + return pages; +} + +async function processPage( + page: NotionPage, + config: NotionSourceConfig, + headers: Record +): Promise { + const title = extractTitle(page.properties) || 'Untitled'; + const services = extractNamedListProperty(page.properties, ['services', 'service']); + const tags = extractNamedListProperty(page.properties, ['tags', 'tag']); + const symptoms = extractNamedListProperty(page.properties, ['symptoms', 'symptom']); + const docType = extractKnowledgeType(page.properties) || inferTypeFromTitle(title); + const severity = extractSeverity(page.properties); + + const blocks = await fetchAllBlocks(page.id, headers); + const markdown = blocksToMarkdown(blocks).trim(); + if (!markdown) { + return null; + } + + const id = `notion_${page.id.replace(/-/g, '')}`; + return { + id, + source: { + type: 'notion', + name: 'notion', + config, + }, + type: docType, + title, + content: markdown, + chunks: chunkMarkdown(id, markdown), + services, + tags, + symptoms, + severityRelevance: severity ? [severity] : [], + createdAt: page.created_time, + updatedAt: page.last_edited_time, + sourceUrl: page.url, + }; +} + +async function fetchAllBlocks( + blockId: string, + headers: Record, + depth: number = 0 +): Promise { + const blocks: NotionBlock[] = []; + let cursor: string | null = null; + + do { + const url = new URL(`${NOTION_API_BASE}/blocks/${blockId}/children`); + url.searchParams.set('page_size', '100'); + if (cursor) { + url.searchParams.set('start_cursor', cursor); + } + + const response = await fetch(url.toString(), { headers }); + if (!response.ok) { + const text = await response.text(); + throw new Error(`Notion block retrieval failed (${response.status}): ${text}`); + } + + const payload = (await response.json()) as NotionBlocksResponse; + for (const block of payload.results) { + blocks.push(block); + + if (block.has_children && depth < 2) { + const childBlocks = await fetchAllBlocks(block.id, headers, depth + 1); + blocks.push(...childBlocks); + } + } + + cursor = payload.has_more ? payload.next_cursor : null; + } while (cursor); + + return blocks; +} + +function blocksToMarkdown(blocks: NotionBlock[]): string { + const lines: string[] = []; + + for (const block of blocks) { + const type = block.type; + const payload = block[type] as Record | undefined; + const richText = (payload?.rich_text as NotionRichTextItem[] | undefined) || []; + const text = richText + .map((item) => item.plain_text || '') + .join('') + .trim(); + + if (!text && type !== 'divider') { + continue; + } + + switch (type) { + case 'heading_1': + lines.push(`# ${text}`); + break; + case 'heading_2': + lines.push(`## ${text}`); + break; + case 'heading_3': + lines.push(`### ${text}`); + break; + case 'bulleted_list_item': + lines.push(`- ${text}`); + break; + case 'numbered_list_item': + lines.push(`1. ${text}`); + break; + case 'to_do': { + const checked = Boolean(payload?.checked); + lines.push(`- [${checked ? 'x' : ' '}] ${text}`); + break; + } + case 'quote': + lines.push(`> ${text}`); + break; + case 'code': { + const language = String(payload?.language || '').trim(); + lines.push(`\`\`\`${language}`); + lines.push(text); + lines.push('```'); + break; + } + case 'divider': + lines.push('---'); + break; + default: + lines.push(text); + break; + } + + lines.push(''); + } + + return lines.join('\n').trim(); +} + +function extractTitle(properties: Record): string | null { + for (const property of Object.values(properties)) { + if (property.type !== 'title') { + continue; + } + + const titleItems = + ( + property as { + title?: Array<{ plain_text?: string }>; + } + ).title || []; + const text = titleItems + .map((item) => item.plain_text || '') + .join('') + .trim(); + if (text) { + return text; + } + } + + return null; +} + +function extractNamedListProperty( + properties: Record, + candidateNames: string[] +): string[] { + for (const [name, property] of Object.entries(properties)) { + if (!candidateNames.includes(name.toLowerCase())) { + continue; + } + + if (property.type === 'multi_select') { + const values = + ( + property as { + multi_select?: Array<{ name?: string }>; + } + ).multi_select || []; + return values + .map((entry) => entry.name || '') + .map((entry) => entry.trim()) + .filter(Boolean); + } + + if (property.type === 'select') { + const value = (property as { select?: { name?: string } }).select?.name?.trim(); + return value ? [value] : []; + } + + if (property.type === 'rich_text') { + const values = + ( + property as { + rich_text?: Array<{ plain_text?: string }>; + } + ).rich_text || []; + const value = values + .map((entry) => entry.plain_text || '') + .join(',') + .split(',') + .map((part) => part.trim()) + .filter(Boolean); + return value; + } + } + + return []; +} + +function extractKnowledgeType(properties: Record): KnowledgeType | null { + for (const [name, property] of Object.entries(properties)) { + if (name.toLowerCase() !== 'type') { + continue; + } + + let rawValue = ''; + if (property.type === 'select') { + rawValue = (property as { select?: { name?: string } }).select?.name || ''; + } else if (property.type === 'status') { + rawValue = (property as { status?: { name?: string } }).status?.name || ''; + } else if (property.type === 'rich_text') { + const values = + ( + property as { + rich_text?: Array<{ plain_text?: string }>; + } + ).rich_text || []; + rawValue = values.map((entry) => entry.plain_text || '').join(''); + } + + const normalized = rawValue.trim().toLowerCase(); + if ( + normalized === 'runbook' || + normalized === 'postmortem' || + normalized === 'architecture' || + normalized === 'known_issue' || + normalized === 'ownership' || + normalized === 'environment' || + normalized === 'playbook' || + normalized === 'faq' + ) { + return normalized; + } + } + + return null; +} + +function inferTypeFromTitle(title: string): KnowledgeType { + const lower = title.toLowerCase(); + if (lower.includes('postmortem') || lower.includes('post-mortem')) { + return 'postmortem'; + } + if (lower.includes('architecture') || lower.includes('design')) { + return 'architecture'; + } + if (lower.includes('known issue') || lower.includes('workaround')) { + return 'known_issue'; + } + return 'runbook'; +} + +function extractSeverity( + properties: Record +): 'sev1' | 'sev2' | 'sev3' | null { + for (const [name, property] of Object.entries(properties)) { + if (!['severity', 'priority'].includes(name.toLowerCase())) { + continue; + } + + let raw = ''; + if (property.type === 'select') { + raw = (property as { select?: { name?: string } }).select?.name || ''; + } else if (property.type === 'status') { + raw = (property as { status?: { name?: string } }).status?.name || ''; + } else if (property.type === 'rich_text') { + const values = + ( + property as { + rich_text?: Array<{ plain_text?: string }>; + } + ).rich_text || []; + raw = values.map((entry) => entry.plain_text || '').join(''); + } + + const normalized = raw.trim().toLowerCase(); + if (normalized === 'sev1' || normalized === 'p1' || normalized === 'critical') { + return 'sev1'; + } + if (normalized === 'sev2' || normalized === 'p2' || normalized === 'high') { + return 'sev2'; + } + if (normalized === 'sev3' || normalized === 'p3' || normalized === 'medium') { + return 'sev3'; + } + } + + return null; +} + +function chunkMarkdown(documentId: string, content: string): KnowledgeChunk[] { + const chunks: KnowledgeChunk[] = []; + const lines = content.split('\n'); + + let currentChunk: string[] = []; + let currentTitle: string | undefined; + let chunkIndex = 0; + let lineStart = 0; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (line.match(/^#{1,3}\s+/)) { + if (currentChunk.length > 0) { + chunks.push({ + id: `${documentId}_${chunkIndex++}`, + documentId, + content: currentChunk.join('\n').trim(), + sectionTitle: currentTitle, + chunkType: inferChunkType(currentChunk.join('\n')), + lineStart, + lineEnd: i - 1, + }); + } + currentTitle = line.replace(/^#+\s+/, '').trim(); + currentChunk = [line]; + lineStart = i; + } else { + currentChunk.push(line); + } + } + + if (currentChunk.length > 0) { + chunks.push({ + id: `${documentId}_${chunkIndex}`, + documentId, + content: currentChunk.join('\n').trim(), + sectionTitle: currentTitle, + chunkType: inferChunkType(currentChunk.join('\n')), + lineStart, + lineEnd: lines.length - 1, + }); + } + + return chunks; +} + +function inferChunkType(content: string): KnowledgeChunk['chunkType'] { + const lower = content.toLowerCase(); + if (content.includes('```')) return 'command'; + if (lower.includes('step') || lower.includes('[ ]') || lower.includes('[x]')) return 'procedure'; + if (lower.includes('if ') || lower.includes('when ') || lower.includes('decision')) + return 'decision'; + if (lower.includes('symptom') || lower.includes('overview') || lower.includes('background')) + return 'context'; + return 'reference'; +}