feat: configured knowledge sources + hybrid retrieval

src/cli.tsx

@@ -18,7 +18,7 @@ import { quickSetup, loadServiceConfig, ONBOARDING_PROMPTS } from './config/onbo
 import { SetupWizard } from './cli/setup-wizard';
 import { ChatInterface } from './cli/chat';
 import { MarkdownText } from './cli/components/markdown';
-import { createRetriever } from './knowledge/retriever';
+import { createConfiguredRetriever } from './knowledge/retriever';
 import type { AgentEvent } from './agent/types';
 import { skillRegistry } from './skills/registry';
 import { getRuntimeTools } from './cli/runtime-tools';
@@ -56,8 +56,8 @@ const VERSION = '0.1.0';
 /**
  * Knowledge retriever adapter for Agent runtime.
  */
-function createAgentKnowledgeRetriever() {
-  const retriever = createRetriever();
+async function createAgentKnowledgeRetriever(config: Awaited<ReturnType<typeof loadConfig>>) {
+  const retriever = await createConfiguredRetriever('.runbook', config);
 
   return {
     retrieve: async (context: {
@@ -99,7 +99,7 @@ async function createRuntimeAgent(config: Awaited<ReturnType<typeof loadConfig>>
     llm,
     tools: runtimeTools,
     skills: runtimeSkills,
-    knowledgeRetriever: createAgentKnowledgeRetriever(),
+    knowledgeRetriever: await createAgentKnowledgeRetriever(config),
     config: {
       maxIterations: config.agent.maxIterations,
       maxHypothesisDepth: config.agent.maxHypothesisDepth,
@@ -667,7 +667,7 @@ async function runStructuredInvestigation(
       availableTools: runtimeTools.map((tool) => tool.name),
       availableSkills: runtimeSkills,
       fetchRelevantRunbooks: async (context: RemediationContext) => {
-        const retriever = createRetriever();
+        const retriever = await createConfiguredRetriever('.runbook', config);
         try {
           const searchQuery = [context.rootCause, ...context.affectedServices].join(' ').trim();
           const results = await retriever.search(
@@ -972,7 +972,7 @@ async function runStructuredInvestigation(
       }
 
       if (applyRunbookUpdates && learning.appliedRunbookUpdates.length > 0) {
-        const retriever = createRetriever();
+        const retriever = await createConfiguredRetriever('.runbook', config);
         try {
           await retriever.sync();
           console.log(chalk.gray('Knowledge index refreshed after runbook updates.'));
@@ -1255,7 +1255,7 @@ knowledge
   .action(async () => {
     console.log(chalk.blue('Syncing knowledge from configured sources...'));
     try {
-      const retriever = createRetriever();
+      const retriever = await createConfiguredRetriever();
       const { added, updated } = await retriever.sync();
       console.log(chalk.green(`Sync complete: ${added} added, ${updated} updated`));
       console.log(chalk.green(`Total documents: ${retriever.getDocumentCount()}`));
@@ -1274,7 +1274,7 @@ knowledge
     const query = queryParts.join(' ');
     console.log(chalk.blue(`Searching for: "${query}"`));
     try {
-      const retriever = createRetriever();
+      const retriever = await createConfiguredRetriever();
       const results = await retriever.search(query, {
         limit: 10,
         typeFilter: options.type
@@ -1359,7 +1359,7 @@ knowledge
       await copyFile(filePath, destPath);
 
       // Sync to update the index
-      const retriever = createRetriever();
+      const retriever = await createConfiguredRetriever();
       await retriever.sync();
 
       console.log(chalk.green(`Added: ${title}`));
@@ -1387,7 +1387,7 @@ knowledge
     console.log(chalk.blue(`Checking for content older than ${staleDays} days...`));
 
     try {
-      const retriever = createRetriever();
+      const retriever = await createConfiguredRetriever();
       await retriever.sync();
 
       // Get all documents directly from the store for accurate counts.
@@ -1444,7 +1444,7 @@ knowledge
     console.log(chalk.blue('Knowledge Base Statistics:'));
 
     try {
-      const retriever = createRetriever();
+      const retriever = await createConfiguredRetriever();
       await retriever.sync();
 
       const counts = retriever.getDocumentCountsByType();

src/cli/chat.tsx

@@ -16,7 +16,7 @@ import type { AgentEvent } from '../agent/types';
 import { MarkdownText } from './components/markdown';
 import { skillRegistry } from '../skills/registry';
 import { getRuntimeTools } from './runtime-tools';
-import { createRetriever } from '../knowledge/retriever';
+import { createConfiguredRetriever } from '../knowledge/retriever';
 import { createMemory, type ConversationMemory } from '../agent/conversation-memory';
 
 const LOGO = `
@@ -91,7 +91,7 @@ export function ChatInterface() {
       await skillRegistry.loadUserSkills();
       const runtimeSkills = skillRegistry.getAll().map((skill) => skill.id);
       const runtimeTools = await getRuntimeTools(config, toolRegistry.getAll());
-      const retriever = createRetriever();
+      const retriever = await createConfiguredRetriever('.runbook', config);
 
       const newAgent = new Agent({
         llm,

src/eval/investigation-benchmark.ts

@@ -5,7 +5,7 @@ import { createLLMClient } from '../model/llm';
 import { toolRegistry } from '../tools/registry';
 import { skillRegistry } from '../skills/registry';
 import { getRuntimeTools } from '../cli/runtime-tools';
-import { createRetriever } from '../knowledge/retriever';
+import { createConfiguredRetriever } from '../knowledge/retriever';
 import {
   createOrchestrator,
   type InvestigationEvent,
@@ -351,7 +351,7 @@ async function main() {
           availableTools: runtimeTools.map((tool) => tool.name),
           availableSkills: runtimeSkills,
           fetchRelevantRunbooks: async (ctx: RemediationContext) => {
-            const retriever = createRetriever();
+            const retriever = await createConfiguredRetriever('.runbook', config);
             try {
               const searchQuery = [ctx.rootCause, ...ctx.affectedServices].join(' ').trim();
               const results = await retriever.search(searchQuery || 'incident remediation', {

src/integrations/hook-handlers.ts

@@ -8,7 +8,7 @@
 import { existsSync } from 'fs';
 import { readFile, writeFile, mkdir } from 'fs/promises';
 import { join } from 'path';
-import { createRetriever, KnowledgeRetriever } from '../knowledge/retriever/index';
+import { createConfiguredRetriever, KnowledgeRetriever } from '../knowledge/retriever/index';
 import type { RetrievedKnowledge, RetrievedChunk } from '../knowledge/types';
 
 /**
@@ -259,7 +259,7 @@ export async function handleSessionStart(
   let knowledgeStats = '';
 
   try {
-    retriever = createRetriever(config.baseDir);
+    retriever = await createConfiguredRetriever(config.baseDir);
     const counts = retriever.getDocumentCountsByType();
     const total = Object.values(counts).reduce((sum, c) => sum + c, 0);
 
@@ -326,7 +326,7 @@ export async function handleUserPromptSubmit(
   let retriever: KnowledgeRetriever | null = null;
 
   try {
-    retriever = createRetriever(config.baseDir);
+    retriever = await createConfiguredRetriever(config.baseDir);
     knowledge = await retriever.search(searchQuery, {
       serviceFilter: services.length > 0 ? services : undefined,
       limit: 10,

src/knowledge/retriever/__tests__/index.test.ts

@@ -0,0 +1,186 @@
+import { afterEach, describe, expect, it } from 'vitest';
+import { mkdtemp, mkdir, rm, writeFile } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { stringify as stringifyYaml } from 'yaml';
+import { createConfiguredRetriever } from '../index';
+
+async function createTempDir(): Promise<string> {
+  return mkdtemp(join(tmpdir(), 'runbook-knowledge-'));
+}
+
+async function writeMarkdownFile(path: string, content: string): Promise<void> {
+  await writeFile(path, content, 'utf-8');
+}
+
+describe('KnowledgeRetriever configuration + ingestion', () => {
+  const createdDirs: string[] = [];
+
+  afterEach(async () => {
+    await Promise.all(
+      createdDirs.splice(0).map(async (dir) => {
+        await rm(dir, { recursive: true, force: true });
+      })
+    );
+  });
+
+  it('loads configured filesystem sources from baseDir config', async () => {
+    const baseDir = await createTempDir();
+    createdDirs.push(baseDir);
+
+    const sourceDir = join(baseDir, 'external-knowledge');
+    await mkdir(sourceDir, { recursive: true });
+    await writeMarkdownFile(
+      join(sourceDir, 'checkout-timeout.md'),
+      `---
+type: runbook
+services: [checkout]
+---
+# Checkout Timeout Recovery
+
+Restart workers and verify queue depth.
+`
+    );
+
+    await writeFile(
+      join(baseDir, 'config.yaml'),
+      stringifyYaml({
+        knowledge: {
+          sources: [
+            {
+              type: 'filesystem',
+              path: sourceDir,
+            },
+          ],
+          store: {
+            path: join(baseDir, 'knowledge.db'),
+          },
+        },
+      }),
+      'utf-8'
+    );
+
+    const retriever = await createConfiguredRetriever(baseDir);
+    try {
+      const syncResult = await retriever.sync();
+      expect(syncResult.added).toBe(1);
+
+      const results = await retriever.search('checkout timeout');
+      expect(results.runbooks.length).toBeGreaterThan(0);
+      expect(results.runbooks[0].title).toContain('Checkout Timeout Recovery');
+    } finally {
+      retriever.close();
+    }
+  });
+
+  it('falls back to default runbooks path when configured sources are invalid', async () => {
+    const baseDir = await createTempDir();
+    createdDirs.push(baseDir);
+
+    const fallbackRunbooksDir = join(baseDir, 'runbooks');
+    await mkdir(fallbackRunbooksDir, { recursive: true });
+    await writeMarkdownFile(
+      join(fallbackRunbooksDir, 'latency-guide.md'),
+      `---
+type: runbook
+services: [api]
+---
+# API Latency Guide
+
+Check connection pools and recent deploys.
+`
+    );
+
+    await writeFile(
+      join(baseDir, 'config.yaml'),
+      stringifyYaml({
+        knowledge: {
+          sources: [
+            {
+              type: 'github',
+            },
+          ],
+          store: {
+            path: join(baseDir, 'knowledge.db'),
+          },
+        },
+      }),
+      'utf-8'
+    );
+
+    const retriever = await createConfiguredRetriever(baseDir);
+    try {
+      await retriever.sync();
+      const results = await retriever.search('latency');
+      expect(results.runbooks.length).toBeGreaterThan(0);
+      expect(results.runbooks[0].title).toContain('API Latency Guide');
+    } finally {
+      retriever.close();
+    }
+  });
+
+  it('uses configured retrieval.topK as the default search limit', async () => {
+    const baseDir = await createTempDir();
+    createdDirs.push(baseDir);
+
+    const sourceDir = join(baseDir, 'knowledge-source');
+    await mkdir(sourceDir, { recursive: true });
+    await writeMarkdownFile(
+      join(sourceDir, 'doc-a.md'),
+      `---
+type: runbook
+services: [payments]
+---
+# Payments Retry Runbook
+
+retry-strategy guidance
+`
+    );
+    await writeMarkdownFile(
+      join(sourceDir, 'doc-b.md'),
+      `---
+type: runbook
+services: [payments]
+---
+# Payments Queue Runbook
+
+retry-strategy fallback
+`
+    );
+
+    await writeFile(
+      join(baseDir, 'config.yaml'),
+      stringifyYaml({
+        knowledge: {
+          sources: [
+            {
+              type: 'filesystem',
+              path: sourceDir,
+            },
+          ],
+          store: {
+            path: join(baseDir, 'knowledge.db'),
+          },
+          retrieval: {
+            topK: 1,
+          },
+        },
+      }),
+      'utf-8'
+    );
+
+    const retriever = await createConfiguredRetriever(baseDir);
+    try {
+      await retriever.sync();
+      const results = await retriever.search('retry-strategy');
+      const total =
+        results.runbooks.length +
+        results.postmortems.length +
+        results.architecture.length +
+        results.knownIssues.length;
+      expect(total).toBe(1);
+    } finally {
+      retriever.close();
+    }
+  });
+});