feat(@helm/api): helm sync command for manual item hydration from GitHub Projects

apps/api/package.json

@@ -6,7 +6,8 @@
     "dev": "bun run --watch src/index.ts",
     "build": "tsc --noEmit && bun build src/index.ts --outdir dist --target bun",
     "test": "vitest run",
-    "lint": "eslint src/"
+    "lint": "eslint src/",
+    "sync": "bun run src/scripts/sync.ts"
   },
   "dependencies": {
     "@helm/adapters": "workspace:*",

apps/api/src/scripts/sync.ts

@@ -0,0 +1,92 @@
+/**
+ * helm sync — manual item hydration from GitHub Projects v2.
+ *
+ * Usage:
+ *   pnpm --filter @helm/api sync <productSlug>
+ *
+ * Required env vars:
+ *   GITHUB_TOKEN                — GitHub PAT with project read access
+ *   HELM_KNOWLEDGE_REPO_PATH    — path to the primary knowledge repo
+ *
+ * Optional env vars:
+ *   HELM_DATA_DIR               — defaults to ./data relative to CWD
+ */
+import { join } from 'node:path';
+import { getProductRegistry } from '../services/index.js';
+import { syncProductItems } from '../services/sync.js';
+
+async function main(): Promise<void> {
+  const slug = process.argv[2]?.trim();
+  if (!slug) {
+    console.error('Usage: pnpm --filter @helm/api sync <productSlug>');
+    process.exit(1);
+  }
+
+  const token = process.env.GITHUB_TOKEN?.trim();
+  if (!token) {
+    console.error('Error: GITHUB_TOKEN environment variable is not set or blank');
+    process.exit(1);
+  }
+
+  const knowledgeRepoPath = process.env.HELM_KNOWLEDGE_REPO_PATH?.trim();
+  if (!knowledgeRepoPath) {
+    console.error('Error: HELM_KNOWLEDGE_REPO_PATH environment variable is not set or blank');
+    process.exit(1);
+  }
+
+  const SAFE_FS_PATH_REGEX = /^[A-Za-z0-9._/\-]+$/;
+  if (!SAFE_FS_PATH_REGEX.test(knowledgeRepoPath)) {
+    console.error('Error: HELM_KNOWLEDGE_REPO_PATH contains invalid characters');
+    process.exit(1);
+  }
+
+  const envDataDir = process.env.HELM_DATA_DIR?.trim();
+  if (envDataDir && !SAFE_FS_PATH_REGEX.test(envDataDir)) {
+    console.error('Error: HELM_DATA_DIR contains invalid characters');
+    process.exit(1);
+  }
+  const dataRoot = envDataDir || join(process.cwd(), 'data');
+
+  let products;
+  try {
+    products = await getProductRegistry();
+  } catch (err) {
+    console.error(
+      `[sync] Failed to load product registry: ${err instanceof Error ? err.message : err}`,
+    );
+    process.exit(1);
+  }
+
+  const product = products.find((p) => p.product.slug === slug);
+  if (!product) {
+    const known = products.map((p) => p.product.slug).join(', ');
+    console.error(`[sync] Product not found: "${slug}". Known products: ${known}`);
+    process.exit(1);
+  }
+
+  if (product.issue_tracker.provider !== 'github_projects') {
+    console.log(
+      `[sync] Product "${slug}" uses provider "${product.issue_tracker.provider}" — sync only supports github_projects. Nothing to do.`,
+    );
+    process.exit(0);
+  }
+
+  console.log(`[sync] Starting sync for product "${slug}"…`);
+
+  try {
+    const result = await syncProductItems(product, token, dataRoot);
+    const noun = result.synced === 1 ? 'item' : 'items';
+    console.log(`Synced ${result.synced} ${noun} in ${result.durationMs}ms`);
+    if (result.skipped > 0) {
+      console.warn(`[sync] Skipped ${result.skipped} item(s) with invalid externalId`);
+    }
+  } catch (err) {
+    console.error(`[sync] Error: ${err instanceof Error ? err.message : err}`);
+    process.exit(1);
+  }
+}
+
+main().catch((err) => {
+  console.error('[sync] Unexpected error:', err);
+  process.exit(1);
+});

apps/api/src/services/sync.test.ts

@@ -0,0 +1,268 @@
+import { tmpdir } from 'node:os';
+import { basename, join } from 'node:path';
+import { mkdir, rm } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import type { ItemState } from './types.js';
+import { syncProductItems } from './sync.js';
+import type { SyncOptions } from './sync.js';
+import type { NormalizedItem } from '@helm/adapters';
+import type { Product } from '@helm/shared';
+import { INITIAL_STAGE } from '@helm/workflow';
+
+// ── Fixtures ──────────────────────────────────────────────────────────────────
+
+const BASE_PRODUCT: Product = {
+  helm_version: '0',
+  product: { slug: 'helm-playground', name: 'Helm Playground' },
+  issue_tracker: {
+    provider: 'github_projects',
+    org: 'lhpaul',
+    project_number: 5,
+    custom_field_name: 'Helm Stage',
+  },
+  code_repos: [
+    { url: 'https://github.com/lhpaul/helm-playground', default_branch: 'main', role: 'app' },
+  ],
+  knowledge_repo: {
+    url: 'https://github.com/lhpaul/helm-playground-knowledge',
+    default_branch: 'main',
+  },
+  workflow: {
+    stages_enabled: ['discovery', 'spec-ready', 'released'],
+    designer_gate: 'skip',
+    qa_gate: 'skip',
+  },
+  specialists: {
+    spec_writer: { runtime: 'claude_code', model: 'claude-sonnet-4-6' },
+    plan_writer: { runtime: 'claude_code', model: 'claude-sonnet-4-6' },
+    implementer: { runtime: 'claude_code', model: 'claude-opus-4-7' },
+    code_reviewer: { runtime: 'claude_code', model: 'claude-sonnet-4-6' },
+    security_reviewer: { runtime: 'claude_code', model: 'claude-sonnet-4-6' },
+    test_reviewer: { runtime: 'claude_code', model: 'claude-sonnet-4-6' },
+    remediation: { runtime: 'claude_code', model: 'claude-sonnet-4-6' },
+  },
+};
+
+function makeItem(overrides: Partial<NormalizedItem> = {}): NormalizedItem {
+  return {
+    externalId: 'issue_1',
+    title: 'Hello world endpoint',
+    subStage: 'discovery',
+    status: 'open',
+    url: 'https://github.com/lhpaul/helm-playground/issues/1',
+    ...overrides,
+  };
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function makeOptions(
+  items: NormalizedItem[],
+  capturedWrites: Array<{ path: string; data: unknown }> = [],
+  existingState: Record<string, ItemState> = {},
+): SyncOptions {
+  return {
+    _listItems: () => Promise.resolve(items),
+    _writeJson: async (filePath, data) => {
+      capturedWrites.push({ path: filePath, data });
+    },
+    _readJson: async <T>(filePath: string): Promise<T | null> => {
+      const key = basename(filePath).replace('.json', '');
+      return (existingState[key] ?? null) as T | null;
+    },
+  };
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+describe('syncProductItems', () => {
+  let tmpDir: string;
+
+  beforeEach(async () => {
+    tmpDir = join(tmpdir(), `helm-sync-test-${randomUUID()}`);
+    await mkdir(join(tmpDir, 'items'), { recursive: true });
+  });
+
+  afterEach(async () => {
+    vi.restoreAllMocks();
+    await rm(tmpDir, { recursive: true, force: true });
+  });
+
+  describe('happy path', () => {
+    it('returns synced count and writes one ItemState file', async () => {
+      const writes: Array<{ path: string; data: unknown }> = [];
+      const result = await syncProductItems(
+        BASE_PRODUCT,
+        'token',
+        tmpDir,
+        makeOptions([makeItem()], writes),
+      );
+
+      expect(result.synced).toBe(1);
+      expect(result.skipped).toBe(0);
+      expect(result.durationMs).toBeGreaterThanOrEqual(0);
+      expect(writes).toHaveLength(1);
+    });
+
+    it('writes the correct ItemState shape', async () => {
+      const writes: Array<{ path: string; data: unknown }> = [];
+      await syncProductItems(
+        BASE_PRODUCT,
+        'token',
+        tmpDir,
+        makeOptions([makeItem({ externalId: 'issue_42', subStage: 'spec-ready' })], writes),
+      );
+
+      const state = writes[0]!.data as Record<string, unknown>;
+      expect(state.externalId).toBe('issue_42');
+      expect(state.productSlug).toBe('helm-playground');
+      expect(state.currentStage).toBe('spec-ready');
+      expect(state.history).toHaveLength(1);
+      expect((state.history as Array<Record<string, unknown>>)[0]!.triggeredBy).toBe(
+        'sync:github-projects',
+      );
+      expect((state.history as Array<Record<string, unknown>>)[0]!.fromStage).toBeNull();
+    });
+
+    it('writes the file path under items/ dir', async () => {
+      const writes: Array<{ path: string; data: unknown }> = [];
+      await syncProductItems(BASE_PRODUCT, 'token', tmpDir, makeOptions([makeItem()], writes));
+
+      expect(writes[0]!.path).toContain('items');
+      expect(writes[0]!.path).toContain('issue_1.json');
+    });
+
+    it('syncs multiple items', async () => {
+      const items = [
+        makeItem({ externalId: 'issue_1' }),
+        makeItem({ externalId: 'issue_2', subStage: 'plan-ready' }),
+        makeItem({ externalId: 'issue_3', subStage: null }),
+      ];
+      const writes: Array<{ path: string; data: unknown }> = [];
+      const result = await syncProductItems(
+        BASE_PRODUCT,
+        'token',
+        tmpDir,
+        makeOptions(items, writes),
+      );
+
+      expect(result.synced).toBe(3);
+      expect(writes).toHaveLength(3);
+    });
+  });
+
+  describe('stage mapping', () => {
+    it('uses item.subStage when set', async () => {
+      const writes: Array<{ path: string; data: unknown }> = [];
+      await syncProductItems(
+        BASE_PRODUCT,
+        'token',
+        tmpDir,
+        makeOptions([makeItem({ subStage: 'code-review' })], writes),
+      );
+
+      expect((writes[0]!.data as Record<string, unknown>).currentStage).toBe('code-review');
+    });
+
+    it('falls back to INITIAL_STAGE when subStage is null', async () => {
+      const writes: Array<{ path: string; data: unknown }> = [];
+      await syncProductItems(
+        BASE_PRODUCT,
+        'token',
+        tmpDir,
+        makeOptions([makeItem({ subStage: null })], writes),
+      );
+
+      expect((writes[0]!.data as Record<string, unknown>).currentStage).toBe(INITIAL_STAGE);
+    });
+  });
+
+  describe('idempotency', () => {
+    it('skips the write on second run when stage is unchanged (no file drift)', async () => {
+      const item = makeItem({ externalId: 'issue_7', subStage: 'discovery' });
+      const writes1: Array<{ path: string; data: unknown }> = [];
+
+      // First run: no existing state → creates fresh
+      await syncProductItems(BASE_PRODUCT, 'token', tmpDir, makeOptions([item], writes1));
+      expect(writes1).toHaveLength(1);
+      const firstState = writes1[0]!.data as ItemState;
+
+      // Second run: existing state has same stage → skip write (continue)
+      const writes2: Array<{ path: string; data: unknown }> = [];
+      const existing: Record<string, ItemState> = { issue_7: firstState };
+      await syncProductItems(BASE_PRODUCT, 'token', tmpDir, makeOptions([item], writes2, existing));
+      expect(writes2).toHaveLength(0); // no write — content unchanged
+    });
+
+    it('preserves createdAt and appends to history when stage changes on re-sync', async () => {
+      const item = makeItem({ externalId: 'issue_8', subStage: 'discovery' });
+      const writes1: Array<{ path: string; data: unknown }> = [];
+      await syncProductItems(BASE_PRODUCT, 'token', tmpDir, makeOptions([item], writes1));
+      const firstState = writes1[0]!.data as ItemState;
+
+      // Re-sync with a different stage
+      const updatedItem = makeItem({ externalId: 'issue_8', subStage: 'spec-ready' });
+      const writes2: Array<{ path: string; data: unknown }> = [];
+      const existing: Record<string, ItemState> = { issue_8: firstState };
+      await syncProductItems(
+        BASE_PRODUCT,
+        'token',
+        tmpDir,
+        makeOptions([updatedItem], writes2, existing),
+      );
+
+      expect(writes2).toHaveLength(1);
+      const secondState = writes2[0]!.data as ItemState;
+      expect(secondState.createdAt).toBe(firstState.createdAt); // preserved
+      expect(secondState.currentStage).toBe('spec-ready');
+      expect(secondState.history).toHaveLength(2); // creation + transition
+    });
+  });
+
+  describe('error cases', () => {
+    it('throws when provider is not github_projects', async () => {
+      const linearProduct: Product = {
+        ...BASE_PRODUCT,
+        issue_tracker: {
+          provider: 'linear',
+          workspace: 'helm-dev',
+          team_key: 'HLM',
+          label_prefix: 'helm:',
+        },
+      };
+
+      await expect(
+        syncProductItems(linearProduct, 'token', tmpDir, makeOptions([])),
+      ).rejects.toThrow("sync only supports provider 'github_projects'");
+    });
+
+    it('skips items with invalid externalIds and increments skipped count', async () => {
+      vi.spyOn(console, 'warn').mockImplementation(() => {});
+      const writes: Array<{ path: string; data: unknown }> = [];
+      const items = [
+        makeItem({ externalId: 'issue_1' }), // valid
+        makeItem({ externalId: '../traversal' }), // invalid
+        makeItem({ externalId: '.hidden' }), // invalid (leading dot)
+        makeItem({ externalId: 'issue_2' }), // valid
+      ];
+
+      const result = await syncProductItems(
+        BASE_PRODUCT,
+        'token',
+        tmpDir,
+        makeOptions(items, writes),
+      );
+
+      expect(result.synced).toBe(2);
+      expect(result.skipped).toBe(2);
+      expect(writes).toHaveLength(2);
+    });
+
+    it('returns synced=0 when adapter returns empty list', async () => {
+      const result = await syncProductItems(BASE_PRODUCT, 'token', tmpDir, makeOptions([]));
+      expect(result.synced).toBe(0);
+      expect(result.skipped).toBe(0);
+    });
+  });
+});