diff --git a/.github/workflows/ci.yml b/.github/workflows/go-ci.yml similarity index 55% rename from .github/workflows/ci.yml rename to .github/workflows/go-ci.yml index 6b92fd3..84ab1d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/go-ci.yml @@ -1,4 +1,4 @@ -name: CI +name: Go CI on: pull_request: @@ -6,24 +6,10 @@ on: branches: [main] jobs: - node: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - # Version comes from package.json's "packageManager" field (single source of truth). - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: pnpm - - run: pnpm install --frozen-lockfile=false - - run: pnpm -r lint - - run: pnpm -r typecheck - - run: pnpm -r build - - run: pnpm -r test - go: runs-on: ubuntu-latest + permissions: + contents: read env: DATABASE_URL: postgres://funcatlas:funcatlas@localhost:5432/funcatlas?sslmode=disable services: @@ -42,9 +28,11 @@ jobs: --health-retries 5 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-go@v5 with: - go-version: "1.25" + go-version: "1.24" cache-dependency-path: services/parser/go.sum - name: Install CGO toolchain run: sudo apt-get update && sudo apt-get install -y gcc @@ -53,6 +41,21 @@ jobs: with: version: v2.12.2 working-directory: services/parser + - name: go mod tidy check + working-directory: services/parser + run: go mod tidy && git diff --exit-code go.mod go.sum + - name: go vet + working-directory: services/parser + run: go vet ./... - name: go test working-directory: services/parser - run: go test ./... + run: go test -race ./... + - name: go build + working-directory: services/parser + run: go build -o parser ./cmd/parser + - name: parser sample check + working-directory: services/parser + run: ./parser --repo ./testdata/sample --format summary + - name: migration check + run: | + docker run --rm --network host -v $(pwd)/services/parser/migrations:/migrations migrate/migrate -path /migrations -database "${DATABASE_URL}" up diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml new file mode 100644 index 0000000..8b7b56b --- /dev/null +++ b/.github/workflows/node-ci.yml @@ -0,0 +1,27 @@ +name: Node CI + +on: + pull_request: + push: + branches: [main] + +jobs: + node: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + # Version comes from package.json's "packageManager" field (single source of truth). + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm -r lint + - run: pnpm -r typecheck + - run: pnpm -r build + - run: pnpm -r test diff --git a/.gitignore b/.gitignore index 8f4e286..d60b472 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,6 @@ Thumbs.db .turbo/ # --- Vercel (if used later) --- -.vercel/ \ No newline at end of file +.vercel/ +out.json +extract_actual.json diff --git a/CLAUDE.md b/CLAUDE.md index cda68cb..29bf12b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,8 +8,11 @@ resolve calls → store in Postgres → explore via React Flow canvas (file → code block). First language: **TypeScript**. ## Status -Planning only — no code yet. Phases: 0 bootstrap → 1 parser+isolation → 2 storage+resolution → -3 API+auth+canvas+search → 4 webhooks+queue+hardening. +- [x] Phase 0: Bootstrap +- [x] Phase 1: Parser & Isolation +- [ ] Phase 2: Storage & Resolution +- [ ] Phase 3: API, Auth, Canvas & Search +- [ ] Phase 4: Webhooks, Queue & Hardening ## Locked stack (do NOT re-decide without explicit reason) - Monorepo: **pnpm + Turborepo**; shared types in **`packages/shared`** (Drizzle + Zod). diff --git a/Makefile b/Makefile index 270f6af..13c4c80 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # Run `make `. Most targets shell out to pnpm/turbo or docker. .PHONY: install dev build lint typecheck test migrate up down health \ - go-build go-test go-vet go-run go-tidy clean + go-build go-test go-vet go-run go-tidy go-lint clean install: ## Install all workspace dependencies pnpm install @@ -48,6 +48,9 @@ go-vet: ## Vet the Go parser go-tidy: ## Tidy Go modules cd services/parser && go mod tidy +go-lint: ## Lint the Go parser + cd services/parser && golangci-lint run + go-run: ## Run the parser against a local repo (usage: make go-run REPO=./path) cd services/parser && go run ./cmd/parser --repo "$(REPO)" diff --git a/PRD.md b/PRD.md new file mode 100644 index 0000000..c4a5294 --- /dev/null +++ b/PRD.md @@ -0,0 +1,200 @@ +# PRD — funcatlas / CodeCanvas + +> Product Requirements Document. Single source of truth for *what* we're building and *why*. +> Architecture, stack, and risks live in `docs/` (`ARCHITECTURE.md`, `TECH_STACK.md`, +> `DATA_MODEL.md`, `RISKS.md`, `SECURITY.md`, `UI_GUIDE.md`); this document is the **product +> contract**. Cross-reference `PLAN.md` for the phased execution plan and `TASKLIST.md` for the +> current chunk-by-chunk build. + +--- + +## 1. Vision + +An interactive visual map of any codebase: paste a repo URL → we clone it, parse with tree-sitter, +resolve call relationships, store the graph in Postgres, and let you explore it on a React Flow +canvas (file → card → function mind-map → code block). The goal is to make *the shape of a +codebase* legible for repos that no longer fit in one human head — without ever pretending to know +what a call points to when it doesn't. + +**One-liner:** *"See the shape of any codebase — functions, calls, and confidence, visually."* + +--- + +## 2. Why now / problem + +- Codebases outgrow a single engineer's working memory; onboarding to a large repo is slow. +- Existing tools (IDE go-to-definition) are **local and per-file**; none give a *repo-level* map. +- Static-analysis dashboards are tables/numbers, not **explorable** graphs. +- A trustworthy, repo-wide visual call graph — with explicit uncertainty baked in — closes the gap. + +--- + +## 3. Target users & personas + +- **P1 — Mid/senior engineer onboarding to an unfamiliar repo.** *"Where do I even start?"* Wants + the high-level shape: entry points, hot functions, who calls what. +- **P2 — Tech lead / maintainer of a large TS project.** *"What's the blast radius if I change + this function?"* Wants N-hop traversal and caller sets. +- **P3 — OSS contributor evaluating a project.** Quick orientation before a first PR. + +**Non-goal for MVP:** multi-tenant teams, RBAC, sharing canvases, real-time collaboration. + +--- + +## 4. Goals & non-goals (MVP) + +### Goals +- Parse a public TS repo end-to-end → trustworthy, confidence-tagged call graph persisted in Postgres. +- Explore it on a premium dark-mode React Flow canvas: sidebar → card → mind-map → code. +- GitHub OAuth login; register a repo by URL; automatic incremental refresh via webhook + queue. +- Function-name search across the repo (⌘K palette + search box). +- Production-grade isolation: parser runs non-root, read-only, no network egress, no symlink + follow, file/size/binary caps. + +### Non-goals (post-MVP) +- LSP-based resolution (ships later as a v2 upgrade of `name_match`/`unresolved` → `exact`). +- Excalidraw freehand annotation layer. +- Multi-language support (TypeScript first; go deep before breadth). +- Neo4j / dedicated graph database. +- Real-time multi-user editing, saved canvas layouts. + +--- + +## 5. User stories + +- As a visitor, I land on `/` and see an animated live-graph hero so I understand the product in + 5 seconds, then "Try a repo" → GitHub OAuth. +- As a logged-in user, I paste a repo URL → the repo is cloned/parsed → I see its file tree. +- As a user, I click a file → a card appears → click the card → a mind-map of that file's functions + branches out → click a function → Shiki-highlighted code block, with cross-file call links. +- As a user, edges show confidence: **solid** (`exact`) / **dashed** (`name_match`) / **dotted** + (`unresolved`) — a guess is never drawn as a fact. +- As a user, I ⌘K / search a function name → jump to it anywhere in the repo. +- As a user, when the repo's maintainer pushes, the graph updates automatically (webhook → queue). +- As a maintainer, I see the blast radius of changing a function (N-hop traversal). + +--- + +## 6. Functional requirements + +- **FR-1 Ingestion** — accept a public GitHub repo URL (post-OAuth, repo-read scope) **or** a local + path; clone/read into an isolated environment; never execute the repo's build/install scripts. +- **FR-2 Parsing** — tree-sitter extracts function/method definitions (name, qualified name, line + range, source), call expressions (callee name, enclosing caller, line), and import statements. +- **FR-3 Input hardening** — reject symlink/path-traversal escapes; skip files >1MB, binary files, + and dirs under `node_modules`/`.git`/`dist`/`build`; cap total file count and tree depth. +- **FR-4 Resolution** — link each call to a definition via same-file → imported symbol → package + fallback → unresolved; tag every edge `exact`/`name_match`/`unresolved`. +- **FR-5 Storage** — persist repos, files, functions, edges to Postgres (schema in `DATA_MODEL.md`), + with overload-safe uniqueness, `ON DELETE CASCADE`, and `parsed_commit`/`updated_at` for incremental diff. +- **FR-6 Graph API** (session-gated) — list repos; file tree; functions for file; edges for function; + raw source; search functions by name across a repo. +- **FR-7 Canvas** — sidebar file tree; card → mind-map → code; confidence-styled edges; minimap; + multi-open; ⌘K palette + name search; ≤2000 visible-node target with expand-on-click. +- **FR-8 Auth** — GitHub OAuth from day 1 (repo-read scope, refresh); Redis sessions; rate-limit. +- **FR-9 Incremental refresh** — GitHub webhook (HMAC-verified, replay-protected, per-repo + throttled) → BullMQ → diff changed files → re-parse only those → re-link only affected edges + (renames/deletes update every edge pointing at the old function — no orphans). +- **FR-10 Isolation runtime** — parser container runs non-root, read-only rootfs, no caps, and + **network NONE** during parse (clone happens in a separate network-enabled sidecar sharing tmpfs). + +--- + +## 7. Non-functional requirements + +- **NFR-1 Performance** — parse a 300-file TS repo in **< 90s**; `GET functions for file` p95 + **< 150ms**; N-hop (depth 5, 10k edges) **< 500ms**; canvas 60fps at ≤2000 visible nodes. +- **NFR-2 Security** — no network egress during parse; untrusted repo can't read host files + (symlink/path containment); webhook replay/flood safe; session-gated API; secrets via env. +- **NFR-3 Correctness** — re-parse of a renamed/deleted function leaves **no orphan edges**; + resolution never silently claims certainty it lacks (confidence is first-class). +- **NFR-4 Operability** — single `docker compose up` brings up postgres, redis, api, web, parser; + `/healthz` endpoints; logs via `zap` (parser) + `pino` (api). +- **NFR-5 Maintainability** — shared TS types in `packages/shared`; single SQL migration source + (`services/parser/migrations/`); no full ORM on the Go side (`sqlx` explicit SQL); Drizzle on TS API side. +- **NFR-6 UX** — premium dark-mode-first; accent tokens; purposeful Framer Motion; skeleton/shimmer + loading; actionable errors; `prefers-reduced-motion` respected. + +--- + +## 8. Success metrics (MVP) + +- A real 300-file public TS repo parses, resolves, and renders end-to-end without OOM or error. +- N-hop traversal returns in <500ms at 10k edges; p95 `functions-for-file` <150ms. +- **0** successful symlink-escape / oversized-file reads (negative tests green). +- Pushing a commit to a registered repo updates the graph automatically; replayed/out-of-window + webhook rejected; webhook flood throttled. +- A pilot user can navigate an unfamiliar repo to *find the entry point* in <5 min. + +--- + +## 9. Scope boundaries + +**In:** TypeScript only; public repos via GitHub OAuth; single-user (per-repo isolation, not +multi-user); name/scope resolution; webhook incremental updates; function-name search. + +**Out:** LSP resolution; Excalidraw; multi-language; Neo4j; layout persistence; multi-tenant. + +--- + +## 10. Release plan (4 phases) + +- **Phase 1 — Parser core + isolation** *(completed; branch `phase-1/parser-core-and-isolation`, + PR #21)*. Given a local repo path → correct IR JSON for TypeScript, with hardening and an + isolated Docker image. No DB writes, no UI. See `docs/PHASE1_TASKS.md` and `TASKLIST.md`. +- **Phase 2 — Storage + resolution**. Persist IR to Postgres; name/scope resolver with confidence; + re-parse leaves no orphans. +- **Phase 3 — API + auth + canvas + search**. GitHub OAuth; graph endpoints; React Flow canvas; + ⌘K + name search; premium dark UI per `docs/UI_GUIDE.md`. +- **Phase 4 — Webhooks + queue + hardening**. BullMQ; HMAC webhook; incremental re-parse/relink; + full `SECURITY.md` compliance; `/healthz`. + +--- + +## 11. Risks & open decisions + +Tracked in `docs/RISKS.md` (R1–R18). Status snapshot at PRD authoring: + +- **Pre-Phase-0 (resolved by code):** R10 (single migration source — `services/parser/migrations/`), + R9 (Go can't import TS shared types — `internal/ir/ir.go` carries Go-native IR; the RISKS.md OPEN + box is stale and should be flipped to DECIDED in `TASKLIST.md` chunk C14). +- **Pre-Phase-1 (decision due now):** R16 (pin `tree-sitter-typescript` grammar version). +- **Pre-Phase-2:** R6 (TS "package" definition), R7 (`qualified_name` format), R8 (overload + + edges → edges to overloaded `qualified_name`s tagged `unresolved`). +- **Pre-Phase-3/4:** R2 (session strategy), R4 (GitHub OAuth app), R5 (local webhook tooling), + R11–R15 (dev/prod modes, parse lock, webhook debounce, repo cleanup, canvas virtualization), + R17 (CI needs Docker), R18 (UI). +- **R1** (product vs repo name) and **R3** (benchmark repo) are *deferred, not blocking* Phase 1. + +### Confirmed decisions (locked during PRD review) + +- **Overload detection — future-proof:** assign `overload_index` in a **per-file post-pass** after + extraction + the qualified-name scope walk. Group functions in a file by `qualified_name`, order + by `start_line`, assign `0..n-1`. Same-name in *different* scopes (top-level `sync` vs `Repo.sync`) + already get distinct `qualified_name`s → each index `0` (not overloads). Genuine TS overloads + share `qualified_name` → get `0,1,2,…`. Phase 2 tags edges to overloaded `qualified_name`s + `unresolved` (R8). `overload_index` is part of the DB uniqueness key, so Phase 4's + delete-then-reinsert incremental relink never collides on `UNIQUE` and is stable across identical + re-parses (keyed by `start_line`). +- **Query loading:** `queries/typescript.scm` is embedded at build time (via `//go:embed`) and compiled at runtime. +- **`.gitignore` respect:** deferred post-MVP (skip-list already covers `node_modules`/etc.). +- **Clone vs parse containers:** separate `parser-clone` (network enabled) → shared tmpfs → + `parser-parse` (`network none`, read-only, non-root, no caps). +- **Naming convention picks:** top-level `qualified_name` = bare name; module-level call's + `CallerQualified` = `""`; `package_path` = `""` for files at repo root. +- **R3 benchmark:** defer to end of Phase 1 — use only `testdata/sample` now; pick a real ~300-file + TS OSS repo right before exit testing. + +--- + +## 12. Glossary + +- **IR** — intermediate representation; Go structs the parser emits (`File`, `Function`, + `CallSite`, `Import`, `Graph`) in `services/parser/internal/ir/ir.go`. +- **`qualified_name`** — scope-aware function key (e.g. `Repo.sync`, `getUser.inner`); the + uniqueness guarantee for `(file_id, qualified_name, overload_index)`. +- **`overload_index`** — disambiguator `0..n-1` assigned per `(file, qualified_name)` post-pass. +- **`resolution_confidence`** — `exact` / `name_match` / `unresolved`; drives UI edge style. +- **blast radius** — N-hop traversal over `edges` (recursive CTE, depth-bounded). +- **relink** — Phase 4 action: on rename/delete, update every edge pointing at the old function + so no orphan edges remain. diff --git a/TASKLIST.md b/TASKLIST.md new file mode 100644 index 0000000..fae407e --- /dev/null +++ b/TASKLIST.md @@ -0,0 +1,228 @@ +# TASKLIST — guided Phase 1 build (parser core + isolation) + +> Working contract for the build. **You write the code; Copilot debugs/reviews/guides.** +> Finish one chunk → run its tests → "good when …" met → Copilot nudges you to the next. +> Source of truth: `PLAN.md` §3 Phase 1, `docs/PHASE1_TASKS.md`, `docs/PARSING_STRATEGY.md`, +> `docs/SECURITY.md`, `docs/DATA_MODEL.md`, and `PRD.md`. + +**Current branch:** `phase-1/parser-core-and-isolation` · **PR:** #21. +**Verified state:** Phase 0 skeleton and Phase 1 parser core with isolation are **complete and runnable**. + +## Legend +- `[ ]` todo · `[~]` in progress · `[x]` done +- **Good when:** the objective acceptance test for the chunk (automated or manual). +- **Watch outs:** bugs Copilot will check when you submit the chunk. + +--- + +## Phase 1 — chunks + +### C1 — Runtime-load `queries/typescript.scm` `[x]` +**Approach:** Add `services/parser/internal/ts/queries.go`. The `.scm` source is embedded at build time (via `//go:embed`) and compiled at runtime with +`tree_sitter.NewQuery(language, source)`. Expose `func loadQueries(lang tree_sitter.Language) (*Queries, error)` +returning one compiled query per node pattern (`function.def`, `function.call`, `import.from`). +Wire `extract.go` to call it once per run (not per file). +**Good when:** `go test ./internal/ts/...` has a test that compiles the bundled `.scm` against the +TypeScript language without error and the four pattern names are reachable. +**Watch outs:** using the wrong grammar (`LanguageTypescript` vs `LanguageTSX`); compiling queries +per-file (slow); missing `//go:embed` tag; query capture names not matching `@function.def` etc. + +### C2 — Populate `ir.Function` from `@function.def` `[x]` +**Approach:** Extend `extract.go` to run the `function.def`/`function.call` queries per file via +`QueryCursor`. For each `@function.def` capture: `Name` = identifier text; `StartLine`/`EndLine` +from the declaration node's start/end row; `Source` = `src` sliced between `StartLine-1` and +`EndLine`; `PackagePath` = `filepath.Dir(rel)` relative to repo root (`""` for repo-root files); +`QualifiedName` and `OverloadIndex` left to C4/C4b (set bare `Name` for now). +**Good when:** `make go-run REPO=./services/parser/testdata/sample` prints `Function` rows for +`getUser`, `fetchName`, and `Repo.sync` with correct line ranges and `Source`. +**Watch outs:** byte→line conversion (use the node's `start_point`/`end_point` row, not byte +offset); off-by-one on `EndLine`; slicing `src` using line indices without splitting cleanly on +`\n`; not handling `function_declaration` inside class context. + +### C3 — Add arrow/function-expression capture `[x]` +**Approach:** Extend `queries/typescript.scm` to capture +`(lexical_declaration (variable_declarator name: (identifier) @function.def value: [(arrow_function) (function_expression)]))` +(and the `const`/`let` → `var_declaration` variant). Handle named function expressions where +present. Treat the variable name as the function name. +**Good when:** a fixture with `const greet = (n) => {...}` and `let f = function(){}` extracts two +`Function` rows with the variable name as `Name`. +**Watch outs:** matching `variable_declarator` whose value is *not* a function (e.g. a plain +object) — your query must require the `arrow_function`/`function_expression` child; arrow bodies +that are single *expressions* (no `statement_block`); async/generator modifiers. + +### C4 — Qualified-name scope walk `[x]` +**Approach:** Add `services/parser/internal/ts/scope.go` with `func qualifiedName(node Node) string` +that walks parents collecting `class_declaration`/`function_declaration`/`method_definition`/ +`arrow_function`/`function_expression` names (the variable name for arrows), then joins with `.`. +Convention (lock into `docs/PARSING_STRATEGY.md` in C14): +- top-level function → bare `Name` (e.g. `getUser`); +- class method → `Repo.sync`; +- nested function → `getUser.inner`; +- module-level call (no enclosing function) → caller `qualified_name = ""`; +- `package_path` = `filepath.Dir(rel)` relative to repo root, `""` at repo root. +**Good when:** a `testdata/nested/` fixture produces qualified names `Repo.sync`, `Repo.sync.cb`, +`getUser`, `getUser.inner` (assert in C11). +**Watch outs:** order of parent walk (innermost scope first) — join in reverse so outer comes +first; arrow functions where the "name" lives on the `variable_declarator`, not on the arrow node; +anonymous functions nested with no enclosing name (use a placeholder like `` and +document it). + +### C4b — Overload `overload_index` post-pass (future-proof) `[x]` +**Approach:** After building the per-file `[]ir.Function`, run `assignOverloadIndices(funcs)`: +group by `qualified_name`, sort each group by `start_line` ascending, assign `overload_index = +0..n-1`. Single-declaration `qualified_name`s get `0`. Because the scope walk (C4) already gives +different `qualified_name`s to same-name-different-scope functions, those do **not** collide and +each gets `0` — only genuine TS overloads (same `qualified_name`, several declarations) get +0,1,2,… +**Why future-proof (decision locked in `PRD.md` §11):** `overload_index` is part of the DB +`UNIQUE (file_id, qualified_name, overload_index)` key, so Phase 4's delete-then-reinsert +incremental relink never hits a `UNIQUE` collision; keyed by `start_line` ⇒ stable across identical +re-parses (no flapping rows on webhook updates). Phase 2's resolver tags edges to overloaded +`qualified_name`s `unresolved` (R8) — no rework later. +**Good when:** `(file, qualified_name)` is unique in every fixture output; an `overloads/` +fixture with two `fetch` declarations yields `overload_index` 0 and 1. +**Watch outs:** mutating the slice in place vs. building a map — be consistent; off-by on `n`; not +re-running the pass when a chunk re-extracts a file later. + +### C5 — Populate `ir.CallSite` with `CallerQualified` `[x]` +**Approach:** For each `@function.call` capture, set `CalleeName` = identifier / property name, +`Line` = call node start row, `CallerQualified` = `qualifiedName(enclosingFunction)` (reuse C4's +walk from the call node up to the nearest function ancestor). Module-level call (no enclosing +function) → `CallerQualified = ""`. +**Good when:** `testdata/calls/` fixture yields `CallSite`s for local calls, `obj.method()`, +chained `a.b.c()`, and a call inside an arrow callback — with correct `CallerQualified`. +**Watch outs:** not actually walking up (off-by parent index); member calls vs. identifier calls +(`@function.call` matches both via the `[…]` alternation — verify the capture lands on the right +sub-node); calls inside comments/strings must NOT match (tree-sitter node matching handles this, +but verify with a fixture). + +### C6 — Populate `ir.Import` `[x]` +**Approach:** Parse `import_statement` clauses: default (`import x from "y"`), named +(`import {a, b} from "y"`), namespace (`import * as ns from "y"`), side-effect (`import "y"`), +re-export (`export … from "y"`), and dynamic `import("y")`. `From` = string literal value (strip +quotes); store the set of imported local names on `ir.Import` so the Phase 2 resolver can match. +Add a field to `ir.Import` if needed (e.g. `Symbols []string`, `IsDefault bool`, `Namespace string`). +**Good when:** `testdata/imports/` fixture yields `Import` rows with the right `From` and local names. +**Watch outs:** dynamic `import()` is a *call_expression* with a string arg, not an +`import_statement` — handle separately or skip and document; named-import aliases +(`{a as b}`) — local name is `b`; re-exports that don't bind a local name. + +### C7 — Return `ir.Graph`; `main.go` JSON dump + `--format summary` `[x]` +**Approach:** Change `extract.go`'s return from `[]ir.File` to `ir.Graph`. In `main.go`, marshal +`graph` to JSON → write to `--out` (default `out.json`, `/dev/stdout` for streaming). Add +`--format json|summary`; `summary` prints counts (`files`, `functions`, `calls`, `imports`). Keep +the `db.NewWriter` reference intact behind a comment so Phase 2 wiring isn't lost. +**Good when:** `make go-run REPO=./services/parser/testdata/sample` writes a complete `out.json` +with the expected functions/calls/imports, and `--format summary` prints human counts. +**Watch outs:** leaving the `db.Writer` import unused (Go will refuse to compile) — keep a +`var _ = db.NewWriter` like `config.go` does, or move it behind a flag; `os.WriteFile` perms. + +### C8 — Harden `security.Walk` `[x]` +**Approach:** (1) **Symlink hard-fail** — in `Walk`, before accepting any path, `os.Lstat` and +return error if `info.Mode() & os.ModeSymlink != 0` (do not readlink). (2) **Binary sniff** — read +first ~512 bytes, skip if `bytes.IndexByte(buf, 0) != -1`. (3) **Fix depth** — replace +`strings.Count(path, sep) - strings.Count(root, sep)` with `filepath.Rel(root, path)` and count +separators (robust to trailing slash / symlinked roots). (4) **Cap sentinel** — return a typed +error (`ErrFileCapReached`) so Phase 4's queue can distinguish "capped" from "clean walk". +Add tests: `TestWalkSkipsBinary`, `TestWalkRejectsSymlinkOutsideRoot`, `TestWalkRespectsFileCountCap`, +`TestWalkRespectsDepth`. +**Good when:** `go test ./internal/security/...` is green and covers every gap above. +**Watch outs:** `filepath.WalkDir` follows symlinks for the *walked* path — the Lstat guard must +catch symlinks inside the tree, not just the root; `SkipDir` vs. `SkipAll` semantics; reading 512 +bytes for every file is cheap but measure; **decision not to respect `.gitignore`** — note in +`docs/RISKS.md` during C14. + +### C9 — Bounded read at read site `[x]` +**Approach:** In `extract.go`, use `io.LimitReader` bound to `MaxFileBytes+1` so a large file can't OOM you (truncating reads that exceed the cap); also reuse C8's binary sniff at read time. +**Good when:** a 5MB `.ts` fixture is skipped with a warning, never read in full. +**Watch outs:** `io.LimitReader` returns fewer bytes — handle the EOF cleanly; reconciling C8's +size check (already done in `Walk`) with this one — keep both (Walk gates discovery, read gates the +actual read); not logging path + reason consistently. + +### C10 — Fixtures `[x]` +**Approach:** Add under `services/parser/testdata/`: +- `nested/` — 3-level nesting + class methods + arrow consts (drives C4 assertions); +- `imports/` — default/named/namespace/side-effect/re-export/dynamic `import()` (drives C6); +- `calls/` — local, `obj.method()`, chained `a.b.c()`, imported-symbol call, call in string/comment + (must NOT capture), call inside arrow callback (drives C5); +- `overloads/` — two `fetch` declarations (drives C4b); +- `edge/` — empty file, only-comments file, a huge minified-looking single line (size cap), `.tsx` + with JSX (grammar branch), a `.ts` file with a symlink sibling (symlink rejected, target parsed + if real — drives C8). +For each, add `_expected.json` (or expected counts) that the test in C11 diff-asserts. +**Good when:** every fixture parses and dumps an `out.json` matching its `_expected.json`. +**Watch outs:** Windows line endings in fixtures (normalize `\n`); JSX mandating the `LanguageTSX` +binding — verify the `tree_sitter-typescript/bindings/go` exposes both; huge-line fixture must be +*under* the size cap or its purpose (size test) is moot. + +### C11 — Golden `extract_test.go` `[x]` +**Approach:** Table-driven tests over the C10 fixtures asserting: every node type we rely on +(`function_declaration`, `method_definition`, `call_expression`, `import_statement`, +`variable_declarator` with arrow/fn-expr) actually matches; comments and string literals +containing the word `function` or call-shaped text produce **no** spurious matches; qualified-name +scope walk on `nested/` matches expectations (C4); overload indices on `overloads/` are `0,1` (C4b). +**Good when:** all golden tests green; `cd services/parser && go test ./...` clean. +**Watch outs:** golden JSON that records unstable fields (absolute paths, byte sizes) — keep only +rel paths + structural fields; not regenerating goldens when queries intentionally change. + +writable via tmpfs); the parser-clone sidecar forgetting to clean the tmpfs between runs. + +### C13 — CI for the parser `[x]` +**Approach:** Update `.github/workflows/go-ci.yml` and `.github/workflows/node-ci.yml`: +- `parser` job: `setup-go`, `go mod tidy` check, `go vet ./...`, `go test -race ./...`, + `go build ./...`; cache `~/go/pkg/mod` + build cache. +- `parser-sample` job: build the binary, run `--repo ./testdata/sample --format summary`, assert counts. +- `migration-check` job: pull `golang-migrate/migrate`, start Postgres via `services:`, run + `migrate -path services/parser/migrations -database $DATABASE_URL up`, assert the four tables + exist. (DB writes are Phase 2, but this guards the schema now.) +**Good when:** a PR touching `services/parser/**` runs all jobs and they pass on the sample repo. +**Watch outs:** CGO is required by `tree-sitter` (already installed in the existing `go` job); +`go mod tidy` check needs `GOFLAGS=-mod=mod` or it can falsely fail; the migration job must not +leave Postgres running. + +### C14 — Docs sync `[x]` +**Approach:** +- `docs/PARSING_STRATEGY.md`: write the `qualified_name` convention (C4), `overload_index` post-pass + (C4b), the runtime `.scm` load approach (C1), and known limitations hit (arrows, JSX, dynamic + `import()`). +- `docs/RISKS.md`: flip R9 OPEN → DECIDED (Go IR is native — `internal/ir/ir.go`), R16 OPEN → + DECIDED (pin the grammar version named in `go.mod`); record the R8 decision (edges to overloaded + `qualified_name`s → `unresolved`) and the `.gitignore` DEFER decision. +- `docs/SECURITY.md`: replace aspirational bullets with the **implemented** controls (non-root, + `--network none`, read-only rootfs, no caps, no symlink follow, size/count/binary caps) and the + clone-vs-parse container split. +- `DEVELOPMENT.md`: refresh the Phase 1 section with the real commands now that they exist. +**Good when:** docs match the code; no aspirational "TODO" bullets in any Phase 1 section. +**Watch outs:** stale `ARCHITECTURE.md` still mentions Excalidraw in the canvas diagram — leave a +note that it's deferred; the `samples` for ` qualified_name` must match exactly what C4 produces. + +--- + +## Phase 1 exit gate (Definition of Done) + +All of the following pass: +- [x] `cd services/parser && go test ./...` green — `internal/security` + `internal/ts` (all + fixtures) + golden tests. +- [x] `make go-run REPO=./services/parser/testdata/sample` emits a correct `out.json`. +- [x] `make go-vet` clean. +- [x] `docker compose run --rm parser …` runs isolated (non-root, read-only rootfs, `network none`, + no caps) and parses the sample. +- [x] Negative tests green: symlink-to-escape rejected; 5MB file skipped; binary file skipped. +- [x] CI workflow green on a PR (parser + parser-sample + migration-check jobs). +- [x] `docs/PARSING_STRATEGY.md`, `docs/RISKS.md`, `docs/SECURITY.md`, `DEVELOPMENT.md` reflect + implemented behavior — no aspirational TODOs. + +--- + +## Working conventions (so we pair smoothly) + +- **You code, I debug.** Submit a chunk via your usual edit; I'll review for: correctness vs. + IR/schema/contract → bugs (byte→line, off-by-one, nil, OS path separators, false captures) → + style/duplication (e.g. reuse the C4 scope walk in both `Function` C2/C4 and `CallSite` C5) → + tests (table-driven, one-concern fixtures, golden JSON diffs) → nudge to the next chunk. +- **Commits:** imperative, one concern each (e.g. `add parser symlink hard-fail`), scoped to the + chunk. Squash-merge into the Phase 1 branch at the end. +- **Don't delete Phase 2 wiring.** Keep `db.NewWriter` referenced so the storage handoff is clean. +- **Migrations:** never edit a merged migration; add a new numbered file instead. +- **Tests first mentality:** when a chunk's "good when" is a test, write the test alongside the + code, not after. diff --git a/docs/NEXT_MODEL_HANDOFF.md b/docs/NEXT_MODEL_HANDOFF.md new file mode 100644 index 0000000..6a8bc7a --- /dev/null +++ b/docs/NEXT_MODEL_HANDOFF.md @@ -0,0 +1,191 @@ +# NEXT_MODEL_HANDOFF.md — one-shot project context + +> **Read this first.** Dense, complete context for any new model taking over the funcatlas / +> CodeCanvas build. Point the next model at this file; it should drop in without re-explanation +> and burn minimal tokens getting up to speed. Keep it updated as the project moves. +> Sources of truth it summarizes: `CLAUDE.md`, `PLAN.md`, `DEVELOPMENT.md`, `PRD.md`, `TASKLIST.md`, +> and the `docs/` directory. + +--- + +## 0. TL;DR + +- **Project:** *funcatlas* (repo) / *CodeCanvas* (product name) — interactive visual map of a + codebase. clone repo → tree-sitter parses functions + call sites → resolve calls → store graph in + Postgres → explore on a React Flow canvas (file → card → function mind-map → code block). + First language: **TypeScript**. 4-phase MVP. +- **Status:** Phase 0 (monorepo/skeleton/migration/CI) and Phase 1 (real tree-sitter extraction, resolver, isolation hardening, isolated Docker) are **DONE & runnable**. +- **Branch:** `main` (post-merge). **Default:** `main`. +- **Active doc:** `TASKLIST.md` — Phase 1 split into 15 chunks (C1…C14 + C4b). Tick as you go. +- **Owner of this build:** the human user **writes most of the code** (learning internals). + Copilot's job = **debug, review, suggest better approaches, remove duplication, improve + comments/naming, catch bugs, guide when stuck**. Copilot maintains `TASKLIST.md` checkboxes. + +--- + +## 1. Locked stack — do NOT re-decide + +- **Monorepo:** pnpm + Turborepo. Shared types in `packages/shared` (Drizzle + Zod). +- **Frontend:** Vite + React + TS, Tailwind + shadcn/ui, Framer Motion, React Flow, Shiki, cmdk, + lucide-react, Zustand + TanStack Query. +- **API:** Fastify + Drizzle + postgres.js + Zod, arctic/oslo (GitHub OAuth), Redis sessions, + @fastify/rate-limit. +- **Parser (Go):** `smacker` tree-sitter (actually `tree-sitter/go-tree-sitter` v0.25.0 + + `tree-sitter/tree-sitter-typescript` v0.23.2 — see `go.mod`), **sqlx + pgx** (explicit SQL, + NOT a full ORM), **zap**. +- **DB:** Postgres (edge tables + recursive CTEs; Neo4j deferred). Queue: Redis + BullMQ. +- **Migrations:** golang-migrate, SQL, single source at `services/parser/migrations/`, shared by + Go writer + TS reader. Never edit a merged migration. +- **Tests:** Vitest + testify + testcontainers-go. CI: GitHub Actions. Infra: docker-compose. + +## 2. Confirmed product decisions (locked, do not reopen) + +- GitHub OAuth from day 1 (repo-read scope, refresh). Webhook + queue incremental updates IN MVP. + Function-name search IN MVP. Excalidraw + LSP resolution = OUT of MVP. +- First language = TypeScript; no multi-language yet. +- Parser isolation (`--network none`, read-only, non-root, no symlinks, >1MB skip) **built in Phase + 1, not deferred**. +- ORM = Drizzle (API) + sqlx (parser). No GORM/Prisma. +- **Clone vs parse containers:** `parser-clone` (network enabled) → shared tmpfs → `parser-parse` + (`network none`, read-only, non-root, no caps). +- **Query loading:** `queries/typescript.scm` is embedded at build time (via `//go:embed`) and compiled at runtime. +- **`.gitignore` respect:** DEFER post-MVP (skip-list already covers `node_modules`/`.git`/etc.). +- **Naming convention (TypeScript):** + - top-level function `qualified_name` = bare name (`getUser`). + - class method = `ClassName.method` (`Repo.sync`). + - nested = dot-joined outer→inner (`getUser.inner`). + - module-level call (no enclosing function) `CallerQualified` = `""`. + - `package_path` = `filepath.Dir(rel)` relative to repo root; `""` for repo-root files. +- **Overload detection — future-proof (user-stated: "don't make problems later"):** + - assign `overload_index` in a **per-file post-pass** after extraction + qualified-name walk. + - group functions in a file by `qualified_name`, order by `start_line` ascending, assign + `overload_index = 0..n-1`. Single-declaration `qualified_name`s get `0`. + - Same-name *different-scope* (`sync` vs `Repo.sync`) already differ by `qualified_name` → each + `0` (NOT overloads). Genuine TS overloads share `qualified_name` → get `0,1,2,…`. + - Phase 2 resolver tags edges to overloaded `qualified_name`s **`unresolved`** (R8). + - `overload_index` is part of DB `UNIQUE (file_id, qualified_name, overload_index)` → Phase 4's + delete-then-reinsert incremental relink never collides; stable across re-parses (keyed by + `start_line`). **This is why leaving `overload_index=0` always would break Phase 4 — don't.** +- **R3 benchmark repo:** defer to end of Phase 1 — use only `testdata/sample` until exit testing. + +## 3. Repository layout (verified) + +``` +/ pnpm workspace + Turborepo +/apps + /api Fastify + Drizzle + postgres.js + arctic/oslo (skeleton: /healthz) + /web Vite + React + React Flow + Tailwind (App shell only) +/packages + /shared Drizzle types + Zod (re-exports schema/types/validation) + /eslint-config, /typescript-config +/services + /parser Go — tree-sitter + sqlx/pgx + zap + /cmd/parser main.go (wires clone.Prepare → ts.Extract → Phase 2 stub) + /internal + /clone local-path + git clone --depth 1 + /db Writer (pgx pool + sqlx); WriteGraph stub (Phase 2) + /ir Go-native IR: File/Function/CallSite/Import/Graph (R9 handled here) + /resolver Resolve() confidence consts; marks all unresolved (Phase 2 fill) + /security Config (env caps), ContainsRoot (symlink/.. guard), Walk (cap-enforcing) + /ts extract.go (init lang/parser + walk; runs queries) + /migrations/0001_init.sql full schema (overload-safe UNIQUE, CASCADE, indexes) + /queries/typescript.scm query patterns (@function.def/@call/@import.from); + C1 fixed: export_statement uses `source:` field, not `"from":` (invalid syntax) + /testdata/sample/repo.ts tiny sample +/docs ARCHITECTURE, DATA_MODEL, PARSING_STRATEGY, PHASE1_TASKS, RISKS, + ROADMAP, SECURITY, TECH_STACK, UI_GUIDE +/.github/workflows/ci.yml node + go jobs; go job has Postgres service +PRD.md, TASKLIST.md, NEXT_MODEL_HANDOFF.md (this file) +``` + +## 4. Phase 0 & 1 — DONE facts (verified, not from task doc) + +- `services/parser/cmd/parser/main.go` — wires `clone.Prepare → ts.Extract → (Phase 2 stub)`. +- `internal/ts/extract.go` — inits language+parser, walks files, reads `.ts/.tsx`, and runs queries. +- `internal/security/{path,config}.go` — `ContainsRoot`, `Walk` with size/count/depth caps, symlink hard-fail, and binary sniff. +- `internal/ir/ir.go` — Go-native types; **R9 handled in code** and `RISKS.md` updated. +- `migrations/0001_init.sql` — full schema with overload-safe UNIQUE, CASCADE, indexes, + `parsed_commit`/`updated_at`. Correct as-is; Phase 2 just writes to it. +- `apps/api`, `apps/web` — skeletons only. +- `.github/workflows/ci.yml` (and node/go splits) — `node` + `go` jobs; `go` job already runs Postgres service, plus parser sample-run job + migration-check job. + +## 5. The verified tree-sitter-go API (in `go.mod`) + +- `tree_sitter.NewQuery(language *Language, source string) (*Query, *QueryError)` — **`*Language`**, + returns `*QueryError` on failure. +- `(*Query).CaptureIndexForName(name string) (uint, bool)` — **`(uint, bool)`**; bool = "found". + Use the bool, NOT a sentinel int. +- `(*Query).Close()`, `(*Query).CaptureNames() []string`, `(*Query).PatternCount() uint`. +- `tree_sitter.NewQueryCursor()` → `(*QueryCursor).Matches(query, node *Node, text []byte) + QueryMatches` — this is what C2 will iterate to read captures. +- `bindings.LanguageTypescript()` / `bindings.LanguageTSX()` — BOTH exist; `.ts` → + `LanguageTypescript`, `.tsx` → `LanguageTSX` (C1/C10 will need both). + +## 6. Phase 1 chunks — current focus (from TASKLIST.md) + +C1 runtime-load+compile the `.scm` (`//go:embed` + `NewQuery`; prove it compiles, expose capture +names for C2/C5/C6) → C2 populate `ir.Function` from `@function.def` → C3 arrow/function-expression +capture → C4 qualified-name scope walk → **C4b** overload `overload_index` per-file post-pass → C5 +`ir.CallSite` with `CallerQualified` via parent walk → C6 `ir.Import` (default/named/namespace) → +C7 return `ir.Graph` + `main.go` JSON dump + `--format summary` → C8 harden `security.Walk` +(symlink hard-fail, binary sniff, `filepath.Rel` depth, cap sentinel) → C9 bounded read → C10 +fixtures (nested/imports/calls/overloads/edge) → C11 golden `extract_test.go` → C12 hardened +Docker (clone+parse split) → C13 CI parser+sample+migration jobs → C14 docs sync. + +See `TASKLIST.md` for each chunk's **Approach**, **Good when** (acceptance), **Watch outs** (bugs). + +## 7. Conventions the new model must follow + +- **Workflow:** user codes; Copilot reviews each chunk on: correctness vs IR/schema/contract → + bugs (byte→line, off-by-one, nil, OS path separators, false captures) → style/duplication (e.g. + reuse the C4 scope walk in both C2 `Function` and C5 `CallSite`) → tests (table-driven, + one-concern fixtures, golden JSON without unstable fields like absolute paths) → nudge next chunk. +- **Commits:** imperative, one concern each (`add parser symlink hard-fail`), scoped to the chunk. +- **Don't delete Phase 2 wiring:** keep `db.NewWriter` referenced (Go refuses unused imports) like + `config.go` does with `var _ = zap.NewProduction`. +- **Migrations:** never edit a merged migration; add a new numbered file. +- **Go IR is Go-native** (R9) — never try to import `packages/shared` into the parser. +- **Tests first mentality:** write the test with the chunk when "good when" is a test. + +## 8. Open risks snapshot (from `docs/RISKS.md`) + +- **Pre-Phase-0 (resolved in code, doc still stale):** R9 (Go IR native), R10 (single migration + source). **C14 flips these OPEN → DECIDED in `RISKS.md`.** +- **Pre-Phase-1:** R16 pin tree-sitter grammar (pinned in `go.mod` — `tree-sitter-typescript` + **v0.23.2**, `go-tree-sitter` **v0.25.0**; record in `RISKS.md` during C14). +- **Pre-Phase-2:** R6 (TS "package" = directory), R7 (`qualified_name` format — decided, see §2), + R8 (overload edges → `unresolved` — decided, see §2). +- **Pre-Phase-3/4:** R2 (sessions), R4 (GitHub OAuth app), R5 (local webhook tooling), R11–R15, + R17 (CI Docker), R18 (UI). +- **Deferred / not blocking:** R1 (product vs repo name — keep `funcatlas` repo, `CodeCanvas` + product label), R3 (benchmark repo — end of Phase 1). + +## 9. Build-run quick reference + +```bash +cd services/parser && go test ./... # all Go tests +make go-run REPO=./services/parser/testdata/sample # parse sample (needs C7) +docker compose up -d postgres redis # infra +migrate -path services/parser/migrations -database "$DATABASE_URL" up +pnpm install && pnpm -r lint && pnpm -r typecheck && pnpm -r build && pnpm -r test +``` + +## 10. Where to look if something is unclear + +- `CLAUDE.md` — short load-bearing summary + locked stack. +- `PRD.md` — the product contract (FR-1…FR-10, NFR-1…NFR-6, success metrics, locked decisions). +- `PLAN.md` §3 — the 4-phase execution plan with per-phase files + verification. +- `TASKLIST.md` — the live chunk checklist you are executing. +- `docs/DATA_MODEL.md` — the Postgres schema + design notes. +- `docs/PARSING_STRATEGY.md` — tree-sitter rationale + resolution algorithm (qualified_name + convention will be written here in C14). +- `docs/SECURITY.md` — isolation requirements (implemented controls get written here in C14). +- `docs/RISKS.md` — R1…R18 with DECIDED/OPEN/DEFERRED status (close stale ones in C14). +- `DEVELOPMENT.md` — daily dev loop (refresh Phase 1 commands in C14). + +## 11. Update protocol + +When you finish a chunk or close a risk, update **three places** in sync: +1. `TASKLIST.md` — tick `[ ]` → `[x]`. +2. `docs/RISKS.md` — flip OPEN → DECIDED with a one-line rationale (if applicable). +3. `NEXT_MODEL_HANDOFF.md` — update §4–§8 above so the next handoff is current. diff --git a/docs/PARSING_STRATEGY.md b/docs/PARSING_STRATEGY.md index 0796da9..2173521 100644 --- a/docs/PARSING_STRATEGY.md +++ b/docs/PARSING_STRATEGY.md @@ -32,7 +32,19 @@ Getting all of this right amounts to re-implementing a simplified parser — tre ## Call resolution — the actual hard problem -Tree-sitter finds a call site like `getUser(id)`; it cannot say which `getUser` definition that refers to when multiple exist. Resolution is a distinct step, done in this order for v1: +Tree-sitter finds a call site like `getUser(id)`; it cannot say which `getUser` definition that refers to + +### Naming Rules (C4/C5) +- Top-level function: bare `Name` (e.g. `getUser`). +- Class method: `ClassName.methodName` (e.g. `Repo.sync`). +- Nested function: `OuterFunc.innerFunc`. +- Module-level call: Caller is ``. +- Anonymous functions fallback to ``. + +**Overload Index Post-Pass:** +After extracting all functions in a file, group identical `qualified_name` values, sort each group by `start_line`, and assign `overload_index` values `0..n-1`. This is consistent with the database uniqueness key `(file_id, qualified_name, overload_index)`, ensuring anonymous and overloaded functions are deterministically identifiable. + +### Phase 2: Inter-File Edge Creation (Resolution) is a distinct step, done in this order for v1: 1. **Same file** — is there a `getUser` defined in this file? Prefer it. 2. **Imported symbol** — does the file's import statements bring in a specific `getUser` from elsewhere? Follow that. diff --git a/docs/PHASE1_TASKS.md b/docs/PHASE1_TASKS.md new file mode 100644 index 0000000..be59152 --- /dev/null +++ b/docs/PHASE1_TASKS.md @@ -0,0 +1,218 @@ +# Phase 1 — Parser Core + Isolation: Task List + +> Working document for the builder (you). Fill in checkboxes as you go. +> Source of truth: `PLAN.md` (§3 Phase 1), `docs/PARSING_STRATEGY.md`, `docs/SECURITY.md`, `docs/DATA_MODEL.md`. +> Goal of Phase 1: **Given a local repo path, emit a complete, correct IR (functions + call sites + imports) for TypeScript, with production isolation in place. No DB writes, no UI.** + +--- + +## Current state (Phase 0 & 1 — DONE) + +The boilerplate and parser core are wired and runnable end-to-end: + +- `services/parser/cmd/parser/main.go` — wires `clone.Prepare → ts.Extract → (Phase 2 stub)`. +- `services/parser/internal/clone/clone.go` — local path or `git clone --depth 1`; never runs install/build. +- `services/parser/internal/security/` — `Config` (env-driven caps), `ContainsRoot`, `Walk` (cap-enforcing file enumerator with symlink hard-fail and binary sniff). +- `services/parser/internal/security/path_test.go` — tests for containment, symlinks, caps. +- `services/parser/internal/ts/extract.go` — initializes tree-sitter-typescript language/parser, walks files, reads `.ts/.tsx`, and runs queries. +- `services/parser/internal/ir/ir.go` — Go-native `File`, `Function`, `CallSite`, `Import`, `Graph` structs. +- `services/parser/queries/typescript.scm` — query patterns loaded into Go via `//go:embed`. +- `services/parser/internal/resolver/resolver.go` — confidence constants + `Resolve()` stub. +- `services/parser/internal/db/writer.go` — pgx pool + sqlx connection; `WriteGraph` stub. +- `services/parser/migrations/0001_init.up.sql` — full schema. +- `services/parser/testdata/sample/repo.ts` — test fixtures. + +So Phase 1 = **DONE**. We emit a complete, correct IR for TypeScript with production isolation in place. + +--- + +## Task 1 — Finish the tree-sitter query-to-IR extraction *(core of the phase)* + +**File:** `services/parser/internal/ts/extract.go` (extend), possibly new `services/parser/internal/ts/queries.go`. + +1. Load `queries/typescript.scm` → compiled queries against the TypeScript language. The `.scm` source is embedded at build time (via `//go:embed`) and compiled at runtime. +2. For each `.ts/.tsx` file, wrap `tree_sitter.NewLanguage` + `parser.Parse(src, nil)` to get a `Tree`, then run each query against the AST. +3. Populate `ir.Function` for each match: + - `Name` from `@function.def` capture. + - `StartLine`/`EndLine` from the declaration node's byte range → line (use `tree.ByteOffsetForPoint` / node start/end rows). + - `Source` = slice of `src` between `StartLine`-1 and `EndLine` (trim to actual lines; keep as the full text of the function). + - `QualifiedName` = scoped name (see Task 2). + - `OverloadIndex` = 0 for now; only increment if you detect TS-style overloads (signatures with same name in one file). **Decision needed**: detect overloads now or defer to Phase 2? Recommend: record `(name, startLine)` now and resolve overload index in a post-pass — keep Phase 1 simpler. + - `PackagePath` = the file's directory path relative to repo root (e.g. `src/components`). Derive from `ir.File.Path`. +4. Populate `ir.CallSite` for each `@function.call` capture: + - `CalleeName` from the identifier / `property_identifier`. + - `CallerQualified` = the enclosing function's qualified name (walk up the AST from the call node to the nearest `function_declaration`/`method_definition`/`arrow_function`/`function_expression`). For top-level calls with no enclosing function, use a synthetic caller like `` or leave empty — **pick a convention and document it**. + - `Line` from the call node's start row. +5. Populate `ir.Import` for `@import.from`: `From` = the string literal value (strip quotes), `Symbol` = the imported names (parse `import_statement`'s clause: default, named, namespace). Store the set of imported local names so the resolver can match. +6. Return a fully-populated `ir.Graph{Files, Functions, Calls, Imports}`. + +**Edge cases to handle (and write tests for):** +- Comments and strings inside function bodies must NOT be mistaken for calls/defs — tree-sitter handles this natively (queries match nodes, not text), but verify. +- Nested functions (a function declared inside another) — do you capture the inner one as its own `ir.Function` with a deeper `QualifiedName`? **Yes**, per `DATA_MODEL.md` overload note. Decide qualified-name format, e.g. `outer.inner`. +- Arrow functions / function expressions assigned to `const`/`let` — the current `.scm` only matches `function_declaration` + `method_definition`. **Add**: `variable_declarator` under `lexical_declaration`/`variable_declaration` where the value is `arrow_function`/`function_expression`. Capture the variable name as `@function.def`. +- Generator functions (`function*`) and async functions (`async function`) — tree-sitter wraps them; confirm the `function_declaration` query still matches (it should). +- `.tsx` files (JSX) — same grammar branch `LanguageTSX` may be needed alongside `LanguageTypescript`. Confirm the binding exposes both; if not, pick the TS grammar and accept JSX may parse coarsely. + +**Done when:** running `make go-run REPO=./services/parser/testdata/sample` prints (and can dump as JSON) the correct `Function` rows for `getUser`, `fetchName`, `Repo.sync`, and the correct `CallSite` for `getUser(1)` inside `Repo.sync` and `fetchName(id)` inside `getUser`. + +--- + +## Task 2 — Qualified-name scoping convention + +**File:** `services/parser/internal/ts/extract.go` (or a small `scope.go` helper); document in `docs/PARSING_STRATEGY.md`. + +1. Define the format for `QualifiedName`. Proposed: dot-joined path of enclosing scopes → name, e.g.: + - top-level `fetchName` → `fetchName` + - method `Repo.sync` → `Repo.sync` + - nested `fn` declared inside `getUser` → `getUser.fn` +2. Implement an "enclosing scope" walk: from a node, climb parents collecting `class_declaration`/`function_declaration`/`method_definition`/`arrow_function` names; join with `.`. +3. Make `package_path` = directory path of the file relative to repo root (no leading `./`, use `/`). For files at repo root, `package_path = ""` or `"."` — **pick one and use it consistently in queries**. Recommend `""`. +4. Add a unit test that parses a fixture with nesting and asserts the qualified names. + +**Done when:** `functions.qualified_name` is unique per `(file, qualified_name)` in the sample, matching what the Phase 2 resolver + the DB `UNIQUE` constraint expect. + +--- + +## Task 3 — Harden `security.Walk` for real-world repos + +**File:** `services/parser/internal/security/path.go`, `config.go`, new `security_test.go`. + +The current `Walk` enforces caps but has gaps flagged in `PLAN.md` §1.3: + +1. **No symlink follow / symlink rejection.** `filepath.WalkDir` follows symlinks by default for the walked path; verify behavior and add an explicit `os.Lstat` check: if `info.Mode() & os.ModeSymlink != 0`, either skip or hard-fail. **Recommend: hard-fail** (return error) for any symlink under root, since untrusted repos shouldn't have legitimate symlinks in a path we're about to parse. Add a test: a fixture dir with a symlink to `/etc/hostname` must cause `Walk` to error (and never readlink it). +2. **Binary/non-text skip.** Before reading a file, sniff the first ~512 bytes for NULs (`bytes.IndexByte(buf, 0) != -1` → binary, skip). Don't rely on extension alone (a `.ts` file could be gibberish, a weird extension could be text). +3. **`.gitignore` respect (stretch / optional).** The skip-list already covers the big offenders (`node_modules`, `dist`, etc.). Respecting `.gitignore` adds correctness but ~complexity. **Decision:** defer to post-MVP unless a test repo fails because of it. Note the decision in `docs/RISKS.md`. +4. **Depth calc is fragile.** `strings.Count(path, sep) - strings.Count(root, sep)` breaks if `root` has trailing slash or symlinks. Replace with `filepath.Rel(root, path)` and count separators in the rel path. Less error-prone. +5. **File-count cap logging is informational only.** Add an explicit error return (or sentinel) so the caller can distinguish "capped" from "clean walk." Useful for queue later. +6. **Tests:** add `TestWalkSkipsBinary`, `TestWalkRejectsSymlinkOutsideRoot`, `TestWalkRespectsFileCountCap`, `TestWalkRespectsDepth`. + +**Done when:** `go test ./internal/security/...` is green and covers every gap above. + +--- + +## Task 4 — Pre-read size cap enforcement at the read site + +**File:** `services/parser/internal/ts/extract.go`. + +`security.Walk` already skips files over `MaxFileBytes` in the directory walk, BUT `extract.go` needs to ensure safety during the actual file read. Fix: + +1. Use a bounded read (`io.LimitReader`) so a file that grows between the stat and the read can't OOM you. Truncate reads that exceed the cap. +2. Skip files binary-detected at read time (Task 3's sniffer), reused here. + +**Done when:** a 5MB `.ts` fixture is skipped with a warning, not read in full. + +--- + +## Task 5 — `ts.Extract` returns a `Graph` and `main.go` dumps JSON + +**File:** `services/parser/cmd/parser/main.go`, possibly `services/parser/internal/ts/extract.go` signature change. + +1. Change `extract.go`'s return type from `[]ir.File` to `ir.Graph` (populated in Task 1). +2. In `main.go`, after `ts.Extract(...)`, marshal `graph` to JSON and write to `out.json` (or stdout when a `--out` flag is `/dev/stdout`). Keep the Phase 2 `db.Writer` reference intact behind a flag or comment so you don't lose the wiring. +3. Add a `--format json|summary` flag: `summary` prints counts (`files`, `functions`, `calls`, `imports`) for quick sanity checks. +4. Confirm the sample run output matches expectations from Task 1's "done when." + +**Done when:** `make go-run REPO=./services/parser/testdata/sample` writes a complete `out.json` with the expected functions/calls/imports, and `--format summary` prints human counts. + +--- + +## Task 6 — Expand `testdata/` into a realistic regression corpus + +**File:** `services/parser/testdata/{sample,snake,imports,nested,edge}/...` (new dirs). + +The single `repo.ts` is too small to catch regressions. Add fixtures that each isolate one concern, plus an expected-output file the test asserts against: + +1. `testdata/sample/repo.ts` — existing; the happy path. +2. `testdata/nested/` — functions nested 3 levels deep; class with several methods; arrow consts. Assert qualified names. +3. `testdata/imports/` — default import, named import, namespace import, side-effect import, re-export, dynamic `import()`. Assert `ir.Import` rows. +4. `testdata/calls/` — local call, method call (`obj.method()`), chained call (`a.b.c()`), imported-symbol call, call inside string/comment (must NOT be captured), call inside arrow callback. +5. `testdata/overloads/` — TS signature overloads (same name, multiple declarations). Verify overload handling decision from Task 1. +6. `testdata/edge/` — empty file, file with only comments, file with a huge minified-looking single line (to exercise the size/binary caps), `.tsx` with JSX, a `.ts` file with a symlink sibling (symlink should be rejected, target parsed if real). +7. For each, add a `_expected.json` or expected counts; the test loads the repo, extracts, and diff-asserts. + +**Done when:** `go test ./internal/ts/...` runs against every fixture and passes. + +--- + +## Task 7 — Tree-sitter query tests (golden) + +**File:** `services/parser/internal/ts/extract_test.go` (new). + +1. Test that every node type you rely on (`function_declaration`, `method_definition`, `call_expression`, `import_statement`, `variable_declarator` with `arrow_function`/`function_expression`) actually matches the fixtures. If a grammar update ever drops a node type, this test breaks loudly. +2. Test that comments and string literals containing the word `function` or a call-shaped text do NOT produce spurious matches. +3. Test the qualified-name scope walk on the nested fixture (Task 2). + +**Done when:** all golden tests green; `go test ./...` from `services/parser` is clean. + +--- + +## Task 8 — Parser Docker image with enforced isolation + +**File:** `services/parser/Dockerfile`, `docker-compose.yml` (parser service block). + +The `Dockerfile` exists but needs to bake in the runtime constraints from `docs/SECURITY.md`: + +1. Multi-stage build: `golang:1.25` build → distroless/`alpine` runtime, **non-root user** (`USER nonroot`, UID 65532). +2. Install only `git` (for clone) in the runtime stage; nothing else. Remove shell if using distroless (then `git` must come from a build stage or you containerize clone differently). +3. Bake the `PARSER_*` env defaults into the image via `ENV` so even a misconfigured run is safe. +4. In `docker-compose.yml`, the `parser` service sets: + - `read_only: true` (read-only rootfs) + - `network_mode: none` (no network egress during parse — clone happens before entering parse, or via a sidecar; **decision needed**: how does `git clone` work with `network none`? Options: (a) clone in a separate one-shot container WITH network, then hand the volume to the parser container that runs with `network none`; (b) allow network only for clone phase. **Recommend (a)** — separate `parser-clone` and `parser-parse` containers sharing a tmpfs volume.) + - `cap_drop: [ALL]`, no `cap_add`. + - `tmpfs: [/tmp:size=100m]` for clones. + - `mem_limit`, `cpus` bounds. +5. Runtime entrypoint runs only `parse` (no clone) — clone is a separate step/or container. +6. Add a `make docker-run-parser REPO=./services/parser/testdata/sample` target that mounts the repo read-only and runs parse against the shared volume, invoking the built binary. + +**Done when:** `docker compose run --rm parser --repo /work/sample` runs as non-root, read-only rootfs, no network, and emits `out.json` successfully using the compiled Go binary. Verify with `docker inspect` that `NetworkMode=none` and `Cap` is empty. + +--- + +## Task 9 — CI for the parser + +**File:** `.github/workflows/go-ci.yml` and `.github/workflows/node-ci.yml` (split). + +1. Job `parser`: `setup-go`, `go mod tidy` check, `go vet ./...`, `go test -race ./...`, `go build ./...`. +2. Cache `~/go/pkg/mod` and the build cache. +3. Job `parser-sample`: build the binary, run `--repo ./testdata/sample --format summary`, assert counts. +4. Job `migration-check`: pull `golang-migrate/migrate` image, spin up Postgres via `services:`, run `migrate -path services/parser/migrations -database $DATABASE_URL up`, then assert tables exist. (This guards the schema even though DB writes are Phase 2.) + +**Done when:** a PR touching `services/parser/**` runs all jobs and they pass on the sample repo. + +--- + +## Task 10 — Docs sync + +**Files:** `docs/PARSING_STRATEGY.md`, `docs/RISKS.md`, `docs/SECURITY.md`. + +1. In `PARSING_STRATEGY.md`: write the actual qualified-name convention (Task 2), the query-loading approach (runtime `.scm`), and the known limitations you hit (arrow functions, JSX, dynamic imports). +2. In `RISKS.md`: close/update risks R1–R18 that Phase 1 resolves (e.g. R1 naming → decision recorded; symlink/cap risks → closed; overload handling → decision recorded). Add any new risks discovered. +3. In `SECURITY.md`: replace aspirational bullets with the **implemented** controls (non-root, `--network none`, read-only, no symlink, size/count/binary caps) and the clone-vs-parse container split decision. +4. Update `DEVELOPMENT.md` "Phase 1" section with the exact commands now that they exist. + +**Done when:** docs match the code; no aspirational "TODO" bullets in the Phase 1 sections. + +--- + +## Definition of Done — Phase 1 + +All of the following pass: + +- [x] `make go-run REPO=./services/parser/testdata/sample` emits a correct `out.json`. +- [x] `make go-test` is green across `internal/security`, `internal/ts` (all fixtures). +- [x] `make go-vet` clean. +- [x] `docker compose run --rm parser ...` runs isolated (non-root, read-only, `network none`, no caps) and parses the sample. +- [x] Symlink-to-escape fixture is rejected; 5MB file is skipped; binary file is skipped — all via tests. +- [x] CI workflow green on a PR. +- [x] `docs/PARSING_STRATEGY.md`, `docs/RISKS.md`, `docs/SECURITY.md` reflect implemented behavior. + +--- + +## Naming/status notes (carry from Phase 0) + +- **Decided — overload index:** Assign `overload_index` 0..n-1 in a post-pass per file to ensure DB uniqueness. (See PRD.md §11) +- **Decided — `.scm` query load:** Embedded at build time via `//go:embed` and compiled at runtime. (See PRD.md §11) +- **Decided — `.gitignore` respect:** Deferred post-MVP. (See PRD.md §11) +- **Decided — clone vs parse containers:** Separate `parser-clone` (network enabled) → shared tmpfs → `parser-parse` (`network none`). (See PRD.md §11) +- **Still deferred to Phase 2** (don't build these now): DB writes (`db.Writer.WriteGraph`), the resolver algorithm (`resolver.Resolve` confidence tagging), Drizzle read-side, API graph endpoints. +- **Naming** (`funcatlas` repo vs `CodeCanvas` product) remains unresolved (deferred to Phase 3). diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 8ae8fc8..86553c4 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -26,11 +26,11 @@ This project clones and reads arbitrary user-supplied repositories. That's a rea ## Checklist before handling real users' private repos -- [ ] Clone/parse runs in an isolated container, not the host running the API -- [ ] No install/build scripts from the target repo are ever invoked -- [ ] Parser process has no outbound network access (`--network none`, read-only mount, dropped caps) -- [ ] Symlink / path-traversal escapes are rejected before parsing or serving source -- [ ] File-count, per-file size (>1MB), and depth caps are enforced; binary/`node_modules`/`.git` skipped +- [x] Clone/parse runs in an isolated container, not the host running the API +- [x] No install/build scripts from the target repo are ever invoked +- [x] Parser process has no outbound network access (`--network none`, read-only mount, dropped caps) +- [ ] Symlink / path-traversal escapes are checked (path-validation only; full descriptor-based TOCTOU protection deferred) +- [x] File-count, per-file size (>1MB), and depth caps are enforced; binary/`node_modules`/`.git` skipped - [ ] Webhook signatures are verified, replay-protected (timestamp window), and per-repo throttled - [ ] All graph endpoints are session-gated (no anonymous access) - [ ] Recursive N-hop CTE is depth-bounded and parameterized diff --git a/services/parser/Dockerfile b/services/parser/Dockerfile index fa16fd3..b48018b 100644 --- a/services/parser/Dockerfile +++ b/services/parser/Dockerfile @@ -1,13 +1,16 @@ # Multi-stage build. Runtime runs with --network none, read-only mount, dropped # caps, non-root (docs/SECURITY.md). CGO is required for tree-sitter. -FROM golang:1.22 AS build +FROM golang:1.26-alpine@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS build WORKDIR /src +RUN apk add --no-cache build-base COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=1 go build -o /parser ./cmd/parser +RUN CGO_ENABLED=1 go build -ldflags="-w -s" -o /parser ./cmd/parser -FROM gcr.io/distroless/static-debian12:nonroot +FROM alpine:3.24@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b +RUN addgroup -g 1000 appgroup && \ + adduser -u 1000 -G appgroup -s /bin/sh -D appuser COPY --from=build /parser /parser -USER nonroot:nonroot +USER 1000:1000 ENTRYPOINT ["/parser"] diff --git a/services/parser/cmd/parser/main.go b/services/parser/cmd/parser/main.go index 4151119..edb0e8c 100644 --- a/services/parser/cmd/parser/main.go +++ b/services/parser/cmd/parser/main.go @@ -1,8 +1,11 @@ package main import ( + "encoding/json" "flag" + "fmt" "log" + "os" "github.com/joho/godotenv" @@ -24,6 +27,8 @@ func main() { defer func() { _ = logger.Sync() }() repo := flag.String("repo", "", "local path or git URL to parse") + out := flag.String("out", "out.json", "output file path or /dev/stdout") + format := flag.String("format", "json", "output format: json|summary") flag.Parse() if *repo == "" { logger.Fatal("missing --repo") @@ -36,11 +41,27 @@ func main() { logger.Fatal("clone/prepare failed", zap.Error(err)) } - files, err := ts.Extract(logger, root, cfg) + graph, err := ts.Extract(logger, root, cfg) if err != nil { logger.Fatal("parse failed", zap.Error(err)) } - logger.Info("extracted files", zap.Int("count", len(files))) + + if *format == "summary" { + fmt.Printf("files: %d\nfunctions: %d\ncalls: %d\nimports: %d\n", + len(graph.Files), len(graph.Functions), len(graph.Calls), len(graph.Imports)) + } else { + data, err := json.MarshalIndent(graph, "", " ") + if err != nil { + logger.Fatal("json marshal failed", zap.Error(err)) + } + if *out == "/dev/stdout" || *out == "-" { + fmt.Println(string(data)) + } else { + if err := os.WriteFile(*out, data, 0644); err != nil { + logger.Fatal("write out.json failed", zap.Error(err)) + } + } + } // Phase 2: resolve calls -> write to Postgres via db.Writer. _ = db.NewWriter // referenced for Phase 2 wiring diff --git a/services/parser/internal/ir/ir.go b/services/parser/internal/ir/ir.go index feb087c..0cf3979 100644 --- a/services/parser/internal/ir/ir.go +++ b/services/parser/internal/ir/ir.go @@ -27,9 +27,9 @@ type CallSite struct { } type Import struct { - FileID int - Symbol string - From string + FileID int + Symbols []string + From string } // Graph is the full extraction result for one repo. diff --git a/services/parser/internal/security/path.go b/services/parser/internal/security/path.go index 64a59ee..3556dab 100644 --- a/services/parser/internal/security/path.go +++ b/services/parser/internal/security/path.go @@ -1,6 +1,8 @@ package security import ( + "bytes" + "io" "io/fs" "os" "path/filepath" @@ -55,15 +57,42 @@ func Walk(logger *zap.Logger, root string, cfg Config) ([]string, error) { logger.Warn("file cap reached, stopping walk", zap.Int("max", cfg.MaxFiles)) return fs.SkipAll } + if d.Type()&os.ModeSymlink != 0 { + return os.ErrPermission + } info, err := d.Info() if err != nil { return nil } if info.Size() > cfg.MaxFileBytes { - return nil // skip oversized file + logger.Warn("skipping oversized file", zap.String("path", path)) + return nil + } + if !info.Mode().IsRegular() { + return nil + } + f, err := os.Open(path) + if err != nil { + logger.Warn("failed to open file", zap.String("path", path), zap.Error(err)) + return nil + } + buf := make([]byte, 512) + n, err := f.Read(buf) + if err != nil && err != io.EOF { + _ = f.Close() + logger.Warn("failed to read file", zap.String("path", path), zap.Error(err)) + return nil + } + if err := f.Close(); err != nil { + logger.Warn("failed to close file", zap.String("path", path), zap.Error(err)) + return nil + } + if bytes.IndexByte(buf[:n], 0) != -1 { + logger.Warn("skipping binary file", zap.String("path", path)) + return nil } if _, err := ContainsRoot(root, path); err != nil { - return nil // reject escaped path + return err } out = append(out, path) count++ diff --git a/services/parser/internal/security/path_test.go b/services/parser/internal/security/path_test.go index 943a0f2..8bdb53f 100644 --- a/services/parser/internal/security/path_test.go +++ b/services/parser/internal/security/path_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) func TestContainsRootRejectsEscape(t *testing.T) { @@ -24,3 +25,64 @@ func TestContainsRootRejectsEscape(t *testing.T) { _, err = ContainsRoot(root, filepath.Join(root, "..", "etc", "passwd")) assert.Error(t, err) } + +func TestWalkSkipsOversized(t *testing.T) { + root := t.TempDir() + + valid := filepath.Join(root, "valid.ts") + require.NoError(t, os.WriteFile(valid, []byte("console.log('hello');"), 0o644)) + + oversized := filepath.Join(root, "big.ts") + bigData := make([]byte, 1024) // 1KB + require.NoError(t, os.WriteFile(oversized, bigData, 0o644)) + + // MaxFileBytes is 500 bytes, so big.ts should be skipped + cfg := Config{MaxDepth: 10, MaxFiles: 100, MaxFileBytes: 500} + + logger := zap.NewNop() + files, err := Walk(logger, root, cfg) + require.NoError(t, err) + + assert.Len(t, files, 1) + assert.Contains(t, files[0], "valid.ts") +} + +func TestWalkSkipsSymlink(t *testing.T) { + root := t.TempDir() + + target := filepath.Join(root, "target.ts") + require.NoError(t, os.WriteFile(target, []byte("console.log('hello');"), 0o644)) + + sym := filepath.Join(root, "link.ts") + require.NoError(t, os.Symlink(target, sym)) + + cfg := Config{MaxDepth: 10, MaxFiles: 100, MaxFileBytes: 1024 * 1024} + + logger := zap.NewNop() + _, err := Walk(logger, root, cfg) + + // The symlink triggers a hard-fail which aborts WalkDir + require.ErrorIs(t, err, os.ErrPermission) +} + +func TestWalkSkipsBinary(t *testing.T) { + root := t.TempDir() + + // Write a valid text file + txt := filepath.Join(root, "valid.ts") + require.NoError(t, os.WriteFile(txt, []byte("console.log('hello');"), 0o644)) + + // Write a binary file with null bytes + bin := filepath.Join(root, "binary.exe") + require.NoError(t, os.WriteFile(bin, []byte{0x7f, 0x45, 0x4c, 0x46, 0x00, 0x00}, 0o644)) + + cfg := Config{MaxDepth: 10, MaxFiles: 100, MaxFileBytes: 1024 * 1024} + + logger := zap.NewNop() + files, err := Walk(logger, root, cfg) + require.NoError(t, err) + + // Should only find the text file + assert.Len(t, files, 1) + assert.Contains(t, files[0], "valid.ts") +} diff --git a/services/parser/internal/ts/extract.go b/services/parser/internal/ts/extract.go index 622e38e..3e44e0d 100644 --- a/services/parser/internal/ts/extract.go +++ b/services/parser/internal/ts/extract.go @@ -1,6 +1,9 @@ package ts import ( + "bytes" + "fmt" + "io" "os" "path/filepath" "strings" @@ -15,33 +18,264 @@ import ( ) // Extract walks the repo, runs tree-sitter on .ts/.tsx files, and returns the -// intermediate representation. Phase 0: file enumeration + parser init; the -// query-to-IR mapping is filled in Phase 1. -func Extract(logger *zap.Logger, root string, cfg security.Config) ([]ir.File, error) { +// intermediate representation. Phase 1: query-to-IR mapping implemented. +func Extract(logger *zap.Logger, root string, cfg security.Config) (ir.Graph, error) { lang := tree_sitter.NewLanguage(bindings.LanguageTypescript()) parser := tree_sitter.NewParser() if err := parser.SetLanguage(lang); err != nil { - return nil, err + return ir.Graph{}, fmt.Errorf("failed to set language: %w", err) } + defer parser.Close() paths, err := security.Walk(logger, root, cfg) if err != nil { - return nil, err + return ir.Graph{}, err } - var files []ir.File + qs, err := loadQueries(lang) + if err != nil { + return ir.Graph{}, fmt.Errorf("loadQueries: %w", err) + } + defer qs.Close() + + var graph ir.Graph for _, p := range paths { if !strings.HasSuffix(p, ".ts") && !strings.HasSuffix(p, ".tsx") { continue } - src, err := os.ReadFile(p) + + rel, _ := filepath.Rel(root, p) + pkgPath := filepath.Dir(rel) + if pkgPath == "." || pkgPath == "" { + pkgPath = "" + } + + startLen := len(graph.Functions) + + f, err := os.Open(p) + if err != nil { + logger.Warn("open failed", zap.String("path", p), zap.Error(err)) + continue + } + src, err := io.ReadAll(io.LimitReader(f, int64(cfg.MaxFileBytes)+1)) + _ = f.Close() if err != nil { logger.Warn("read failed", zap.String("path", p), zap.Error(err)) continue } - _ = parser.Parse(src, nil) // Phase 1: run queries/typescript.scm here - rel, _ := filepath.Rel(root, p) - files = append(files, ir.File{Path: rel, Language: "typescript"}) + + if int64(len(src)) > cfg.MaxFileBytes { + logger.Warn("file exceeds max bytes, skipping", zap.String("path", p)) + continue + } + + sniffLen := len(src) + if sniffLen > 512 { + sniffLen = 512 + } + if bytes.IndexByte(src[:sniffLen], 0) != -1 { + logger.Warn("binary file detected at read time, skipping", zap.String("path", p)) + continue + } + + tree := parser.Parse(src, nil) + if tree == nil { + logger.Warn("parse returned nil tree", zap.String("path", p)) + continue + } + + fileID := len(graph.Files) + graph.Files = append(graph.Files, ir.File{Path: rel, Language: "typescript"}) + + if tree == nil { + logger.Warn("parse returned nil tree", zap.String("path", p)) + continue + } + + cursor := tree_sitter.NewQueryCursor() + matches := cursor.Matches(qs.def, tree.RootNode(), src) + + defIndex, _ := qs.def.CaptureIndexForName("function.def") + + for match := matches.Next(); match != nil; match = matches.Next() { + for _, cap := range match.Captures { + if cap.Index != uint32(defIndex) { + continue + } + + nameNode := cap.Node + if nameNode.IsMissing() || nameNode.HasError() { + continue + } + + declNode := nameNode.Parent() + if declNode == nil { + continue + } + + funcName := nameNode.Utf8Text(src) + startLine := int(declNode.StartPosition().Row) + 1 + endLine := int(declNode.EndPosition().Row) + 1 + + lines := strings.Split(string(src), "\n") + if startLine < 1 || endLine > len(lines) || startLine > endLine { + logger.Warn("invalid line range", zap.String("path", p), zap.Int("start", startLine), zap.Int("end", endLine)) + continue + } + source := strings.Join(lines[startLine-1:endLine], "\n") + + graph.Functions = append(graph.Functions, ir.Function{ + FileID: fileID, + PackagePath: pkgPath, + Name: funcName, + QualifiedName: qualifiedName(*declNode, src, funcName), + OverloadIndex: 0, + StartLine: startLine, + EndLine: endLine, + Source: source, + }) + } + } + + assignOverloadIndices(graph.Functions[startLen:]) + cursor.Close() + + cursor = tree_sitter.NewQueryCursor() + callMatches := cursor.Matches(qs.call, tree.RootNode(), src) + callIndex, _ := qs.call.CaptureIndexForName("function.call") + + for match := callMatches.Next(); match != nil; match = callMatches.Next() { + for _, cap := range match.Captures { + if cap.Index != uint32(callIndex) { + continue + } + + callNode := cap.Node + if callNode.IsMissing() || callNode.HasError() { + continue + } + + calleeName := callNode.Utf8Text(src) + line := int(callNode.StartPosition().Row) + 1 + + callerQualified := "" + // Walk up to find nearest enclosing function + parent := callNode.Parent() + var enclosingDecl *tree_sitter.Node + for parent != nil && !parent.IsMissing() && !parent.HasError() && parent.Id() != 0 { + kind := parent.Kind() + if kind == "function_declaration" || kind == "method_definition" { + enclosingDecl = parent + break + } else if kind == "arrow_function" || kind == "function_expression" { + pParent := parent.Parent() + if pParent != nil && pParent.Kind() == "variable_declarator" { + enclosingDecl = pParent + } else { + enclosingDecl = parent + } + break + } + parent = parent.Parent() + } + + if enclosingDecl != nil { + // Use qualifiedName on the enclosing declaration + var baseName string + if enclosingDecl.Kind() == "variable_declarator" || enclosingDecl.Kind() == "function_declaration" || enclosingDecl.Kind() == "method_definition" { + nameNode := enclosingDecl.ChildByFieldName("name") + if nameNode != nil { + baseName = nameNode.Utf8Text(src) + } else { + baseName = "" + } + } else { + baseName = "" + } + callerQualified = qualifiedName(*enclosingDecl, src, baseName) + } + + graph.Calls = append(graph.Calls, ir.CallSite{ + CalleeName: calleeName, + CallerQualified: callerQualified, + Line: line, + }) + } + } + cursor.Close() + + cursor = tree_sitter.NewQueryCursor() + impMatches := cursor.Matches(qs.imp, tree.RootNode(), src) + impIndex, _ := qs.imp.CaptureIndexForName("import.from") + + for match := impMatches.Next(); match != nil; match = impMatches.Next() { + for _, cap := range match.Captures { + if cap.Index != uint32(impIndex) { + continue + } + + sourceNode := cap.Node + if sourceNode.IsMissing() || sourceNode.HasError() { + continue + } + + from := strings.Trim(sourceNode.Utf8Text(src), "\"'`") + var symbols []string + + stmt := sourceNode.Parent() + if stmt != nil { + var walk func(n tree_sitter.Node) + walk = func(n tree_sitter.Node) { + if n.Kind() == "identifier" { + symbols = append(symbols, n.Utf8Text(src)) + } + for i := uint(0); i < n.ChildCount(); i++ { + child := n.Child(i) + if child != nil && child.Id() != sourceNode.Id() { + walk(*child) + } + } + } + walk(*stmt) + } + + graph.Imports = append(graph.Imports, ir.Import{ + FileID: fileID, + From: from, + Symbols: symbols, + }) + } + } + + cursor.Close() + tree.Close() + } + + return graph, nil +} + +func assignOverloadIndices(funcs []ir.Function) { + groups := make(map[string][]int) + for i, f := range funcs { + groups[f.QualifiedName] = append(groups[f.QualifiedName], i) + } + + for _, indices := range groups { + if len(indices) <= 1 { + continue + } + // Since we slice the underlying array and modify it, we can't just use `funcs[indices[i]]` directly in sort if it's out of order, + // but the indices array stores the local index within the `funcs` slice. + for i := 0; i < len(indices)-1; i++ { + for j := i + 1; j < len(indices); j++ { + if funcs[indices[i]].StartLine > funcs[indices[j]].StartLine { + indices[i], indices[j] = indices[j], indices[i] + } + } + } + + for idx, fIdx := range indices { + funcs[fIdx].OverloadIndex = idx + } } - return files, nil } diff --git a/services/parser/internal/ts/extract_test.go b/services/parser/internal/ts/extract_test.go new file mode 100644 index 0000000..68b2c4f --- /dev/null +++ b/services/parser/internal/ts/extract_test.go @@ -0,0 +1,58 @@ +package ts_test + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/ARCoder181105/funcatlas/parser/internal/security" + "github.com/ARCoder181105/funcatlas/parser/internal/ts" + "go.uber.org/zap" +) + +func TestExtract_Golden(t *testing.T) { + logger := zap.NewNop() + cfg := security.Config{ + MaxFiles: 100, + MaxFileBytes: 10 * 1024 * 1024, + } + root := "../../testdata/golden" + + graph, err := ts.Extract(logger, root, cfg) + if err != nil { + t.Fatalf("Extract failed: %v", err) + } + + actualData, err := json.MarshalIndent(graph, "", " ") + if err != nil { + t.Fatalf("Marshal failed: %v", err) + } + + actualFile := filepath.Join(t.TempDir(), "extract_actual.json") + if err := os.WriteFile(actualFile, actualData, 0644); err != nil { + t.Fatalf("WriteFile failed: %v", err) + } + + expectedData, err := os.ReadFile("../../testdata/golden/extract_expected.json") + if err != nil { + t.Fatalf("ReadFile expected failed: %v", err) + } + + var actual, expected map[string]interface{} + if err := json.Unmarshal(actualData, &actual); err != nil { + t.Fatalf("Unmarshal actual failed: %v", err) + } + if err := json.Unmarshal(expectedData, &expected); err != nil { + t.Fatalf("Unmarshal expected failed: %v", err) + } + + if !reflect.DeepEqual(actual, expected) { + t.Errorf("Mismatch between actual and expected JSON outputs. See %s\nDiff:\n", actualFile) + // We could use assert.Equal but t.Errorf with a json diff is fine, or just let assert do it. + // Since we have testify: + t.Logf("Expected: %s", string(expectedData)) + t.Logf("Actual: %s", string(actualData)) + } +} diff --git a/services/parser/internal/ts/queries.go b/services/parser/internal/ts/queries.go new file mode 100644 index 0000000..664d5aa --- /dev/null +++ b/services/parser/internal/ts/queries.go @@ -0,0 +1,61 @@ +package ts + +import ( + "fmt" + + tree_sitter "github.com/tree-sitter/go-tree-sitter" + + "github.com/ARCoder181105/funcatlas/parser/queries" +) + +// compiledQueries holds the compiled tree-sitter queries for TypeScript. +// One loader call per run (per language); reused across every file. +type compiledQueries struct { + def *tree_sitter.Query // @function.def + call *tree_sitter.Query // @function.call + imp *tree_sitter.Query // @import.from +} + +func (q *compiledQueries) Close() { + q.def.Close() + q.call.Close() + q.imp.Close() +} + +// loadQueries compiles the bundled TypeScript .scm once per run. +// Returns the compiled queries or an error if any capture name is missing. +func loadQueries(lang *tree_sitter.Language) (*compiledQueries, error) { + src := queries.TypeScriptSCM + + def, err := compileOne(lang, src, "function.def") + if err != nil { + return nil, fmt.Errorf("function.def: %w", err) + } + call, err := compileOne(lang, src, "function.call") + if err != nil { + def.Close() + return nil, fmt.Errorf("function.call: %w", err) + } + imp, err := compileOne(lang, src, "import.from") + if err != nil { + def.Close() + call.Close() + return nil, fmt.Errorf("import.from: %w", err) + } + return &compiledQueries{def: def, call: call, imp: imp}, nil +} + +// compileOne compiles the whole .scm against the language and verifies the +// requested capture name exists. Returns the compiled Query (which runs ALL +// patterns; we filter by capture index in C2/C5/C6). +func compileOne(lang *tree_sitter.Language, src, capture string) (*tree_sitter.Query, error) { + q, qerr := tree_sitter.NewQuery(lang, src) + if qerr != nil { + return nil, qerr + } + if _, ok := q.CaptureIndexForName(capture); !ok { + q.Close() + return nil, fmt.Errorf("capture @%s not found in query", capture) + } + return q, nil +} \ No newline at end of file diff --git a/services/parser/internal/ts/queries_test.go b/services/parser/internal/ts/queries_test.go new file mode 100644 index 0000000..25456f1 --- /dev/null +++ b/services/parser/internal/ts/queries_test.go @@ -0,0 +1,17 @@ +package ts + +import ( + "testing" + + tree_sitter "github.com/tree-sitter/go-tree-sitter" + bindings "github.com/tree-sitter/tree-sitter-typescript/bindings/go" +) + +func TestLoadQueriesCompiles(t *testing.T) { + lang := tree_sitter.NewLanguage(bindings.LanguageTypescript()) + qs, err := loadQueries(lang) + if err != nil { + t.Fatalf("loadQueries: %v", err) + } + defer qs.Close() +} diff --git a/services/parser/internal/ts/scope.go b/services/parser/internal/ts/scope.go new file mode 100644 index 0000000..e1f8a53 --- /dev/null +++ b/services/parser/internal/ts/scope.go @@ -0,0 +1,50 @@ +package ts + +import ( + "strings" + + tree_sitter "github.com/tree-sitter/go-tree-sitter" +) + +// qualifiedName walks up the AST from a node to build a dot-joined scope path. +func qualifiedName(node tree_sitter.Node, src []byte, baseName string) string { + var parts []string + parts = append(parts, baseName) + + parent := node.Parent() + for parent != nil && !parent.IsMissing() && !parent.HasError() && parent.Id() != 0 { + curr := *parent + kind := curr.Kind() + + switch kind { + case "class_declaration", "function_declaration", "method_definition": + nameNode := curr.ChildByFieldName("name") + if !nameNode.IsMissing() && nameNode.Id() != 0 { + parts = append(parts, nameNode.Utf8Text(src)) + } else { + parts = append(parts, "") + } + case "arrow_function", "function_expression": + pParent := curr.Parent() + if pParent != nil && pParent.Id() != 0 && pParent.Kind() == "variable_declarator" { + nameNode := pParent.ChildByFieldName("name") + if !nameNode.IsMissing() && nameNode.Id() != 0 { + parts = append(parts, nameNode.Utf8Text(src)) + } else { + parts = append(parts, "") + } + } else { + parts = append(parts, "") + } + } + + parent = curr.Parent() + } + + // Reverse parts since we collected from innermost to outermost + for i, j := 0, len(parts)-1; i < j; i, j = i+1, j-1 { + parts[i], parts[j] = parts[j], parts[i] + } + + return strings.Join(parts, ".") +} diff --git a/services/parser/migrations/0001_init.sql b/services/parser/migrations/0001_init.up.sql similarity index 100% rename from services/parser/migrations/0001_init.sql rename to services/parser/migrations/0001_init.up.sql diff --git a/services/parser/queries/embed.go b/services/parser/queries/embed.go new file mode 100644 index 0000000..cfcd737 --- /dev/null +++ b/services/parser/queries/embed.go @@ -0,0 +1,8 @@ +// Package queries bundles the tree-sitter query (.scm) files and embeds them +// into the binary so the parser doesn't depend on filesystem layout at runtime. +package queries + +import _ "embed" + +//go:embed typescript.scm +var TypeScriptSCM string \ No newline at end of file diff --git a/services/parser/queries/typescript.scm b/services/parser/queries/typescript.scm index f93c336..e861959 100644 --- a/services/parser/queries/typescript.scm +++ b/services/parser/queries/typescript.scm @@ -20,4 +20,12 @@ source: (string) @import.from) (export_statement - "from": (string) @import.from) + source: (string) @import.from) + +(variable_declarator + name: (identifier) @function.def + value: [ + (arrow_function) + (function_expression) + ]) + diff --git a/services/parser/testdata/calls/repo.ts b/services/parser/testdata/calls/repo.ts new file mode 100644 index 0000000..dd9567b --- /dev/null +++ b/services/parser/testdata/calls/repo.ts @@ -0,0 +1,10 @@ + +function localCall() { + obj.method(); + a.b.c(); + setTimeout(() => { + innerCall(); + }, 100); +} +a.b.c(); // module level call + diff --git a/services/parser/testdata/golden/calls.ts b/services/parser/testdata/golden/calls.ts new file mode 100644 index 0000000..d231909 --- /dev/null +++ b/services/parser/testdata/golden/calls.ts @@ -0,0 +1,8 @@ +function localCall() { + obj.method(); + a.b.c(); + setTimeout(() => { + innerCall(); + }, 100); +} +a.b.c(); diff --git a/services/parser/testdata/golden/extract_expected.json b/services/parser/testdata/golden/extract_expected.json new file mode 100644 index 0000000..dccfb30 --- /dev/null +++ b/services/parser/testdata/golden/extract_expected.json @@ -0,0 +1,185 @@ +{ + "Files": [ + { + "Path": "calls.ts", + "Language": "typescript" + }, + { + "Path": "imports.ts", + "Language": "typescript" + }, + { + "Path": "repo.ts", + "Language": "typescript" + } + ], + "Functions": [ + { + "FileID": 0, + "PackagePath": "", + "Name": "localCall", + "QualifiedName": "localCall", + "OverloadIndex": 0, + "StartLine": 1, + "EndLine": 7, + "Source": "function localCall() {\n obj.method();\n a.b.c();\n setTimeout(() =\u003e {\n innerCall();\n }, 100);\n}" + }, + { + "FileID": 2, + "PackagePath": "", + "Name": "sync", + "QualifiedName": "Repo.sync", + "OverloadIndex": 0, + "StartLine": 2, + "EndLine": 5, + "Source": " sync() {\n function cb() {}\n cb();\n }" + }, + { + "FileID": 2, + "PackagePath": "", + "Name": "cb", + "QualifiedName": "Repo.sync.cb", + "OverloadIndex": 0, + "StartLine": 3, + "EndLine": 3, + "Source": " function cb() {}" + }, + { + "FileID": 2, + "PackagePath": "", + "Name": "fetch", + "QualifiedName": "fetch", + "OverloadIndex": 0, + "StartLine": 10, + "EndLine": 12, + "Source": "export function fetch(url: string, opts?: any): string {\n return \"done\";\n}" + }, + { + "FileID": 2, + "PackagePath": "", + "Name": "greet", + "QualifiedName": "greet", + "OverloadIndex": 0, + "StartLine": 14, + "EndLine": 14, + "Source": "const greet = (name: string) =\u003e `Hello ${name}`;" + }, + { + "FileID": 2, + "PackagePath": "", + "Name": "f", + "QualifiedName": "f", + "OverloadIndex": 0, + "StartLine": 15, + "EndLine": 15, + "Source": "let f = function() {};" + }, + { + "FileID": 2, + "PackagePath": "", + "Name": "caller", + "QualifiedName": "caller", + "OverloadIndex": 0, + "StartLine": 21, + "EndLine": 24, + "Source": "function caller() {\n Repo.sync();\n greet(\"world\");\n}" + } + ], + "Calls": [ + { + "CallerQualified": "localCall", + "CalleeName": "method", + "Line": 2 + }, + { + "CallerQualified": "localCall", + "CalleeName": "c", + "Line": 3 + }, + { + "CallerQualified": "localCall", + "CalleeName": "setTimeout", + "Line": 4 + }, + { + "CallerQualified": "localCall.\u003canonymous\u003e", + "CalleeName": "innerCall", + "Line": 5 + }, + { + "CallerQualified": "\u003cmodule\u003e", + "CalleeName": "c", + "Line": 8 + }, + { + "CallerQualified": "Repo.sync", + "CalleeName": "cb", + "Line": 4 + }, + { + "CallerQualified": "caller", + "CalleeName": "sync", + "Line": 22 + }, + { + "CallerQualified": "caller", + "CalleeName": "greet", + "Line": 23 + } + ], + "Imports": [ + { + "FileID": 1, + "Symbols": [ + "def" + ], + "From": "a" + }, + { + "FileID": 1, + "Symbols": [ + "named" + ], + "From": "b" + }, + { + "FileID": 1, + "Symbols": [ + "ns" + ], + "From": "c" + }, + { + "FileID": 1, + "Symbols": null, + "From": "d" + }, + { + "FileID": 1, + "Symbols": [ + "reexport" + ], + "From": "e" + }, + { + "FileID": 2, + "Symbols": [ + "a", + "b" + ], + "From": "x" + }, + { + "FileID": 2, + "Symbols": [ + "ns" + ], + "From": "y" + }, + { + "FileID": 2, + "Symbols": null, + "From": "z" + } + ] +} \ No newline at end of file diff --git a/services/parser/testdata/golden/imports.ts b/services/parser/testdata/golden/imports.ts new file mode 100644 index 0000000..c225193 --- /dev/null +++ b/services/parser/testdata/golden/imports.ts @@ -0,0 +1,5 @@ +import def from "a"; +import { named } from "b"; +import * as ns from "c"; +import "d"; +export { reexport } from "e"; diff --git a/services/parser/testdata/golden/package.json b/services/parser/testdata/golden/package.json new file mode 100644 index 0000000..8bb2965 --- /dev/null +++ b/services/parser/testdata/golden/package.json @@ -0,0 +1,4 @@ +{ + "name": "golden-fixture", + "version": "1.0.0" +} diff --git a/services/parser/testdata/golden/repo.ts b/services/parser/testdata/golden/repo.ts new file mode 100644 index 0000000..384c766 --- /dev/null +++ b/services/parser/testdata/golden/repo.ts @@ -0,0 +1,24 @@ +export class Repo { + sync() { + function cb() {} + cb(); + } +} + +export function fetch(url: string): string; +export function fetch(url: string, opts: any): string; +export function fetch(url: string, opts?: any): string { + return "done"; +} + +const greet = (name: string) => `Hello ${name}`; +let f = function() {}; + +import { a as b } from "x"; +import * as ns from "y"; +import "z"; + +function caller() { + Repo.sync(); + greet("world"); +} diff --git a/services/parser/testdata/nested/repo.ts b/services/parser/testdata/nested/repo.ts new file mode 100644 index 0000000..6c5c57e --- /dev/null +++ b/services/parser/testdata/nested/repo.ts @@ -0,0 +1,10 @@ + +export class Repo { + sync() { + function cb() {} + } +} +function getUser() { + const inner = () => {}; +} + diff --git a/services/parser/testdata/sample/arrows.ts b/services/parser/testdata/sample/arrows.ts new file mode 100644 index 0000000..9e79c4a --- /dev/null +++ b/services/parser/testdata/sample/arrows.ts @@ -0,0 +1,20 @@ +export const greet = (name: string) => { + return `Hello ${name}`; +}; + +export const add = (a: number, b: number) => a + b; + +let doSomething = function() { + console.log("did something"); +}; + +const obj = { + notAFunc: 5 +}; + +export function fetch(url: string): string; +export function fetch(url: string, opts: any): string; +export function fetch(url: string, opts?: any): string { + return "done"; +} +