diff --git a/docs/code-graph-support.md b/docs/code-graph-support.md index a98c8b9..c799ad1 100644 --- a/docs/code-graph-support.md +++ b/docs/code-graph-support.md @@ -26,6 +26,7 @@ extractor registry lives in | **Supported** | JSX | `.jsx` | [`jsx-component.jsx`](../src/graph/__tests__/fixtures/jsx-component.jsx) and [`extraction-regression.test.ts`](../src/graph/__tests__/extraction-regression.test.ts) cover components, imports, calls, and construction. | | **Supported** | Python | `.py` | [`sample.py`](../src/graph/__tests__/fixtures/sample.py), [`extractor-python.test.ts`](../src/graph/__tests__/extractor-python.test.ts), and the [`python-package`](../src/graph/__tests__/fixtures/python-package) integration fixture cover extraction and cross-file package resolution. | | **Supported** | Rust | `.rs` | [`sample.rs`](../src/graph/__tests__/fixtures/sample.rs) and [`extractor-rust.test.ts`](../src/graph/__tests__/extractor-rust.test.ts) cover structs, traits, enums, modules, functions, methods, generics, imports, calls, implementations, construction, returns, and field types. | +| **Supported** | Ruby | `.rb` | [`sample.rb`](../src/graph/__tests__/fixtures/sample.rb) and [`extractor-ruby.test.ts`](../src/graph/__tests__/extractor-ruby.test.ts) cover modules, classes, instance and singleton methods, constants, superclass inheritance, mixins, construction, `require`/`require_relative`, and calls. Parsed with Prism (`@ruby/prism`), not a tree-sitter grammar — see [Vendored Grammars](extractors.md#vendored-grammars). | | **Unsupported** | Go and other languages | All other extensions | These names may be reserved in [`src/graph/types.ts`](../src/graph/types.ts), but no grammar or extractor is registered for them. Unsupported files are skipped rather than failing a graph build. | `src/graph/types.ts` contains a wider future-facing language vocabulary. A name diff --git a/docs/extractors.md b/docs/extractors.md index b9cbfef..ddb2865 100644 --- a/docs/extractors.md +++ b/docs/extractors.md @@ -115,6 +115,14 @@ When adding a language, document the grammar source and version. - **Upstream grammar:** [tree-sitter/tree-sitter-rust](https://github.com/tree-sitter/tree-sitter-rust) - **Upstream grammar license:** MIT +### Ruby +- **Parser source:** `@ruby/prism` npm package, version `^1.9.0` — not tree-sitter. Prism is Ruby core's own parser (the one CRuby 3.3+ uses), shipped as a WASM build maintained by the Ruby core team; it is more semantically accurate for Ruby than the community-maintained `tree-sitter-ruby` grammar. +- **No vendored `.wasm`:** it ships inside the npm package itself (`node_modules/@ruby/prism/src/prism.wasm`), loaded lazily by `src/graph/extraction/prism-runtime.ts` rather than through `grammars.ts`'s tree-sitter `WASM_GRAMMAR_FILES` map. +- **Package license:** MIT +- **Upstream parser:** [ruby/prism](https://github.com/ruby/prism) +- **Upstream parser license:** MIT +- **Deviation note:** `rubyExtractor.extract()` ignores the `tree: TSTree` parameter and re-parses `source` with Prism directly — Prism's typed AST has no tree-sitter node shape to adapt into. `grammars.ts#parse()` still returns a placeholder `TSTree` for Ruby so `extractFile`'s generic null-check behaves the same across languages. + ## Pull request proof Before opening a pull request, run: diff --git a/package-lock.json b/package-lock.json index 3977045..c1d2504 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "packages/*" ], "dependencies": { + "@ruby/prism": "^1.9.0", "chalk": "^5.4.1", "commander": "^13.1.0", "cross-spawn": "^7.0.6", @@ -964,6 +965,12 @@ "win32" ] }, + "node_modules/@ruby/prism": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@ruby/prism/-/prism-1.9.0.tgz", + "integrity": "sha512-/j+JfP1eA0nCjx6c/8L6K/ENErmsqc6RMAba8rKYNz4RadrT7AW20u4WrjpWYWzdlk6k1YWHxoW9OS68w02+Vw==", + "license": "MIT" + }, "node_modules/@simple-git/args-pathspec": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@simple-git/args-pathspec/-/args-pathspec-1.0.3.tgz", diff --git a/package.json b/package.json index 6bf270c..a197475 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "prepare": "npm run build" }, "dependencies": { + "@ruby/prism": "^1.9.0", "chalk": "^5.4.1", "commander": "^13.1.0", "cross-spawn": "^7.0.6", diff --git a/src/graph/__tests__/extractor-ruby.test.ts b/src/graph/__tests__/extractor-ruby.test.ts new file mode 100644 index 0000000..275a92d --- /dev/null +++ b/src/graph/__tests__/extractor-ruby.test.ts @@ -0,0 +1,69 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { beforeAll, describe, expect, it } from "vitest"; +import { extractFile, loadGrammars } from "../extraction/index.js"; +import type { FileExtraction } from "../extraction/index.js"; + +const FIXTURE = join(dirname(fileURLToPath(import.meta.url)), "fixtures", "sample.rb"); + +describe("Ruby extractor", () => { + let result: FileExtraction; + + beforeAll(async () => { + await loadGrammars(["ruby"]); + const source = readFileSync(FIXTURE, "utf-8"); + result = extractFile(FIXTURE, source, "ruby")!; + expect(result).not.toBeNull(); + }); + + const node = (kind: string, name: string) => + result.nodes.find((n) => n.kind === kind && n.name === name); + const hasEdge = (kind: string, targetName: string) => + result.edges.some((e) => e.kind === kind && e.targetName === targetName); + + it("emits a file node and stamps the language", () => { + expect(result.language).toBe("ruby"); + expect(node("file", "sample.rb")).toBeDefined(); + }); + + it("extracts a module and nested classes with containment", () => { + expect(node("module", "Greetings")).toBeDefined(); + const greeter = node("class", "Greeter"); + expect(greeter).toBeDefined(); + expect(greeter!.qualifiedName).toBe("Greetings::Greeter"); + }); + + it("extracts instance and singleton methods, marking singleton defs static", () => { + const greet = node("method", "greet"); + expect(greet).toBeDefined(); + expect(greet!.isStatic).not.toBe(true); + + const defaultMethod = node("method", "default"); + expect(defaultMethod).toBeDefined(); + expect(defaultMethod!.isStatic).toBe(true); + }); + + it("extracts a module-scoped constant", () => { + expect(node("constant", "PREFIX")).toBeDefined(); + }); + + it("emits a require edge for require/require_relative", () => { + expect(hasEdge("imports", "json")).toBe(true); + expect(hasEdge("imports", "helpers")).toBe(true); + }); + + it("emits extends, implements (mixin), instantiates, and calls edges", () => { + expect(hasEdge("extends", "Base")).toBe(true); // Greeter < Base + expect(hasEdge("implements", "Comparable")).toBe(true); // include Comparable + expect(hasEdge("instantiates", "Greeter")).toBe(true); // Greeter.new(...) in .default + expect(hasEdge("calls", "warm_up")).toBe(true); // speak -> warm_up + expect(hasEdge("calls", "format_name")).toBe(true); // greet -> format_name + }); + + it("degrades safely on a construct with no direct graph representation", () => { + // Blocks (do...end / { ... }) passed to calls aren't modeled as their own + // node kind; the surrounding call is still extracted without throwing. + expect(() => extractFile(FIXTURE, "[1, 2].each { |n| puts n }", "ruby")).not.toThrow(); + }); +}); diff --git a/src/graph/__tests__/fixtures/sample.rb b/src/graph/__tests__/fixtures/sample.rb new file mode 100644 index 0000000..6c6fe9d --- /dev/null +++ b/src/graph/__tests__/fixtures/sample.rb @@ -0,0 +1,28 @@ +require "json" +require_relative "helpers" + +module Greetings + PREFIX = "Hello" + + class Base + def speak + warm_up + end + end + + class Greeter < Base + include Comparable + + def initialize(name) + @name = name + end + + def self.default + Greeter.new("World") + end + + def greet + format_name(@name) + end + end +end diff --git a/src/graph/extraction/grammars.ts b/src/graph/extraction/grammars.ts index 816d74c..167a3f8 100644 --- a/src/graph/extraction/grammars.ts +++ b/src/graph/extraction/grammars.ts @@ -13,8 +13,9 @@ import { Parser, Language as WasmLanguage } from "web-tree-sitter"; import type { Language } from "../types.js"; -import type { TSTree } from "./types.js"; +import type { TSNode, TSTree } from "./types.js"; import { grammarWasmPath } from "../assets.js"; +import { isPrismLoaded, loadPrismRuntime, parseRubySource } from "./prism-runtime.js"; /** * Languages 0.7.0 can parse, mapped to the vendored grammar WASM basename. @@ -42,6 +43,34 @@ const EXTENSION_MAP: Record = { ".jsx": "jsx", ".py": "python", ".rs": "rust", + ".rb": "ruby", +}; + +/** + * Placeholder root node for the `TSTree` `parse()` must return for Ruby, whose + * grammar-less path (below) never walks it: `rubyExtractor` re-parses `source` + * with Prism directly, since Prism's AST has no tree-sitter shape to adapt. + * `parse()` still returns a real `TSTree` so `extractFile`'s generic + * `if (!tree) return null` gate behaves the same for every language. + */ +const NULL_TS_NODE: TSNode = { + type: "program", + text: "", + startIndex: 0, + endIndex: 0, + startPosition: { row: 0, column: 0 }, + endPosition: { row: 0, column: 0 }, + parent: null, + childCount: 0, + namedChildCount: 0, + children: [], + namedChildren: [], + previousNamedSibling: null, + nextNamedSibling: null, + child: () => null, + namedChild: () => null, + childForFieldName: () => null, + descendantsOfType: () => [], }; /** Glob pattern for every extension registered above. */ @@ -69,14 +98,19 @@ export async function initRuntime(): Promise { * documented WASM-heap race when grammars load concurrently on Node. */ export async function loadGrammars(languages: Language[]): Promise { - await initRuntime(); - const toLoad = [...new Set(languages)].filter( - (lang) => lang in WASM_GRAMMAR_FILES && !languageCache.has(lang), - ); - for (const lang of toLoad) { - const wasmFile = WASM_GRAMMAR_FILES[lang]!; - const grammar = await WasmLanguage.load(grammarWasmPath(wasmFile)); - languageCache.set(lang, grammar); + const unique = [...new Set(languages)]; + const treeSitterLangs = unique.filter((lang) => lang in WASM_GRAMMAR_FILES); + if (treeSitterLangs.length > 0) { + await initRuntime(); + const toLoad = treeSitterLangs.filter((lang) => !languageCache.has(lang)); + for (const lang of toLoad) { + const wasmFile = WASM_GRAMMAR_FILES[lang]!; + const grammar = await WasmLanguage.load(grammarWasmPath(wasmFile)); + languageCache.set(lang, grammar); + } + } + if (unique.includes("ruby") && !isPrismLoaded()) { + await loadPrismRuntime(); } } @@ -108,7 +142,7 @@ export function detectLanguage(filePath: string): Language { /** Every language 0.7.0 ships a grammar for. */ export function supportedLanguages(): Language[] { - return Object.keys(WASM_GRAMMAR_FILES) as Language[]; + return [...(Object.keys(WASM_GRAMMAR_FILES) as Language[]), "ruby"]; } /** @@ -120,6 +154,11 @@ export function supportedLanguages(): Language[] { * web-tree-sitter directly. */ export function parse(source: string, language: Language): TSTree | null { + if (language === "ruby") { + if (!isPrismLoaded()) return null; + const result = parseRubySource(source); + return result ? { rootNode: NULL_TS_NODE } : null; + } const parser = getParser(language); if (!parser) return null; const tree = parser.parse(source); diff --git a/src/graph/extraction/languages/index.ts b/src/graph/extraction/languages/index.ts index de5e34a..6e17ece 100644 --- a/src/graph/extraction/languages/index.ts +++ b/src/graph/extraction/languages/index.ts @@ -12,6 +12,7 @@ import { typescriptExtractor, tsxExtractor } from "./typescript.js"; import { javascriptExtractor, jsxExtractor } from "./javascript.js"; import { pythonExtractor } from "./python.js"; import { rustExtractor } from "./rust.js"; +import { rubyExtractor } from "./ruby.js"; /** Registered extractors, keyed by the language id they emit. */ export const EXTRACTORS: Partial> = { @@ -21,6 +22,7 @@ export const EXTRACTORS: Partial> = { jsx: jsxExtractor, python: pythonExtractor, rust: rustExtractor, + ruby: rubyExtractor, }; /** The extractor for a language, or undefined if unsupported in this release. */ diff --git a/src/graph/extraction/languages/ruby.ts b/src/graph/extraction/languages/ruby.ts new file mode 100644 index 0000000..4401fac --- /dev/null +++ b/src/graph/extraction/languages/ruby.ts @@ -0,0 +1,275 @@ +import { + CallNode, + ClassNode, + ConstantPathNode, + ConstantReadNode, + ConstantWriteNode, + DefNode, + ModuleNode, + SingletonClassNode, + StringNode, + type Node as PrismNode, +} from "@ruby/prism"; +import type { NodeKind } from "../../types.js"; +import type { ExtractedEdge, ExtractedNode, LanguageExtractor, TSTree } from "../types.js"; +import { generateNodeId } from "../node-id.js"; +import { parseRubySource } from "../prism-runtime.js"; + +// Ruby mixins (`include`/`extend`/`prepend`) have no dedicated EdgeKind; they +// graft a module's methods onto a class much like implementing an interface, +// so they're recorded as `implements` rather than added as a new edge kind. +const MIXIN_CALL_NAMES = new Set(["include", "extend", "prepend"]); +const REQUIRE_CALL_NAMES = new Set(["require", "require_relative"]); + +class LineIndex { + private readonly starts: number[] = [0]; + + constructor(source: string) { + for (let i = 0; i < source.length; i++) { + if (source.charCodeAt(i) === 10) this.starts.push(i + 1); + } + } + + pointAt(offset: number): { row: number; column: number } { + let lo = 0; + let hi = this.starts.length - 1; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if (this.starts[mid] <= offset) lo = mid; + else hi = mid - 1; + } + return { row: lo, column: offset - this.starts[lo] }; + } +} + +class RubyWalker { + private readonly nodes: ExtractedNode[] = []; + private readonly edges: ExtractedEdge[] = []; + private readonly scopeStack: string[] = []; + private readonly lines: LineIndex; + + constructor( + private readonly filePath: string, + private readonly source: string, + ) { + this.lines = new LineIndex(source); + } + + run(programBody: PrismNode[], endOffset: number): { nodes: ExtractedNode[]; edges: ExtractedEdge[] } { + const fileId = `file:${this.filePath}`; + this.nodes.push({ + id: fileId, + kind: "file", + name: baseName(this.filePath), + qualifiedName: this.filePath, + filePath: this.filePath, + language: "ruby", + startLine: 1, + endLine: this.lines.pointAt(endOffset).row + 1, + startColumn: 0, + endColumn: 0, + isExported: false, + }); + + this.scopeStack.push(fileId); + for (const child of programBody) this.visit(child); + this.scopeStack.pop(); + + return { nodes: this.nodes, edges: this.edges }; + } + + private visit(node: PrismNode | null): void { + if (!node) return; + + if (node instanceof ClassNode) return this.extractClass(node); + if (node instanceof ModuleNode) return this.extractModule(node); + if (node instanceof DefNode) return this.extractDef(node); + if (node instanceof ConstantWriteNode && this.atModuleOrClassScope()) { + return this.extractConstant(node); + } + if (node instanceof SingletonClassNode) { + // `class << self` reopens the singleton class; its defs are effectively + // static methods of the enclosing class, so splice its body into the + // current scope rather than emitting a node for the wrapper itself. + for (const child of node.body ? childrenOf(node.body) : []) this.visit(child); + return; + } + if (node instanceof CallNode) return this.extractTopLevelCall(node); + + for (const child of node.compactChildNodes()) this.visit(child); + } + + private createNode(kind: NodeKind, name: string, node: PrismNode, extra?: Partial): string | null { + if (!name) return null; + const id = generateNodeId(this.filePath, kind, name); + const start = this.lines.pointAt(node.location.startOffset); + const end = this.lines.pointAt(node.location.startOffset + node.location.length); + this.nodes.push({ + id, + kind, + name, + qualifiedName: this.qualify(name), + filePath: this.filePath, + language: "ruby", + startLine: start.row + 1, + endLine: end.row + 1, + startColumn: start.column, + endColumn: end.column, + isExported: true, + ...extra, + }); + + const parent = this.scopeStack[this.scopeStack.length - 1]; + if (parent) this.edges.push({ source: parent, target: id, kind: "contains" }); + return id; + } + + private qualify(name: string): string { + const parts: string[] = []; + for (const scopeId of this.scopeStack) { + const scope = this.nodes.find((n) => n.id === scopeId); + if (scope && scope.kind !== "file") parts.push(scope.name); + } + parts.push(name); + return parts.join("::"); + } + + private atModuleOrClassScope(): boolean { + const parentId = this.scopeStack[this.scopeStack.length - 1]; + const parent = parentId ? this.nodes.find((n) => n.id === parentId) : null; + return !!parent && (parent.kind === "file" || parent.kind === "class" || parent.kind === "module"); + } + + private extractClass(node: ClassNode): void { + const id = this.createNode("class", node.name, node); + if (!id) return; + + if (node.superclass) { + const superName = constantPathName(node.superclass); + if (superName) this.addRef(id, superName, "extends", node.superclass); + } + + this.scopeStack.push(id); + for (const child of node.body ? childrenOf(node.body) : []) this.visit(child); + this.scopeStack.pop(); + } + + private extractModule(node: ModuleNode): void { + const id = this.createNode("module", node.name, node); + if (!id) return; + + this.scopeStack.push(id); + for (const child of node.body ? childrenOf(node.body) : []) this.visit(child); + this.scopeStack.pop(); + } + + private extractDef(node: DefNode): void { + const isStatic = node.receiver !== null; + const isMethod = this.atModuleOrClassScope(); + const id = this.createNode(isMethod ? "method" : "function", node.name, node, { + signature: this.signatureOf(node), + isStatic, + }); + if (!id) return; + + this.scopeStack.push(id); + for (const child of node.body ? childrenOf(node.body) : []) this.visit(child); + this.scopeStack.pop(); + } + + private extractConstant(node: ConstantWriteNode): void { + this.createNode("constant", node.name, node, { + signature: this.source.slice(node.value.location.startOffset, node.value.location.startOffset + node.value.location.length).slice(0, 200), + }); + } + + private extractTopLevelCall(node: CallNode): void { + const ownerId = this.scopeStack[this.scopeStack.length - 1]; + if (ownerId) this.extractCall(node, ownerId); + for (const child of node.compactChildNodes()) this.visit(child); + } + + private extractCall(node: CallNode, ownerId: string): void { + if (REQUIRE_CALL_NAMES.has(node.name)) { + const target = firstStringArgument(node); + if (target) this.addRef(`file:${this.filePath}`, target, "imports", node); + return; + } + + if (MIXIN_CALL_NAMES.has(node.name) && node.receiver === null) { + const target = firstConstantArgument(node); + if (target) this.addRef(ownerId, target, "implements", node); + return; + } + + if (node.name === "new" && node.receiver) { + const className = constantPathName(node.receiver); + if (className) { + this.addRef(ownerId, className, "instantiates", node); + return; + } + } + + this.addRef(ownerId, node.name, "calls", node); + } + + private signatureOf(node: DefNode): string { + if (!node.parameters) return "()"; + const { startOffset, length } = node.parameters.location; + return `(${this.source.slice(startOffset, startOffset + length)})`; + } + + private addRef(source: string, targetName: string, kind: ExtractedEdge["kind"], node: PrismNode): void { + if (!targetName) return; + const point = this.lines.pointAt(node.location.startOffset); + this.edges.push({ source, targetName, kind, line: point.row, column: point.column }); + } +} + +function childrenOf(node: PrismNode): PrismNode[] { + // A `StatementsNode` body is the common case; anything else (a bare + // single-statement body) is walked as the one child it is. + return "body" in node && Array.isArray((node as { body: unknown }).body) + ? ((node as unknown as { body: PrismNode[] }).body) + : [node]; +} + +function constantPathName(node: PrismNode): string { + if (node instanceof ConstantReadNode) return node.name; + if (node instanceof ConstantPathNode) { + const prefix = node.parent ? constantPathName(node.parent) : ""; + return prefix ? `${prefix}::${node.name}` : (node.name ?? ""); + } + return ""; +} + +function firstStringArgument(node: CallNode): string { + const args = node.arguments_?.arguments_ ?? []; + const first = args[0]; + return first instanceof StringNode ? first.unescaped.value : ""; +} + +function firstConstantArgument(node: CallNode): string { + const args = node.arguments_?.arguments_ ?? []; + const first = args[0]; + return first ? constantPathName(first) : ""; +} + +function baseName(filePath: string): string { + const normalized = filePath.replace(/\\/g, "/"); + const slash = normalized.lastIndexOf("/"); + return slash < 0 ? normalized : normalized.slice(slash + 1); +} + +export const rubyExtractor: LanguageExtractor = { + language: "ruby", + fileExtensions: [".rb"], + grammarWasm: "prism", + extract(_tree: TSTree, filePath: string, source: string) { + // Ignores `_tree`: Prism's AST isn't tree-sitter-shaped, so this re-parses + // `source` directly rather than adapting `parse()`'s placeholder tree. + const result = parseRubySource(source); + if (!result) return { nodes: [], edges: [] }; + return new RubyWalker(filePath, source).run(result.value.statements.body, result.value.location.startOffset + result.value.location.length); + }, +}; diff --git a/src/graph/extraction/prism-runtime.ts b/src/graph/extraction/prism-runtime.ts new file mode 100644 index 0000000..a0452da --- /dev/null +++ b/src/graph/extraction/prism-runtime.ts @@ -0,0 +1,38 @@ +// ============================================================================ +// mex code-graph — Prism (Ruby) parser runtime +// ============================================================================ +// +// Ruby's own parser, not a tree-sitter grammar: `@ruby/prism` ships Ruby core's +// Prism parser as WASM, which is more semantically accurate for Ruby than the +// community tree-sitter-ruby grammar (it's the same parser CRuby 3.3+ uses). +// It has no relationship to web-tree-sitter's `Parser`/`Language`, so it can't +// live in `grammars.ts`'s tree-sitter loader — this module is Ruby's parallel, +// minimal loader: load the WASM once, cache the returned sync parse function. + +import { loadPrism } from "@ruby/prism"; + +type PrismParseFn = Awaited>; +export type PrismParseResult = ReturnType; + +let parseFn: PrismParseFn | null = null; + +/** Load the Prism WASM module. Idempotent. */ +export async function loadPrismRuntime(): Promise { + if (parseFn) return; + parseFn = await loadPrism(); +} + +/** Whether {@link loadPrismRuntime} has completed. */ +export function isPrismLoaded(): boolean { + return parseFn !== null; +} + +/** Parse Ruby source, or null if the runtime was never loaded. */ +export function parseRubySource(source: string): PrismParseResult | null { + return parseFn ? parseFn(source) : null; +} + +/** Reset the cached parser (tests / teardown, mirrors `disposeParsers`). */ +export function disposePrismRuntime(): void { + parseFn = null; +}