Enable structured documentation collection based on doc-comments

[necto]

Summary

For my tool I would like to extract API documentation from a .jo file. While jo can generate HTML documentation, it has no option to produce documentation in machine-readable format.

I propose adding a new CLI command to jo to parse a given .jo file and produce a JSON file with all declarations documentation.

Motivation

This should help tools built on-top of jo reliably extract documentation and declaration from jo code. Since jo is LLM oriented it might be useful for LLM agents to feed LLM precise documentation of relevant parts of the API rather than feeding it entire HTML documentation or full jo source.

Proposed design

I have no opinion about how the option should be called. Here is an example JSON file it might produce:

[
  {
    "name": "FsReadAPI.FileRead",
    "signature": "interface FileRead",
    "doc": "An open file handle for reading text.\n\nEvery `FileRead` returned by `FsRead.openForRead` must be closed with\n`close()` when reading is complete."
  },
  {
    "name": "FsReadAPI.FileRead.read",
    "signature": "read(bytes: Int = -1): String",
    "doc": "Reads up to `bytes` characters, or all remaining text when omitted."
  },
  {
    "name": "FsReadAPI.FsRead",
    "signature": "interface FsRead",
    "doc": "File system read operations backed by Python's `os` module.\n\nRelative paths resolve from Pi's current project directory. This capability\nonly reads or probes the file system; use `SearchAPI.Search` for\ncommand-backed searches."
  },
  {
    "name": "FsReadAPI.FsRead.openForRead",
    "signature": "openForRead(path: String, create: Bool = false): FileRead",
    "doc": "Opens `path` for text reading.\n\nWhen `create` is true, creates an empty file if `path` does not exist.\nAlways close the returned file, for example:\n```jo\nval file = fs.openForRead(\"notes.txt\")\nval contents = file.read()\nfile.close()\n```"
  },
  {
    "name": "FsReadAPI.FsRead.isFile",
    "signature": "isFile(path: String): Bool",
    "doc": "Returns whether `path` identifies a regular file."
  },
...
]

Alternatives considered

Parse jo code directly. Fragile as the doc format or declaration syntax might change. Also disrespects the separation of responsibilities. I resort to this approach as a work around, see the code below.
Parse HTML generated by jo doc: as fragile as parsing jo code.

/**
 * Extract documented Jo declarations as JSON.
 *
 * Usage:
 *   jo-doc-json.ts INPUT.jo OUTPUT.json
 */

import { readFileSync, writeFileSync } from "node:fs";

interface DocEntry {
  declaration: string;
  documentation: string;
}

interface DocumentedDeclaration {
  name: string;
  signature: string;
  doc: string;
}

function getDocumentationLines(source: string): DocEntry[] {
  const lines = source.split("\n");
  const entries: DocEntry[] = [];
  let index = 0;

  while (index < lines.length) {
    const match = lines[index].match(/^\s*\/\/\[\s?(.*)$/);
    if (!match) {
      index++;
      continue;
    }

    const openingText = match[1];
    let documentation: string[];
    let closingIndex: number;

    const singleLineMatch = openingText.match(/^(.*?)\s*\/\/\]\s*$/);
    if (singleLineMatch) {
      documentation = [singleLineMatch[1]];
      closingIndex = index;
    } else {
      documentation = [openingText];
      closingIndex = index + 1;
      while (closingIndex < lines.length) {
        if (/^\s*\/\/\]\s*$/.test(lines[closingIndex])) {
          break;
        }
        const contentMatch = lines[closingIndex].match(/^\s*!\s?(.*)$/);
        if (!contentMatch) {
          throw new Error(
            `Invalid Jo documentation line: ${lines[closingIndex]}`,
          );
        }
        documentation.push(contentMatch[1]);
        closingIndex++;
      }

      if (closingIndex >= lines.length) {
        throw new Error("Unterminated Jo documentation block");
      }
    }

    let declarationIndex = closingIndex + 1;
    while (declarationIndex < lines.length) {
      const stripped = lines[declarationIndex].trim();
      if (stripped !== "" && !stripped.startsWith("@")) {
        break;
      }
      declarationIndex++;
    }

    if (declarationIndex >= lines.length) {
      throw new Error(
        `Expected a documented interface or method after: ${lines[closingIndex]}`,
      );
    }

    const declaration = lines[declarationIndex].trim();
    if (!/^(interface|def)\s/.test(declaration)) {
      throw new Error(
        `Expected a documented interface or method after: ${lines[closingIndex]}`,
      );
    }

    entries.push({
      declaration,
      documentation: documentation.join("\n").trim(),
    });
    index = closingIndex + 1;
  }

  return entries;
}

export function extractDocumentedDeclarations(
  source: string,
): DocumentedDeclaration[] {
  const namespaceMatch = source.match(/^\s*namespace\s+(\S+)\s*$/m);
  if (!namespaceMatch) {
    throw new Error("Expected Jo source to declare a namespace");
  }

  const namespace = namespaceMatch[1];
  const declarations: DocumentedDeclaration[] = [];
  let currentInterface = "";

  for (const entry of getDocumentationLines(source)) {
    if (entry.declaration.startsWith("interface ")) {
      currentInterface = entry.declaration.slice("interface ".length);
      declarations.push({
        name: `${namespace}.${currentInterface}`,
        signature: entry.declaration,
        doc: entry.documentation,
      });
      continue;
    }

    if (!currentInterface) {
      throw new Error(
        `Documented method must follow a documented interface: ${entry.declaration}`,
      );
    }

    const signature = entry.declaration.slice("def ".length);
    const methodName = signature.match(/^([A-Za-z_]\w*)/)?.[1];
    if (!methodName) {
      throw new Error(`Invalid documented method declaration: ${entry.declaration}`);
    }
    declarations.push({
      name: `${namespace}.${currentInterface}.${methodName}`,
      signature,
      doc: entry.documentation,
    });
  }

  return declarations;
}

function main(): void {
  const [inputPath, outputPath] = process.argv.slice(2);
  if (!inputPath || !outputPath || process.argv.length !== 4) {
    throw new Error("Usage: jo-doc-json.ts INPUT.jo OUTPUT.json");
  }
  if (!inputPath.endsWith(".jo")) {
    throw new Error(`Input file must have a .jo extension: ${inputPath}`);
  }

  const source = readFileSync(inputPath, "utf-8");
  const declarations = extractDocumentedDeclarations(source);
  writeFileSync(outputPath, JSON.stringify(declarations, null, 2) + "\n", "utf-8");
}

if (process.argv[1]?.endsWith("jo-doc-json.ts")) {
  main();
}