From 2dbc6fabb926238d8f4ba4bacf8988a5b1056635 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 00:02:21 +0200 Subject: [PATCH 01/36] Spec: support output docs in json Signed-off-by: Fengyun Liu --- docs/usage/commands/compile.md | 27 ++++++++++++++++++++++-- docs/usage/reference/compiler-options.md | 15 +++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/usage/commands/compile.md b/docs/usage/commands/compile.md index 05abbf5d..08d4bc77 100644 --- a/docs/usage/commands/compile.md +++ b/docs/usage/commands/compile.md @@ -13,8 +13,9 @@ jo compile --python|--ruby|--js [--sast ] ... \ [--lib ]... [--link-lib ]... [--link =]... -o # Generate documentation from source files (experimental) -jo compile --doc [--out ] [--title ] [--readme ] \ - [--include-private] [--include-source] ... +jo compile --doc [--format html|json] [--query ] [--out ] \ + [--title ] [--readme ] [--include-private] \ + [--include-source] ... ``` Without a backend flag, the compiler type-checks only. With a backend flag, it produces an executable or script. `--sast ` is optional in both cases — if present, `.sast` files are written to `` alongside the primary output. @@ -44,12 +45,22 @@ Experimental. | Flag | Description | |------|-------------| | `--doc` | Generate documentation instead of normal compile output | +| `--format html|json` | Documentation output format. Default: `html` | +| `--query ` | Comma-separated JSON selectors, such as `MyAPI.*` or `file:src/API.jo`; implies `--format json` | | `--out ` | Documentation output directory | | `--title ` | Documentation title | | `--readme ` | Markdown file to use as the generated documentation home page | | `--include-private` | Include private symbols | | `--include-source` | Embed source code in output | +`--format json` writes a JSON array to stdout. `--query` implies +`--format json`. JSON output does not use `--out`. +Without `--query`, JSON output contains the public API surface from the +positional source files. With `--query`, selectors are comma-separated and may +name symbols, wildcard descendants with `.*`, or source files with `file:`. +Symbol selectors are resolved from the language default scope; for example `~` +is just an ordinary symbol query. + ### App compilation | Flag | Description | @@ -104,6 +115,18 @@ jo compile --doc lib/Core.jo lib/List.jo \ --title "Jo Standard Library" ``` +Emit machine-readable docs for a source file: + +```sh +jo compile --doc --format json src/API.jo +``` + +Emit machine-readable docs for selected symbols: + +```sh +jo compile --doc --query 'MyAPI.*,jo.py.*' src/API.jo +``` + ## Notes `jo compile` is a low-level escape hatch for scripts, CI pipelines, or experiments that don't fit the standard project layout. For normal project workflows, prefer the higher-level commands such as `jo build`, `jo run `, and [`jo doc`](doc.md). diff --git a/docs/usage/reference/compiler-options.md b/docs/usage/reference/compiler-options.md index 775f7513..83d65afc 100644 --- a/docs/usage/reference/compiler-options.md +++ b/docs/usage/reference/compiler-options.md @@ -94,8 +94,23 @@ above plus these documentation-specific options. | Option | Form | Description | |--------|------|-------------| +| `--format ` | single value | Documentation output format. Default: `html`. | +| `--query ` | comma list | Select symbols or source files for JSON documentation output. Implies `--format json`. | | `--out ` | single value | Documentation output directory. Default: `docs`. | | `--title ` | single value | Documentation title. Default: `API Documentation`. | | `--readme ` | single value | Markdown file to use as the generated documentation home page. | | `--include-private` | flag | Include private symbols. | | `--include-source` | flag | Embed source code in the generated documentation. | + +`--format json` writes a bare JSON array to stdout and rejects `--out`. +`--query` implies `--format json`. +Without `--query`, it emits the public API surface from the positional source +files. `--query` accepts comma-separated selectors: + +| Selector | Meaning | +|----------|---------| +| `` | Exact symbol resolved from the language default scope. | +| `.*` | The symbol and its recursive members, emitted structurally. | +| `file:` | Queryable symbols whose source file matches the path. | + +Kind-qualified selectors such as `def:` are reserved for future use. From 66ec14a52a4e7ab807c311743b14eac16f2da56c Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 00:04:02 +0200 Subject: [PATCH 02/36] Support output json for docs Signed-off-by: Fengyun Liu --- stack-lang/doc/Compiler.scala | 46 +++++-- stack-lang/doc/DocJsonEmitter.scala | 75 ++++++++++ stack-lang/doc/DocModel.scala | 207 ++++++++++++++++++++++++++++ stack-lang/doc/DocQuery.scala | 65 +++++++++ stack-lang/doc/JsonUtil.scala | 23 ++++ 5 files changed, 408 insertions(+), 8 deletions(-) create mode 100644 stack-lang/doc/DocJsonEmitter.scala create mode 100644 stack-lang/doc/DocModel.scala create mode 100644 stack-lang/doc/DocQuery.scala create mode 100644 stack-lang/doc/JsonUtil.scala diff --git a/stack-lang/doc/Compiler.scala b/stack-lang/doc/Compiler.scala index 3a1624e8..7846481e 100644 --- a/stack-lang/doc/Compiler.scala +++ b/stack-lang/doc/Compiler.scala @@ -17,21 +17,37 @@ object Compiler: val readme: Config.StringSetting = Config.StringSetting("--readme", "", "markdown file to use as home page") val includePrivate: Config.BooleanSetting = Config.BooleanSetting("--include-private", false, "include private symbols") val includeSource: Config.BooleanSetting = Config.BooleanSetting("--include-source", false, "embed source code") + val format: Config.StringSetting = DocFormatSetting("--format", "html", "documentation output format: html or json") + val query: Config.StringSetting = Config.StringSetting("--query", "", "comma-separated documentation selectors for JSON output") val docOptions: List[cli.OptionParser.Setting[?]] = - outputDir :: title :: readme :: includePrivate :: includeSource :: Config.commonOptions + outputDir :: title :: readme :: includePrivate :: includeSource :: format :: query :: Config.commonOptions + + class DocFormatSetting(flag: String, default: String, desc: String) + extends Config.StringSetting(flag, default, desc): + override def validate()(using cf: Config, rp: Reporter): Unit = + val fmt = value + if fmt != "html" && fmt != "json" then + rp.error(s"Option $flag must be one of: html, json") def main(args: Array[String]): Unit = given Reporter = Reporter.createReporter() val (config, sources) = cli.OptionParser.parseConfig(args, docOptions) - if sources.isEmpty then + given Config = config + + val queryText = query.value.trim + val jsonOutput = format.value == "json" || queryText.nonEmpty + + if sources.isEmpty && !jsonOutput then println("Usage: jo doc [options]") println() println("Options:") println(" --out Output directory (default: docs)") println(" --title Project title for documentation") + println(" --format Output format (default: html)") + println(" --query JSON selectors, e.g. jo.py.*,file:src/API.jo") println(" --include-private Include private symbols") println(" --include-source Embed source code in output") println() @@ -40,8 +56,6 @@ object Compiler: println(" jo doc src/main.jo --out docs --title MyProject") return - given Config = config - Reporter.monitor(): compile(sources) @@ -49,16 +63,32 @@ object Compiler: def compile(sources: List[String])(using rp: Reporter, config: Config): Unit = val rootNameTable = new NameTable given lazyDefn: Definitions.Lazy = Definitions.Lazy(rootNameTable) + val queryText = query.value.trim + val jsonOutput = format.value == "json" || queryText.nonEmpty // Parse and type check - val (units, _) = sources |> Typer.parseStep |> Typer.typeStep + val (units, delayedUnits) = sources |> Typer.parseStep |> Typer.typeStep - if rp.hasErrors then - println("Errors occurred during type checking. Documentation not generated.") - return + if rp.hasErrors then return given Definitions = lazyDefn.value + if jsonOutput then + if outputDir.value != outputDir.default then + Reporter.error("--out is only supported with --format html") + return + + val libraryUnits = + if queryText.nonEmpty then delayedUnits.forceAll() + else Nil + val index = DocModel.build(units, libraryUnits, includePrivate.value) + val selected = DocQuery.select(index, queryText) + if rp.hasErrors then return + val out = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8)) + DocJsonEmitter.emit(selected, out) + out.flush() + return + val outputPath = Paths.get(outputDir.value) val includePrivateVal = includePrivate.value diff --git a/stack-lang/doc/DocJsonEmitter.scala b/stack-lang/doc/DocJsonEmitter.scala new file mode 100644 index 00000000..b6b436b9 --- /dev/null +++ b/stack-lang/doc/DocJsonEmitter.scala @@ -0,0 +1,75 @@ +package doc + +import sast.Constant + +import java.io.PrintWriter + +object DocJsonEmitter: + def emit(entries: List[DocEntry], out: PrintWriter): Unit = + out.println("[") + emitEntryList(entries, out, " ") + if entries.nonEmpty then out.println() + out.println("]") + + private def emitEntryList(entries: List[DocEntry], out: PrintWriter, indent: String): Unit = + var first = true + for entry <- entries do + if !first then out.println(",") + first = false + emitEntry(entry, out, indent) + + private def emitEntry(entry: DocEntry, out: PrintWriter, indent: String): Unit = + val next = indent + " " + out.println(indent + "{") + emitField("name", JsonUtil.string(entry.name), out, next) + emitField("kind", JsonUtil.string(entry.kind), out, next) + emitField("signature", JsonUtil.string(entry.signature), out, next) + emitField("source", sourceJson(entry.source), out, next) + emitField("visibility", JsonUtil.string(entry.visibility), out, next) + emitField("flags", stringArray(entry.flags), out, next) + emitField("annotations", annotationsJson(entry.annotations), out, next) + emitField("doc", entry.doc.map(JsonUtil.string).getOrElse("null"), out, next, comma = entry.views.nonEmpty || entry.members.nonEmpty) + + if entry.views.nonEmpty then + emitField("views", stringArray(entry.views), out, next, comma = entry.members.nonEmpty) + + if entry.members.nonEmpty then + out.println(next + JsonUtil.string("members") + ": [") + emitEntryList(entry.members, out, next + " ") + out.println() + out.println(next + "]") + + out.print(indent + "}") + + private def emitField(name: String, value: String, out: PrintWriter, indent: String, comma: Boolean = true): Unit = + out.print(indent) + out.print(JsonUtil.string(name)) + out.print(": ") + out.print(value) + if comma then out.print(",") + out.println() + + private def sourceJson(source: Option[SourceLoc]): String = + source match + case Some(SourceLoc(file, line)) => + s"""{ "file": ${JsonUtil.string(file)}, "line": $line }""" + case None => + "null" + + private def stringArray(values: List[String]): String = + values.map(JsonUtil.string).mkString("[", ", ", "]") + + private def annotationsJson(annotations: List[DocAnnotation]): String = + annotations.map { annot => + s"""{ "name": ${JsonUtil.string(annot.name)}, "args": ${valuesJson(annot.args)} }""" + }.mkString("[", ", ", "]") + + private def valuesJson(values: List[Constant]): String = + values.map(constantJson).mkString("[", ", ", "]") + + private def constantJson(value: Constant): String = + value match + case Constant.String(v) => JsonUtil.string(v) + case Constant.Int(v) => v.toString + case Constant.Float(v) => v.toString + case Constant.Bool(v) => v.toString diff --git a/stack-lang/doc/DocModel.scala b/stack-lang/doc/DocModel.scala new file mode 100644 index 00000000..7b09969c --- /dev/null +++ b/stack-lang/doc/DocModel.scala @@ -0,0 +1,207 @@ +package doc + +import sast.* +import sast.Symbols.* +import sast.Trees.* +import sast.Types.* +import sast.Denotations.* +import sast.Flags + +import scala.collection.mutable + +case class SourceLoc(file: String, line: Int) + +case class DocAnnotation(name: String, args: List[Constant]) + +case class DocEntry( + symbol: Symbol, + name: String, + kind: String, + signature: String, + source: Option[SourceLoc], + visibility: String, + flags: List[String], + annotations: List[DocAnnotation], + doc: Option[String], + members: List[DocEntry] = Nil, + views: List[String] = Nil, +): + def isAggregate: Boolean = + kind == "namespace" || kind == "section" || kind == "class" + +case class DocIndex(entries: List[DocEntry], sourceRoots: List[DocEntry]) + +object DocModel: + def build(sourceUnits: List[FileUnit], libraryUnits: List[FileUnit], includePrivate: Boolean)(using Definitions): DocIndex = + val allUnits = sourceUnits ++ libraryUnits + val cache = mutable.HashMap.empty[Symbol, Option[DocEntry]] + + def visible(sym: Symbol): Boolean = + includePrivate || !sym.isPrivate + + def sourceLoc(sym: Symbol): Option[SourceLoc] = + if sym.sourcePos == null then None + else Some(SourceLoc(sym.source.file, sym.sourcePos.startLine + 1)) + + def visibility(sym: Symbol): String = + sym.visibility match + case Visibility.Default => "public" + case Visibility.Private(within) => + if within == sym.owner then "private" + else s"private[${within.fullName}]" + + def docs(sym: Symbol): Option[String] = + val lines = summon[Definitions].index.docComment(sym) + if lines.isEmpty then None else Some(lines.mkString("\n")) + + def annotations(sym: Symbol): List[DocAnnotation] = + sym.annotations.map: annot => + DocAnnotation(annot.symbol.fullName, annot.args) + + def flags(sym: Symbol): List[String] = + Flags.flagStrings(sym.flags) + + def typeParams(params: List[Symbol]): String = + if params.isEmpty then "" else params.map(_.name).mkString("[", ", ", "]") + + def symbolBase(sym: Symbol, kind: String, signature: String, members: List[DocEntry] = Nil, views: List[String] = Nil): DocEntry = + DocEntry( + symbol = sym, + name = sym.fullName, + kind = kind, + signature = signature, + source = sourceLoc(sym), + visibility = visibility(sym), + flags = flags(sym), + annotations = annotations(sym), + doc = docs(sym), + members = sortEntries(members), + views = views, + ) + + def procSignature(sym: Symbol): String = + sym.tpe.asProcType.show + + def funSignature(fd: FunDef): String = + val sym = fd.symbol + if sym.is(Flags.Annotation) then + "annotation " + sym.name + procSignature(sym).stripSuffix(": void receives none").stripSuffix(": void") + else if sym.is(Flags.Constructor) then + "constructor" + procSignature(sym) + else + "def " + sym.name + procSignature(sym) + + def patternSignature(pd: PatDef): String = + "pattern " + pd.symbol.name + procSignature(pd.symbol) + + def typeSignature(td: TypeDef): String = + val sym = td.symbol + val info = sym.info + val rhs = + if sym.is(Flags.Defer) then "" + else + info match + case TypeOperatorInfo(_, body, _) => " = " + body.show + case tp: Type => " = " + tp.show + case other => " = " + other.toString + "type " + sym.name + typeParams(td.tparams) + rhs + + def classSignature(cd: ClassDef): String = + val sym = cd.symbol + val prefix = + if sym.is(Flags.Object) then "object" + else if sym.isInterface then "interface" + else "class" + prefix + " " + sym.name + typeParams(cd.tparams) + + def interfaceSignature(id: InterfaceDef): String = + "interface " + id.symbol.name + typeParams(id.tparams) + + def fieldSignature(field: FieldDecl): String = + val sym = field.symbol + val prefix = if sym.isMutable then "var " else "val " + prefix + sym.name + ": " + field.tpt.tpe.show + + def entryForField(field: FieldDecl): Option[DocEntry] = + val sym = field.symbol + if !visible(sym) then None + else Some(symbolBase(sym, "field", fieldSignature(field))) + + def entryForFun(fd: FunDef): Option[DocEntry] = + val sym = fd.symbol + if !visible(sym) || sym.is(Flags.Object) then None + else + val kind = if sym.is(Flags.Constructor) then "constructor" else "def" + Some(symbolBase(sym, kind, funSignature(fd))) + + def entryForDef(defn: Def): Option[DocEntry] = + cache.getOrElseUpdate(defn.symbol, + if !visible(defn.symbol) then None + else + defn match + case pd: ParamDef => + Some(symbolBase(pd.symbol, "param", "param " + pd.name + ": " + pd.tpt.tpe.show)) + + case td: TypeDef => + Some(symbolBase(td.symbol, "type", typeSignature(td))) + + case fd: FunDef => + entryForFun(fd) + + case pd: PatDef => + if pd.resultType.tpe.isSingletonObjectType then None + else Some(symbolBase(pd.symbol, "pattern", patternSignature(pd))) + + case cd: ClassDef => + val fields = cd.vals.flatMap(entryForField) + val funs = cd.funs.flatMap(entryForFun) + val views = cd.views.map(_.tpe.show) + Some(symbolBase(cd.symbol, "class", classSignature(cd), fields ++ funs, views)) + + case id: InterfaceDef => + val methods = id.methods.flatMap(entryForFun) + Some(symbolBase(id.symbol, "class", interfaceSignature(id), methods)) + + case sec: Section => + val members = sec.defs.flatMap(entryForDef) + Some(symbolBase(sec.symbol, "section", "section " + sec.symbol.name, members)) + ) + + def namespaceEntry(sym: Symbol, units: List[FileUnit]): Option[DocEntry] = + if !visible(sym) then None + else + val members = units.flatMap(_.defs.flatMap(entryForDef)) + Some(symbolBase(sym, "namespace", "namespace " + sym.fullName, members)) + + val namespaceEntries = + allUnits.groupBy(_.owner).toList.sortBy(_._1.fullName).flatMap(namespaceEntry) + + val sourceRoots = + sourceUnits.flatMap(unit => unit.defs.flatMap(entryForDef)) + + val entries = + distinctEntries(namespaceEntries.flatMap(flattenEntry) ++ sourceRoots.flatMap(flattenEntry)) + + DocIndex(entries, sortEntries(sourceRoots)) + + def flattenEntry(entry: DocEntry): List[DocEntry] = + entry :: entry.members.flatMap(flattenEntry) + + def distinctEntries(entries: List[DocEntry]): List[DocEntry] = + val seen = mutable.HashSet.empty[Symbol] + entries.filter: entry => + if seen.contains(entry.symbol) then false + else + seen += entry.symbol + true + + def sortEntries(entries: List[DocEntry]): List[DocEntry] = + entries.map(sortEntry).sortBy(sortKey) + + private def sortEntry(entry: DocEntry): DocEntry = + entry.copy(members = sortEntries(entry.members)) + + private def sortKey(entry: DocEntry): (String, Int, String, String) = + val file = entry.source.map(_.file).getOrElse("") + val line = entry.source.map(_.line).getOrElse(0) + (file, line, entry.kind, entry.name) diff --git a/stack-lang/doc/DocQuery.scala b/stack-lang/doc/DocQuery.scala new file mode 100644 index 00000000..2eeb885b --- /dev/null +++ b/stack-lang/doc/DocQuery.scala @@ -0,0 +1,65 @@ +package doc + +import reporting.Reporter +import sast.Symbols.Symbol + +import java.nio.file.Paths +import scala.collection.mutable + +object DocQuery: + def select(index: DocIndex, rawQuery: String)(using Reporter): List[DocEntry] = + val selectors = parseSelectors(rawQuery) + val selected = + if selectors.isEmpty then index.sourceRoots + else selectors.flatMap(resolveSelector(index, _)) + + DocModel.sortEntries(pruneDescendants(DocModel.distinctEntries(selected))) + + private def parseSelectors(rawQuery: String): List[String] = + rawQuery.split(",").map(_.trim).filter(_.nonEmpty).toList + + private def resolveSelector(index: DocIndex, selector: String)(using Reporter): List[DocEntry] = + if selector.startsWith("file:") then + val file = selector.stripPrefix("file:") + val matches = index.entries.filter(entry => matchesFile(entry, file)) + if matches.isEmpty then Reporter.error(s"No documentation entries match file selector `$selector`") + matches + else + val symbolSelector = + if selector.endsWith(".*") then selector.stripSuffix(".*") + else selector + + val matches = resolveSymbol(index, symbolSelector) + if matches.isEmpty then Reporter.error(s"No documentation entries match symbol selector `$selector`") + matches + + private def resolveSymbol(index: DocIndex, selector: String): List[DocEntry] = + val exactNames = + if selector.contains(".") then List(selector) + else List(selector, "jo." + selector) + + val exact = index.entries.filter(entry => exactNames.contains(entry.name)) + if exact.nonEmpty then exact + else if selector.contains(".") then Nil + else index.entries.filter(_.symbol.name == selector) + + private def matchesFile(entry: DocEntry, rawFile: String): Boolean = + entry.source.exists: source => + val rawNormalized = normalize(rawFile) + val sourceNormalized = normalize(source.file) + rawFile == source.file || + rawNormalized == sourceNormalized || + absolute(rawFile) == absolute(source.file) + + private def normalize(path: String): String = + Paths.get(path).normalize().toString + + private def absolute(path: String): String = + Paths.get(path).toAbsolutePath.normalize().toString + + private def pruneDescendants(entries: List[DocEntry]): List[DocEntry] = + val selected = mutable.HashSet.empty[Symbol] + selected ++= entries.map(_.symbol) + + entries.filterNot: entry => + entry.symbol.ownersIterator.exists(selected.contains) diff --git a/stack-lang/doc/JsonUtil.scala b/stack-lang/doc/JsonUtil.scala new file mode 100644 index 00000000..857ac291 --- /dev/null +++ b/stack-lang/doc/JsonUtil.scala @@ -0,0 +1,23 @@ +package doc + +import java.io.PrintWriter + +object JsonUtil: + def string(s: String): String = + val sb = new StringBuilder("\"") + for c <- s do + c match + case '"' => sb.append("\\\"") + case '\\' => sb.append("\\\\") + case '\b' => sb.append("\\b") + case '\f' => sb.append("\\f") + case '\n' => sb.append("\\n") + case '\r' => sb.append("\\r") + case '\t' => sb.append("\\t") + case c if c < 32 => sb.append("\\u%04x".format(c.toInt)) + case c => sb.append(c) + sb.append("\"") + sb.toString + + def emitString(s: String, out: PrintWriter): Unit = + out.print(string(s)) From 1ab9842c6da0182f295392308d6a2510ed4b61b2 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 00:05:59 +0200 Subject: [PATCH 03/36] Spec: Support query without source code Signed-off-by: Fengyun Liu --- docs/usage/commands/compile.md | 12 ++++++++++-- docs/usage/reference/compiler-options.md | 4 ++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/usage/commands/compile.md b/docs/usage/commands/compile.md index 08d4bc77..1d15de70 100644 --- a/docs/usage/commands/compile.md +++ b/docs/usage/commands/compile.md @@ -15,7 +15,7 @@ jo compile --python|--ruby|--js [--sast ] ... \ # Generate documentation from source files (experimental) jo compile --doc [--format html|json] [--query ] [--out ] \ [--title ] [--readme ] [--include-private] \ - [--include-source] ... + [--include-source] [...] ``` Without a backend flag, the compiler type-checks only. With a backend flag, it produces an executable or script. `--sast ` is optional in both cases — if present, `.sast` files are written to `` alongside the primary output. @@ -59,7 +59,9 @@ Without `--query`, JSON output contains the public API surface from the positional source files. With `--query`, selectors are comma-separated and may name symbols, wildcard descendants with `.*`, or source files with `file:`. Symbol selectors are resolved from the language default scope; for example `~` -is just an ordinary symbol query. +is just an ordinary symbol query. A query may omit positional source files; in +that case it searches the loaded SAST libraries, including stdlib unless +`--no-stdlib` is set and any `--lib` paths. ### App compilation @@ -127,6 +129,12 @@ Emit machine-readable docs for selected symbols: jo compile --doc --query 'MyAPI.*,jo.py.*' src/API.jo ``` +Query stdlib docs from SAST files only: + +```sh +jo compile --doc --query jo.List +``` + ## Notes `jo compile` is a low-level escape hatch for scripts, CI pipelines, or experiments that don't fit the standard project layout. For normal project workflows, prefer the higher-level commands such as `jo build`, `jo run `, and [`jo doc`](doc.md). diff --git a/docs/usage/reference/compiler-options.md b/docs/usage/reference/compiler-options.md index 83d65afc..eb913451 100644 --- a/docs/usage/reference/compiler-options.md +++ b/docs/usage/reference/compiler-options.md @@ -113,4 +113,8 @@ files. `--query` accepts comma-separated selectors: | `.*` | The symbol and its recursive members, emitted structurally. | | `file:` | Queryable symbols whose source file matches the path. | +When `--query` is present, positional source files are optional. With no source +files, the query searches the loaded SAST libraries: stdlib by default, runtime +API libraries selected by `--use-runtime-api`, and any `--lib` paths. + Kind-qualified selectors such as `def:` are reserved for future use. From ffbb59d5bde9db19dfdcfba7861e2373a115f7fa Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 00:07:48 +0200 Subject: [PATCH 04/36] Support query sast without source Signed-off-by: Fengyun Liu --- stack-lang/cli/Main.scala | 2 +- stack-lang/doc/Compiler.scala | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/stack-lang/cli/Main.scala b/stack-lang/cli/Main.scala index 57fa71f4..c7d26149 100644 --- a/stack-lang/cli/Main.scala +++ b/stack-lang/cli/Main.scala @@ -274,7 +274,7 @@ object Main: | jo versions use Switch the active compiler version | jo versions remove Remove an installed compiler version | jo compile [options] Compile application or library - | jo compile --doc [options] Generate documentation from source files + | jo compile --doc [options] [files...] Generate documentation from source files or queried SAST libraries | jo doc [module] Generate module documentation | jo help Show this help message | diff --git a/stack-lang/doc/Compiler.scala b/stack-lang/doc/Compiler.scala index 7846481e..f81db5c3 100644 --- a/stack-lang/doc/Compiler.scala +++ b/stack-lang/doc/Compiler.scala @@ -38,9 +38,8 @@ object Compiler: given Config = config val queryText = query.value.trim - val jsonOutput = format.value == "json" || queryText.nonEmpty - if sources.isEmpty && !jsonOutput then + if sources.isEmpty && queryText.isEmpty then println("Usage: jo doc [options]") println() println("Options:") @@ -54,7 +53,7 @@ object Compiler: println("Examples:") println(" jo doc lib/Core.jo lib/List.jo --out site/api") println(" jo doc src/main.jo --out docs --title MyProject") - return + System.exit(1) Reporter.monitor(): compile(sources) From 1140c344481d1b9126a07e13c68637d42d4510e7 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 00:22:59 +0200 Subject: [PATCH 05/36] Add tests for json doc output --- tests/custom/doc-query/all.json.check | 76 +++++++++++++++ tests/custom/doc-query/api.jo | 28 ++++++ tests/custom/doc-query/lib-query.json.check | 88 ++++++++++++++++++ tests/custom/doc-query/structural.json.check | 88 ++++++++++++++++++ tests/custom/doc-query/test.sh | 97 ++++++++++++++++++++ 5 files changed, 377 insertions(+) create mode 100644 tests/custom/doc-query/all.json.check create mode 100644 tests/custom/doc-query/api.jo create mode 100644 tests/custom/doc-query/lib-query.json.check create mode 100644 tests/custom/doc-query/structural.json.check create mode 100755 tests/custom/doc-query/test.sh diff --git a/tests/custom/doc-query/all.json.check b/tests/custom/doc-query/all.json.check new file mode 100644 index 00000000..ac0e1a45 --- /dev/null +++ b/tests/custom/doc-query/all.json.check @@ -0,0 +1,76 @@ +[ + { + "name": "DocQueryAPI.FileLike", + "kind": "class", + "signature": "interface FileLike", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 5 }, + "visibility": "public", + "flags": ["interface"], + "annotations": [{ "name": "jo.py.interop", "args": [] }], + "doc": "File operations exposed to a Python-backed host.", + "members": [ + { + "name": "DocQueryAPI.FileLike.readText", + "kind": "def", + "signature": "def readText(path: String, limit: Int = 10): String receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 8 }, + "visibility": "public", + "flags": ["defer", "fun", "method"], + "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], + "doc": "Read a text file with an optional limit." + } + ] + }, + { + "name": "DocQueryAPI.Box", + "kind": "class", + "signature": "class Box", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 12 }, + "visibility": "public", + "flags": ["class"], + "annotations": [], + "doc": "A small documented class.", + "members": [ + { + "name": "DocQueryAPI.Box.name", + "kind": "field", + "signature": "val name: String", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 14 }, + "visibility": "public", + "flags": ["field"], + "annotations": [], + "doc": "Stored display name." + }, + { + "name": "DocQueryAPI.Box.", + "kind": "constructor", + "signature": "constructor(name: String): Box receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 17 }, + "visibility": "public", + "flags": ["fun", "method", "constructor"], + "annotations": [], + "doc": "Create a box with a name." + }, + { + "name": "DocQueryAPI.Box.label", + "kind": "def", + "signature": "def label(prefix: String = \"box\"): String receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 21 }, + "visibility": "public", + "flags": ["fun", "method"], + "annotations": [], + "doc": "Render a label." + } + ] + }, + { + "name": "DocQueryAPI.helper", + "kind": "def", + "signature": "def helper(value: Int): Int receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 25 }, + "visibility": "public", + "flags": ["fun"], + "annotations": [], + "doc": "Public helper." + } +] diff --git a/tests/custom/doc-query/api.jo b/tests/custom/doc-query/api.jo new file mode 100644 index 00000000..1f8a5abf --- /dev/null +++ b/tests/custom/doc-query/api.jo @@ -0,0 +1,28 @@ +namespace DocQueryAPI + +//[ File operations exposed to a Python-backed host. //] +@py.interop +interface FileLike + //[ Read a text file with an optional limit. //] + @py.targetName("read_text") + def readText(path: String, limit: Int = 10): String +end + +//[ A small documented class. //] +class Box + //[ Stored display name. //] + val name: String + + //[ Create a box with a name. //] + def Box(name: String): Box = + this.name = name + + //[ Render a label. //] + def label(prefix: String = "box"): String = prefix + ":" + name +end + +//[ Public helper. //] +def helper(value: Int): Int = value + 1 + +//[ Hidden helper. //] +private def hidden: Int = 42 diff --git a/tests/custom/doc-query/lib-query.json.check b/tests/custom/doc-query/lib-query.json.check new file mode 100644 index 00000000..e62f3102 --- /dev/null +++ b/tests/custom/doc-query/lib-query.json.check @@ -0,0 +1,88 @@ +[ + { + "name": "DocQueryAPI", + "kind": "namespace", + "signature": "namespace DocQueryAPI", + "source": null, + "visibility": "public", + "flags": ["namespace"], + "annotations": [], + "doc": null, + "members": [ + { + "name": "DocQueryAPI.FileLike", + "kind": "class", + "signature": "interface FileLike", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 5 }, + "visibility": "public", + "flags": ["interface"], + "annotations": [{ "name": "jo.py.interop", "args": [] }], + "doc": "File operations exposed to a Python-backed host.", + "members": [ + { + "name": "DocQueryAPI.FileLike.readText", + "kind": "def", + "signature": "def readText(path: String, limit: Int = 10): String receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 8 }, + "visibility": "public", + "flags": ["defer", "loaded", "fun", "method"], + "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], + "doc": "Read a text file with an optional limit." + } + ] + }, + { + "name": "DocQueryAPI.Box", + "kind": "class", + "signature": "class Box", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 12 }, + "visibility": "public", + "flags": ["class"], + "annotations": [], + "doc": "A small documented class.", + "members": [ + { + "name": "DocQueryAPI.Box.name", + "kind": "field", + "signature": "val name: String", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 14 }, + "visibility": "public", + "flags": ["field"], + "annotations": [], + "doc": "Stored display name." + }, + { + "name": "DocQueryAPI.Box.", + "kind": "constructor", + "signature": "constructor(name: String): Box receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 17 }, + "visibility": "public", + "flags": ["loaded", "fun", "method", "constructor"], + "annotations": [], + "doc": "Create a box with a name." + }, + { + "name": "DocQueryAPI.Box.label", + "kind": "def", + "signature": "def label(prefix: String = \"box\"): String receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 21 }, + "visibility": "public", + "flags": ["loaded", "fun", "method"], + "annotations": [], + "doc": "Render a label." + } + ] + }, + { + "name": "DocQueryAPI.helper", + "kind": "def", + "signature": "def helper(value: Int): Int receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 25 }, + "visibility": "public", + "flags": ["loaded", "fun"], + "annotations": [], + "doc": "Public helper." + } + ] + } +] diff --git a/tests/custom/doc-query/structural.json.check b/tests/custom/doc-query/structural.json.check new file mode 100644 index 00000000..d148f082 --- /dev/null +++ b/tests/custom/doc-query/structural.json.check @@ -0,0 +1,88 @@ +[ + { + "name": "DocQueryAPI", + "kind": "namespace", + "signature": "namespace DocQueryAPI", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 1 }, + "visibility": "public", + "flags": ["namespace"], + "annotations": [], + "doc": null, + "members": [ + { + "name": "DocQueryAPI.FileLike", + "kind": "class", + "signature": "interface FileLike", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 5 }, + "visibility": "public", + "flags": ["interface"], + "annotations": [{ "name": "jo.py.interop", "args": [] }], + "doc": "File operations exposed to a Python-backed host.", + "members": [ + { + "name": "DocQueryAPI.FileLike.readText", + "kind": "def", + "signature": "def readText(path: String, limit: Int = 10): String receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 8 }, + "visibility": "public", + "flags": ["defer", "fun", "method"], + "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], + "doc": "Read a text file with an optional limit." + } + ] + }, + { + "name": "DocQueryAPI.Box", + "kind": "class", + "signature": "class Box", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 12 }, + "visibility": "public", + "flags": ["class"], + "annotations": [], + "doc": "A small documented class.", + "members": [ + { + "name": "DocQueryAPI.Box.name", + "kind": "field", + "signature": "val name: String", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 14 }, + "visibility": "public", + "flags": ["field"], + "annotations": [], + "doc": "Stored display name." + }, + { + "name": "DocQueryAPI.Box.", + "kind": "constructor", + "signature": "constructor(name: String): Box receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 17 }, + "visibility": "public", + "flags": ["fun", "method", "constructor"], + "annotations": [], + "doc": "Create a box with a name." + }, + { + "name": "DocQueryAPI.Box.label", + "kind": "def", + "signature": "def label(prefix: String = \"box\"): String receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 21 }, + "visibility": "public", + "flags": ["fun", "method"], + "annotations": [], + "doc": "Render a label." + } + ] + }, + { + "name": "DocQueryAPI.helper", + "kind": "def", + "signature": "def helper(value: Int): Int receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 25 }, + "visibility": "public", + "flags": ["fun"], + "annotations": [], + "doc": "Public helper." + } + ] + } +] diff --git a/tests/custom/doc-query/test.sh b/tests/custom/doc-query/test.sh new file mode 100755 index 00000000..9d38a803 --- /dev/null +++ b/tests/custom/doc-query/test.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$DIR/../../.." && pwd)" +TEST_NAME="$(basename "$DIR")" +REL_DIR="tests/custom/doc-query" +API_FILE="$REL_DIR/api.jo" +SAST_DIR="${TMPDIR:-/tmp}/jo-doc-query-sast-$$" +SAST_LOG="$SAST_DIR.log" +FAIL_LOG="$DIR/fail.log" + +cleanup() { + rm -f "$DIR"/*.json + rm -rf "$SAST_DIR" "$SAST_LOG" "$FAIL_LOG" +} + +finish() { + local status=$? + if [ "$status" -eq 0 ]; then + cleanup + else + echo " failed; temporary JSON/log files were left for inspection" + fi +} + +trap finish EXIT + +echo "Testing $TEST_NAME" + +cleanup + +run_doc() { + "$PROJECT_ROOT/bin/jo" compile --doc --use-runtime-api python "$@" +} + +run_plain_doc() { + "$PROJECT_ROOT/bin/jo" compile --doc "$@" +} + +run_json_doc() { + "$PROJECT_ROOT/bin/jo" compile --doc --format json --use-runtime-api python "$@" +} + +expect_fail() { + local out="$1" + shift + if "$@" > "$out" 2>&1; then + echo "Expected command to fail: $*" + exit 1 + fi +} + +cd "$PROJECT_ROOT" + +"$PROJECT_ROOT/bin/jo" compile --sast "$SAST_DIR" --use-runtime-api python "$API_FILE" > "$SAST_LOG" + +run_json_doc "$API_FILE" > "$DIR/all.json" +run_json_doc --query "DocQueryAPI.*,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/structural.json" +run_json_doc --query "file:$API_FILE" "$API_FILE" > "$DIR/file.json" +run_json_doc --query "file:$PROJECT_ROOT/$API_FILE" "$API_FILE" > "$DIR/file-absolute.json" +run_json_doc --query "DocQueryAPI.Box.label" "$API_FILE" > "$DIR/exact-member.json" +run_json_doc --include-private --query "DocQueryAPI.hidden" "$API_FILE" > "$DIR/private.json" +run_doc --query "DocQueryAPI.*,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/query-implies-json.json" +run_doc --lib "$SAST_DIR" --query "DocQueryAPI.*" > "$DIR/lib-query.json" +run_plain_doc --query "jo.List" > "$DIR/stdlib-list.json" + +diff -u "$DIR/all.json.check" "$DIR/all.json" +diff -u "$DIR/structural.json.check" "$DIR/structural.json" +diff -u "$DIR/structural.json.check" "$DIR/file.json" +diff -u "$DIR/structural.json.check" "$DIR/file-absolute.json" +diff -u "$DIR/structural.json.check" "$DIR/query-implies-json.json" +diff -u "$DIR/lib-query.json.check" "$DIR/lib-query.json" + +python3 "$DIR/assert_json.py" source-all "$DIR/all.json" +python3 "$DIR/assert_json.py" structural "$DIR/structural.json" +python3 "$DIR/assert_json.py" exact-member "$DIR/exact-member.json" +python3 "$DIR/assert_json.py" private "$DIR/private.json" +python3 "$DIR/assert_json.py" stdlib-list "$DIR/stdlib-list.json" + +expect_fail "$FAIL_LOG" "$PROJECT_ROOT/bin/jo" compile --doc --format yaml "$API_FILE" +rg -q "Option --format must be one of: html, json" "$FAIL_LOG" + +expect_fail "$FAIL_LOG" run_json_doc --out "$SAST_DIR/out" "$API_FILE" +rg -q -- "--out is only supported with --format html" "$FAIL_LOG" + +expect_fail "$FAIL_LOG" run_plain_doc --query "NoSuchSymbol" +rg -q "No documentation entries match symbol selector" "$FAIL_LOG" + +expect_fail "$FAIL_LOG" run_plain_doc --no-stdlib --query "jo.List" +rg -q "No documentation entries match symbol selector" "$FAIL_LOG" + +expect_fail "$FAIL_LOG" run_plain_doc --format json +rg -q "Usage: jo doc" "$FAIL_LOG" + +echo " ✓ All tests passed for $TEST_NAME" From 88294f3325089a410be4e803c46ec80e867db157 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 00:39:23 +0200 Subject: [PATCH 06/36] Add different definitions to the test --- tests/custom/doc-query/all.json.check | 108 +++++++++++++++++-- tests/custom/doc-query/api.jo | 30 ++++++ tests/custom/doc-query/lib-query.json.check | 108 +++++++++++++++++-- tests/custom/doc-query/structural.json.check | 108 +++++++++++++++++-- 4 files changed, 333 insertions(+), 21 deletions(-) diff --git a/tests/custom/doc-query/all.json.check b/tests/custom/doc-query/all.json.check index ac0e1a45..4910aa8b 100644 --- a/tests/custom/doc-query/all.json.check +++ b/tests/custom/doc-query/all.json.check @@ -1,9 +1,103 @@ [ + { + "name": "DocQueryAPI.defaultLimit", + "kind": "param", + "signature": "param defaultLimit: Int", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 4 }, + "visibility": "public", + "flags": ["context"], + "annotations": [], + "doc": "Default limit used by context-aware helpers." + }, + { + "name": "DocQueryAPI.Path", + "kind": "type", + "signature": "type Path = String", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 7 }, + "visibility": "public", + "flags": [], + "annotations": [], + "doc": "File path alias used in host calls." + }, + { + "name": "DocQueryAPI.Positive", + "kind": "pattern", + "signature": "pattern Positive(value: Int): Partial[Int] receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 10 }, + "visibility": "public", + "flags": ["fun"], + "annotations": [], + "doc": "Match positive integers." + }, + { + "name": "DocQueryAPI.HostOps", + "kind": "section", + "signature": "section HostOps", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 14 }, + "visibility": "public", + "flags": ["section"], + "annotations": [], + "doc": "Host-backed operations grouped in a section.", + "members": [ + { + "name": "DocQueryAPI.HostOps.remoteSize", + "kind": "def", + "signature": "def remoteSize(path: Path): Int receives defaultLimit", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 16 }, + "visibility": "public", + "flags": ["defer", "fun"], + "annotations": [], + "doc": "Return a remote file size." + }, + { + "name": "DocQueryAPI.HostOps.normalizeLimit", + "kind": "def", + "signature": "def normalizeLimit(value: Int): Int receives defaultLimit", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 19 }, + "visibility": "public", + "flags": ["fun"], + "annotations": [], + "doc": "Clamp a caller supplied limit." + } + ] + }, + { + "name": "DocQueryAPI.Singleton", + "kind": "class", + "signature": "object Singleton", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "visibility": "public", + "flags": ["object", "class"], + "annotations": [], + "doc": "Shared singleton helpers.", + "members": [ + { + "name": "DocQueryAPI.Singleton.", + "kind": "constructor", + "signature": "constructor(): Singleton receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "visibility": "public", + "flags": ["fun", "method", "constructor"], + "annotations": [], + "doc": null + }, + { + "name": "DocQueryAPI.Singleton.label", + "kind": "def", + "signature": "def label(): String receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 26 }, + "visibility": "public", + "flags": ["fun", "method"], + "annotations": [], + "doc": "Return the singleton label." + } + ] + }, { "name": "DocQueryAPI.FileLike", "kind": "class", "signature": "interface FileLike", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 5 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 31 }, "visibility": "public", "flags": ["interface"], "annotations": [{ "name": "jo.py.interop", "args": [] }], @@ -13,7 +107,7 @@ "name": "DocQueryAPI.FileLike.readText", "kind": "def", "signature": "def readText(path: String, limit: Int = 10): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 8 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 34 }, "visibility": "public", "flags": ["defer", "fun", "method"], "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], @@ -25,7 +119,7 @@ "name": "DocQueryAPI.Box", "kind": "class", "signature": "class Box", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 12 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, "visibility": "public", "flags": ["class"], "annotations": [], @@ -35,7 +129,7 @@ "name": "DocQueryAPI.Box.name", "kind": "field", "signature": "val name: String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 14 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 40 }, "visibility": "public", "flags": ["field"], "annotations": [], @@ -45,7 +139,7 @@ "name": "DocQueryAPI.Box.", "kind": "constructor", "signature": "constructor(name: String): Box receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 17 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 43 }, "visibility": "public", "flags": ["fun", "method", "constructor"], "annotations": [], @@ -55,7 +149,7 @@ "name": "DocQueryAPI.Box.label", "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 21 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 47 }, "visibility": "public", "flags": ["fun", "method"], "annotations": [], @@ -67,7 +161,7 @@ "name": "DocQueryAPI.helper", "kind": "def", "signature": "def helper(value: Int): Int receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 25 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, "visibility": "public", "flags": ["fun"], "annotations": [], diff --git a/tests/custom/doc-query/api.jo b/tests/custom/doc-query/api.jo index 1f8a5abf..47f0b50f 100644 --- a/tests/custom/doc-query/api.jo +++ b/tests/custom/doc-query/api.jo @@ -1,5 +1,35 @@ namespace DocQueryAPI +//[ Default limit used by context-aware helpers. //] +param defaultLimit: Int + +//[ File path alias used in host calls. //] +type Path = String + +//[ Match positive integers. //] +pattern Positive(value: Int): Partial[Int] = + case value if value > 0 + +//[ Host-backed operations grouped in a section. //] +section HostOps + //[ Return a remote file size. //] + defer def remoteSize(path: Path): Int receives defaultLimit + + //[ Clamp a caller supplied limit. //] + def normalizeLimit(value: Int): Int receives defaultLimit = + if value > 0 then value else defaultLimit +end + +//[ Render a value with auto formatting and context. //] +def describe[T](value: T)(auto show: Show[T] with [[T].toString]): String receives defaultLimit = + show.show(value) + ":" + defaultLimit.toString() + +//[ Shared singleton helpers. //] +object Singleton + //[ Return the singleton label. //] + def label: String = "singleton" +end + //[ File operations exposed to a Python-backed host. //] @py.interop interface FileLike diff --git a/tests/custom/doc-query/lib-query.json.check b/tests/custom/doc-query/lib-query.json.check index e62f3102..f9e3f6ee 100644 --- a/tests/custom/doc-query/lib-query.json.check +++ b/tests/custom/doc-query/lib-query.json.check @@ -9,11 +9,105 @@ "annotations": [], "doc": null, "members": [ + { + "name": "DocQueryAPI.defaultLimit", + "kind": "param", + "signature": "param defaultLimit: Int", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 4 }, + "visibility": "public", + "flags": ["context"], + "annotations": [], + "doc": "Default limit used by context-aware helpers." + }, + { + "name": "DocQueryAPI.Path", + "kind": "type", + "signature": "type Path = String", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 7 }, + "visibility": "public", + "flags": [], + "annotations": [], + "doc": "File path alias used in host calls." + }, + { + "name": "DocQueryAPI.Positive", + "kind": "pattern", + "signature": "pattern Positive(value: Int): Partial[Int] receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 10 }, + "visibility": "public", + "flags": ["fun"], + "annotations": [], + "doc": "Match positive integers." + }, + { + "name": "DocQueryAPI.HostOps", + "kind": "section", + "signature": "section HostOps", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 14 }, + "visibility": "public", + "flags": ["section"], + "annotations": [], + "doc": "Host-backed operations grouped in a section.", + "members": [ + { + "name": "DocQueryAPI.HostOps.remoteSize", + "kind": "def", + "signature": "def remoteSize(path: Path): Int receives defaultLimit", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 16 }, + "visibility": "public", + "flags": ["defer", "loaded", "fun"], + "annotations": [], + "doc": "Return a remote file size." + }, + { + "name": "DocQueryAPI.HostOps.normalizeLimit", + "kind": "def", + "signature": "def normalizeLimit(value: Int): Int receives defaultLimit", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 19 }, + "visibility": "public", + "flags": ["loaded", "fun"], + "annotations": [], + "doc": "Clamp a caller supplied limit." + } + ] + }, + { + "name": "DocQueryAPI.Singleton", + "kind": "class", + "signature": "object Singleton", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "visibility": "public", + "flags": ["object", "class"], + "annotations": [], + "doc": "Shared singleton helpers.", + "members": [ + { + "name": "DocQueryAPI.Singleton.", + "kind": "constructor", + "signature": "constructor(): Singleton receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "visibility": "public", + "flags": ["loaded", "fun", "method", "constructor"], + "annotations": [], + "doc": null + }, + { + "name": "DocQueryAPI.Singleton.label", + "kind": "def", + "signature": "def label(): String receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 26 }, + "visibility": "public", + "flags": ["loaded", "fun", "method"], + "annotations": [], + "doc": "Return the singleton label." + } + ] + }, { "name": "DocQueryAPI.FileLike", "kind": "class", "signature": "interface FileLike", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 5 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 31 }, "visibility": "public", "flags": ["interface"], "annotations": [{ "name": "jo.py.interop", "args": [] }], @@ -23,7 +117,7 @@ "name": "DocQueryAPI.FileLike.readText", "kind": "def", "signature": "def readText(path: String, limit: Int = 10): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 8 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 34 }, "visibility": "public", "flags": ["defer", "loaded", "fun", "method"], "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], @@ -35,7 +129,7 @@ "name": "DocQueryAPI.Box", "kind": "class", "signature": "class Box", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 12 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, "visibility": "public", "flags": ["class"], "annotations": [], @@ -45,7 +139,7 @@ "name": "DocQueryAPI.Box.name", "kind": "field", "signature": "val name: String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 14 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 40 }, "visibility": "public", "flags": ["field"], "annotations": [], @@ -55,7 +149,7 @@ "name": "DocQueryAPI.Box.", "kind": "constructor", "signature": "constructor(name: String): Box receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 17 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 43 }, "visibility": "public", "flags": ["loaded", "fun", "method", "constructor"], "annotations": [], @@ -65,7 +159,7 @@ "name": "DocQueryAPI.Box.label", "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 21 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 47 }, "visibility": "public", "flags": ["loaded", "fun", "method"], "annotations": [], @@ -77,7 +171,7 @@ "name": "DocQueryAPI.helper", "kind": "def", "signature": "def helper(value: Int): Int receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 25 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, "visibility": "public", "flags": ["loaded", "fun"], "annotations": [], diff --git a/tests/custom/doc-query/structural.json.check b/tests/custom/doc-query/structural.json.check index d148f082..30b5b2cd 100644 --- a/tests/custom/doc-query/structural.json.check +++ b/tests/custom/doc-query/structural.json.check @@ -9,11 +9,105 @@ "annotations": [], "doc": null, "members": [ + { + "name": "DocQueryAPI.defaultLimit", + "kind": "param", + "signature": "param defaultLimit: Int", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 4 }, + "visibility": "public", + "flags": ["context"], + "annotations": [], + "doc": "Default limit used by context-aware helpers." + }, + { + "name": "DocQueryAPI.Path", + "kind": "type", + "signature": "type Path = String", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 7 }, + "visibility": "public", + "flags": [], + "annotations": [], + "doc": "File path alias used in host calls." + }, + { + "name": "DocQueryAPI.Positive", + "kind": "pattern", + "signature": "pattern Positive(value: Int): Partial[Int] receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 10 }, + "visibility": "public", + "flags": ["fun"], + "annotations": [], + "doc": "Match positive integers." + }, + { + "name": "DocQueryAPI.HostOps", + "kind": "section", + "signature": "section HostOps", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 14 }, + "visibility": "public", + "flags": ["section"], + "annotations": [], + "doc": "Host-backed operations grouped in a section.", + "members": [ + { + "name": "DocQueryAPI.HostOps.remoteSize", + "kind": "def", + "signature": "def remoteSize(path: Path): Int receives defaultLimit", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 16 }, + "visibility": "public", + "flags": ["defer", "fun"], + "annotations": [], + "doc": "Return a remote file size." + }, + { + "name": "DocQueryAPI.HostOps.normalizeLimit", + "kind": "def", + "signature": "def normalizeLimit(value: Int): Int receives defaultLimit", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 19 }, + "visibility": "public", + "flags": ["fun"], + "annotations": [], + "doc": "Clamp a caller supplied limit." + } + ] + }, + { + "name": "DocQueryAPI.Singleton", + "kind": "class", + "signature": "object Singleton", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "visibility": "public", + "flags": ["object", "class"], + "annotations": [], + "doc": "Shared singleton helpers.", + "members": [ + { + "name": "DocQueryAPI.Singleton.", + "kind": "constructor", + "signature": "constructor(): Singleton receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "visibility": "public", + "flags": ["fun", "method", "constructor"], + "annotations": [], + "doc": null + }, + { + "name": "DocQueryAPI.Singleton.label", + "kind": "def", + "signature": "def label(): String receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 26 }, + "visibility": "public", + "flags": ["fun", "method"], + "annotations": [], + "doc": "Return the singleton label." + } + ] + }, { "name": "DocQueryAPI.FileLike", "kind": "class", "signature": "interface FileLike", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 5 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 31 }, "visibility": "public", "flags": ["interface"], "annotations": [{ "name": "jo.py.interop", "args": [] }], @@ -23,7 +117,7 @@ "name": "DocQueryAPI.FileLike.readText", "kind": "def", "signature": "def readText(path: String, limit: Int = 10): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 8 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 34 }, "visibility": "public", "flags": ["defer", "fun", "method"], "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], @@ -35,7 +129,7 @@ "name": "DocQueryAPI.Box", "kind": "class", "signature": "class Box", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 12 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, "visibility": "public", "flags": ["class"], "annotations": [], @@ -45,7 +139,7 @@ "name": "DocQueryAPI.Box.name", "kind": "field", "signature": "val name: String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 14 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 40 }, "visibility": "public", "flags": ["field"], "annotations": [], @@ -55,7 +149,7 @@ "name": "DocQueryAPI.Box.", "kind": "constructor", "signature": "constructor(name: String): Box receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 17 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 43 }, "visibility": "public", "flags": ["fun", "method", "constructor"], "annotations": [], @@ -65,7 +159,7 @@ "name": "DocQueryAPI.Box.label", "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 21 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 47 }, "visibility": "public", "flags": ["fun", "method"], "annotations": [], @@ -77,7 +171,7 @@ "name": "DocQueryAPI.helper", "kind": "def", "signature": "def helper(value: Int): Int receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 25 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, "visibility": "public", "flags": ["fun"], "annotations": [], From 34101d25f9fe1b4a7de9f25c9306367e84f81778 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 00:52:55 +0200 Subject: [PATCH 07/36] Fix signature for receives and autos Signed-off-by: Fengyun Liu --- stack-lang/doc/Compiler.scala | 19 ++++++++-- stack-lang/doc/DocModel.scala | 71 ++++++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/stack-lang/doc/Compiler.scala b/stack-lang/doc/Compiler.scala index f81db5c3..f450fa1f 100644 --- a/stack-lang/doc/Compiler.scala +++ b/stack-lang/doc/Compiler.scala @@ -65,6 +65,21 @@ object Compiler: val queryText = query.value.trim val jsonOutput = format.value == "json" || queryText.nonEmpty + def writeJson(entries: List[DocEntry]): Unit = + val out = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8)) + DocJsonEmitter.emit(entries, out) + out.flush() + + if jsonOutput && sources.isEmpty && Config.libPaths.value.isEmpty then + if outputDir.value != outputDir.default then + Reporter.error("--out is only supported with --format html") + return + + val selected = DocQuery.select(DocIndex(Nil, Nil), queryText) + if rp.hasErrors then return + writeJson(selected) + return + // Parse and type check val (units, delayedUnits) = sources |> Typer.parseStep |> Typer.typeStep @@ -83,9 +98,7 @@ object Compiler: val index = DocModel.build(units, libraryUnits, includePrivate.value) val selected = DocQuery.select(index, queryText) if rp.hasErrors then return - val out = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8)) - DocJsonEmitter.emit(selected, out) - out.flush() + writeJson(selected) return val outputPath = Paths.get(outputDir.value) diff --git a/stack-lang/doc/DocModel.scala b/stack-lang/doc/DocModel.scala index 7b09969c..6c56a3a5 100644 --- a/stack-lang/doc/DocModel.scala +++ b/stack-lang/doc/DocModel.scala @@ -79,20 +79,79 @@ object DocModel: views = views, ) - def procSignature(sym: Symbol): String = - sym.tpe.asProcType.show + def defaultValue(value: DefaultValue): String = + value match + case DefaultValue.Lit(const) => const match + case Constant.Bool(v) => v.toString + case Constant.Int(v) => v.toString + case Constant.Float(v) => v.toString + case Constant.String(v) => JsonUtil.string(v) + case DefaultValue.Ref(sym) => sym.name + + def paramSignature(sym: Symbol, default: Option[DefaultValue] = None): String = + val base = sym.name + ": " + sym.tpe.show + default match + case Some(value) => base + " = " + defaultValue(value) + case None => base + + def autoCandidateSignature(candidate: AutoCandidate): String = + candidate.show + + def autoSignature(sym: Symbol, candidates: List[AutoCandidate]): String = + val candidateText = + if candidates.isEmpty then "" + else " with [" + candidates.map(autoCandidateSignature).mkString(", ") + "]" + paramSignature(sym) + candidateText + + def receivesSignature(procType: ProcType): String = + def showEffects(effects: List[Symbol]): String = + if effects.isEmpty then " receives none" + else " receives " + effects.map(_.name).mkString(", ") + + procType.receivesInfo match + case sym: Symbol => + summon[Definitions].index.effectEngine.getKnownEffects(sym) match + case Some(effects) => showEffects(effects) + case None => "" + case effects: List[Symbol] => + showEffects(effects) + + def procSignature(fd: FunDef): String = + val procType = fd.symbol.tpe.asProcType + val tparamText = + if fd.tparams.isEmpty then "" + else fd.tparams.map(_.name).mkString("[", ", ", "]") + + val preParamText = + if procType.preParamCount == 0 then "" + else fd.params.take(procType.preParamCount).map(paramSignature(_)).mkString("(", ", ", ")") + + val postParams = fd.params.drop(procType.preParamCount) + val defaultCount = procType.defaults.size + val postParamText = + val split = postParams.size - defaultCount + val withoutDefaults = postParams.take(split).map(paramSignature(_)) + val withDefaults = postParams.drop(split).zip(procType.defaults).map: + case (param, default) => paramSignature(param, Some(default)) + (withoutDefaults ++ withDefaults).mkString("(", ", ", ")") + + val autoText = + if fd.autos.isEmpty then "" + else fd.autos.zip(fd.candidates).map(autoSignature).mkString("(auto ", ", ", ")") + + tparamText + preParamText + postParamText + autoText + ": " + fd.resultType.tpe.show + receivesSignature(procType) def funSignature(fd: FunDef): String = val sym = fd.symbol if sym.is(Flags.Annotation) then - "annotation " + sym.name + procSignature(sym).stripSuffix(": void receives none").stripSuffix(": void") + "annotation " + sym.name + procSignature(fd).stripSuffix(": void receives none").stripSuffix(": void") else if sym.is(Flags.Constructor) then - "constructor" + procSignature(sym) + "constructor" + procSignature(fd) else - "def " + sym.name + procSignature(sym) + "def " + sym.name + procSignature(fd) def patternSignature(pd: PatDef): String = - "pattern " + pd.symbol.name + procSignature(pd.symbol) + "pattern " + pd.symbol.name + pd.symbol.tpe.asProcType.show def typeSignature(td: TypeDef): String = val sym = td.symbol From a5768471ff4798b23959e32372d4b0a48b0aaef3 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 00:55:01 +0200 Subject: [PATCH 08/36] Update check files Signed-off-by: Fengyun Liu --- tests/custom/doc-query/all.json.check | 30 +++++++++++++------- tests/custom/doc-query/lib-query.json.check | 30 +++++++++++++------- tests/custom/doc-query/structural.json.check | 30 +++++++++++++------- 3 files changed, 60 insertions(+), 30 deletions(-) diff --git a/tests/custom/doc-query/all.json.check b/tests/custom/doc-query/all.json.check index 4910aa8b..dafc12aa 100644 --- a/tests/custom/doc-query/all.json.check +++ b/tests/custom/doc-query/all.json.check @@ -61,11 +61,21 @@ } ] }, + { + "name": "DocQueryAPI.describe", + "kind": "def", + "signature": "def describe[T](value: T)(auto show: Show[T] with [[T].toString]): String receives defaultLimit", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "visibility": "public", + "flags": ["fun"], + "annotations": [], + "doc": "Render a value with auto formatting and context." + }, { "name": "DocQueryAPI.Singleton", "kind": "class", "signature": "object Singleton", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 28 }, "visibility": "public", "flags": ["object", "class"], "annotations": [], @@ -75,7 +85,7 @@ "name": "DocQueryAPI.Singleton.", "kind": "constructor", "signature": "constructor(): Singleton receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 28 }, "visibility": "public", "flags": ["fun", "method", "constructor"], "annotations": [], @@ -85,7 +95,7 @@ "name": "DocQueryAPI.Singleton.label", "kind": "def", "signature": "def label(): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 26 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 30 }, "visibility": "public", "flags": ["fun", "method"], "annotations": [], @@ -97,7 +107,7 @@ "name": "DocQueryAPI.FileLike", "kind": "class", "signature": "interface FileLike", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 31 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 35 }, "visibility": "public", "flags": ["interface"], "annotations": [{ "name": "jo.py.interop", "args": [] }], @@ -107,7 +117,7 @@ "name": "DocQueryAPI.FileLike.readText", "kind": "def", "signature": "def readText(path: String, limit: Int = 10): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 34 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, "visibility": "public", "flags": ["defer", "fun", "method"], "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], @@ -119,7 +129,7 @@ "name": "DocQueryAPI.Box", "kind": "class", "signature": "class Box", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 42 }, "visibility": "public", "flags": ["class"], "annotations": [], @@ -129,7 +139,7 @@ "name": "DocQueryAPI.Box.name", "kind": "field", "signature": "val name: String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 40 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 44 }, "visibility": "public", "flags": ["field"], "annotations": [], @@ -139,7 +149,7 @@ "name": "DocQueryAPI.Box.", "kind": "constructor", "signature": "constructor(name: String): Box receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 43 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 47 }, "visibility": "public", "flags": ["fun", "method", "constructor"], "annotations": [], @@ -149,7 +159,7 @@ "name": "DocQueryAPI.Box.label", "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 47 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, "visibility": "public", "flags": ["fun", "method"], "annotations": [], @@ -161,7 +171,7 @@ "name": "DocQueryAPI.helper", "kind": "def", "signature": "def helper(value: Int): Int receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 55 }, "visibility": "public", "flags": ["fun"], "annotations": [], diff --git a/tests/custom/doc-query/lib-query.json.check b/tests/custom/doc-query/lib-query.json.check index f9e3f6ee..6d3497e9 100644 --- a/tests/custom/doc-query/lib-query.json.check +++ b/tests/custom/doc-query/lib-query.json.check @@ -71,11 +71,21 @@ } ] }, + { + "name": "DocQueryAPI.describe", + "kind": "def", + "signature": "def describe[T](value: T)(auto show: Show[T] with [[T].toString]): String receives defaultLimit", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "visibility": "public", + "flags": ["loaded", "fun"], + "annotations": [], + "doc": "Render a value with auto formatting and context." + }, { "name": "DocQueryAPI.Singleton", "kind": "class", "signature": "object Singleton", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 28 }, "visibility": "public", "flags": ["object", "class"], "annotations": [], @@ -85,7 +95,7 @@ "name": "DocQueryAPI.Singleton.", "kind": "constructor", "signature": "constructor(): Singleton receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 28 }, "visibility": "public", "flags": ["loaded", "fun", "method", "constructor"], "annotations": [], @@ -95,7 +105,7 @@ "name": "DocQueryAPI.Singleton.label", "kind": "def", "signature": "def label(): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 26 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 30 }, "visibility": "public", "flags": ["loaded", "fun", "method"], "annotations": [], @@ -107,7 +117,7 @@ "name": "DocQueryAPI.FileLike", "kind": "class", "signature": "interface FileLike", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 31 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 35 }, "visibility": "public", "flags": ["interface"], "annotations": [{ "name": "jo.py.interop", "args": [] }], @@ -117,7 +127,7 @@ "name": "DocQueryAPI.FileLike.readText", "kind": "def", "signature": "def readText(path: String, limit: Int = 10): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 34 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, "visibility": "public", "flags": ["defer", "loaded", "fun", "method"], "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], @@ -129,7 +139,7 @@ "name": "DocQueryAPI.Box", "kind": "class", "signature": "class Box", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 42 }, "visibility": "public", "flags": ["class"], "annotations": [], @@ -139,7 +149,7 @@ "name": "DocQueryAPI.Box.name", "kind": "field", "signature": "val name: String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 40 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 44 }, "visibility": "public", "flags": ["field"], "annotations": [], @@ -149,7 +159,7 @@ "name": "DocQueryAPI.Box.", "kind": "constructor", "signature": "constructor(name: String): Box receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 43 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 47 }, "visibility": "public", "flags": ["loaded", "fun", "method", "constructor"], "annotations": [], @@ -159,7 +169,7 @@ "name": "DocQueryAPI.Box.label", "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 47 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, "visibility": "public", "flags": ["loaded", "fun", "method"], "annotations": [], @@ -171,7 +181,7 @@ "name": "DocQueryAPI.helper", "kind": "def", "signature": "def helper(value: Int): Int receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 55 }, "visibility": "public", "flags": ["loaded", "fun"], "annotations": [], diff --git a/tests/custom/doc-query/structural.json.check b/tests/custom/doc-query/structural.json.check index 30b5b2cd..23b9fd49 100644 --- a/tests/custom/doc-query/structural.json.check +++ b/tests/custom/doc-query/structural.json.check @@ -71,11 +71,21 @@ } ] }, + { + "name": "DocQueryAPI.describe", + "kind": "def", + "signature": "def describe[T](value: T)(auto show: Show[T] with [[T].toString]): String receives defaultLimit", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "visibility": "public", + "flags": ["fun"], + "annotations": [], + "doc": "Render a value with auto formatting and context." + }, { "name": "DocQueryAPI.Singleton", "kind": "class", "signature": "object Singleton", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 28 }, "visibility": "public", "flags": ["object", "class"], "annotations": [], @@ -85,7 +95,7 @@ "name": "DocQueryAPI.Singleton.", "kind": "constructor", "signature": "constructor(): Singleton receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 28 }, "visibility": "public", "flags": ["fun", "method", "constructor"], "annotations": [], @@ -95,7 +105,7 @@ "name": "DocQueryAPI.Singleton.label", "kind": "def", "signature": "def label(): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 26 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 30 }, "visibility": "public", "flags": ["fun", "method"], "annotations": [], @@ -107,7 +117,7 @@ "name": "DocQueryAPI.FileLike", "kind": "class", "signature": "interface FileLike", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 31 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 35 }, "visibility": "public", "flags": ["interface"], "annotations": [{ "name": "jo.py.interop", "args": [] }], @@ -117,7 +127,7 @@ "name": "DocQueryAPI.FileLike.readText", "kind": "def", "signature": "def readText(path: String, limit: Int = 10): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 34 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, "visibility": "public", "flags": ["defer", "fun", "method"], "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], @@ -129,7 +139,7 @@ "name": "DocQueryAPI.Box", "kind": "class", "signature": "class Box", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 42 }, "visibility": "public", "flags": ["class"], "annotations": [], @@ -139,7 +149,7 @@ "name": "DocQueryAPI.Box.name", "kind": "field", "signature": "val name: String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 40 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 44 }, "visibility": "public", "flags": ["field"], "annotations": [], @@ -149,7 +159,7 @@ "name": "DocQueryAPI.Box.", "kind": "constructor", "signature": "constructor(name: String): Box receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 43 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 47 }, "visibility": "public", "flags": ["fun", "method", "constructor"], "annotations": [], @@ -159,7 +169,7 @@ "name": "DocQueryAPI.Box.label", "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 47 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, "visibility": "public", "flags": ["fun", "method"], "annotations": [], @@ -171,7 +181,7 @@ "name": "DocQueryAPI.helper", "kind": "def", "signature": "def helper(value: Int): Int receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 55 }, "visibility": "public", "flags": ["fun"], "annotations": [], From 11ed5cd329350baf438558fcf73b3e14d3457a4a Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 00:55:33 +0200 Subject: [PATCH 09/36] Add more tests Signed-off-by: Fengyun Liu --- tests/custom/doc-query/describe.json.check | 12 ++++++++++++ tests/custom/doc-query/exact-member.json.check | 12 ++++++++++++ tests/custom/doc-query/private.json.check | 12 ++++++++++++ tests/custom/doc-query/test.sh | 13 +++++-------- 4 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 tests/custom/doc-query/describe.json.check create mode 100644 tests/custom/doc-query/exact-member.json.check create mode 100644 tests/custom/doc-query/private.json.check diff --git a/tests/custom/doc-query/describe.json.check b/tests/custom/doc-query/describe.json.check new file mode 100644 index 00000000..0c3c8e20 --- /dev/null +++ b/tests/custom/doc-query/describe.json.check @@ -0,0 +1,12 @@ +[ + { + "name": "DocQueryAPI.describe", + "kind": "def", + "signature": "def describe[T](value: T)(auto show: Show[T] with [[T].toString]): String receives defaultLimit", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "visibility": "public", + "flags": ["fun"], + "annotations": [], + "doc": "Render a value with auto formatting and context." + } +] diff --git a/tests/custom/doc-query/exact-member.json.check b/tests/custom/doc-query/exact-member.json.check new file mode 100644 index 00000000..369b7dcf --- /dev/null +++ b/tests/custom/doc-query/exact-member.json.check @@ -0,0 +1,12 @@ +[ + { + "name": "DocQueryAPI.Box.label", + "kind": "def", + "signature": "def label(prefix: String = \"box\"): String receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, + "visibility": "public", + "flags": ["fun", "method"], + "annotations": [], + "doc": "Render a label." + } +] diff --git a/tests/custom/doc-query/private.json.check b/tests/custom/doc-query/private.json.check new file mode 100644 index 00000000..2607bee3 --- /dev/null +++ b/tests/custom/doc-query/private.json.check @@ -0,0 +1,12 @@ +[ + { + "name": "DocQueryAPI.hidden", + "kind": "def", + "signature": "def hidden(): Int receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 58 }, + "visibility": "private", + "flags": ["fun"], + "annotations": [], + "doc": "Hidden helper." + } +] diff --git a/tests/custom/doc-query/test.sh b/tests/custom/doc-query/test.sh index 9d38a803..504d9483 100755 --- a/tests/custom/doc-query/test.sh +++ b/tests/custom/doc-query/test.sh @@ -60,11 +60,11 @@ run_json_doc "$API_FILE" > "$DIR/all.json" run_json_doc --query "DocQueryAPI.*,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/structural.json" run_json_doc --query "file:$API_FILE" "$API_FILE" > "$DIR/file.json" run_json_doc --query "file:$PROJECT_ROOT/$API_FILE" "$API_FILE" > "$DIR/file-absolute.json" +run_json_doc --query "DocQueryAPI.describe" "$API_FILE" > "$DIR/describe.json" run_json_doc --query "DocQueryAPI.Box.label" "$API_FILE" > "$DIR/exact-member.json" run_json_doc --include-private --query "DocQueryAPI.hidden" "$API_FILE" > "$DIR/private.json" run_doc --query "DocQueryAPI.*,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/query-implies-json.json" run_doc --lib "$SAST_DIR" --query "DocQueryAPI.*" > "$DIR/lib-query.json" -run_plain_doc --query "jo.List" > "$DIR/stdlib-list.json" diff -u "$DIR/all.json.check" "$DIR/all.json" diff -u "$DIR/structural.json.check" "$DIR/structural.json" @@ -72,12 +72,9 @@ diff -u "$DIR/structural.json.check" "$DIR/file.json" diff -u "$DIR/structural.json.check" "$DIR/file-absolute.json" diff -u "$DIR/structural.json.check" "$DIR/query-implies-json.json" diff -u "$DIR/lib-query.json.check" "$DIR/lib-query.json" - -python3 "$DIR/assert_json.py" source-all "$DIR/all.json" -python3 "$DIR/assert_json.py" structural "$DIR/structural.json" -python3 "$DIR/assert_json.py" exact-member "$DIR/exact-member.json" -python3 "$DIR/assert_json.py" private "$DIR/private.json" -python3 "$DIR/assert_json.py" stdlib-list "$DIR/stdlib-list.json" +diff -u "$DIR/describe.json.check" "$DIR/describe.json" +diff -u "$DIR/exact-member.json.check" "$DIR/exact-member.json" +diff -u "$DIR/private.json.check" "$DIR/private.json" expect_fail "$FAIL_LOG" "$PROJECT_ROOT/bin/jo" compile --doc --format yaml "$API_FILE" rg -q "Option --format must be one of: html, json" "$FAIL_LOG" @@ -88,7 +85,7 @@ rg -q -- "--out is only supported with --format html" "$FAIL_LOG" expect_fail "$FAIL_LOG" run_plain_doc --query "NoSuchSymbol" rg -q "No documentation entries match symbol selector" "$FAIL_LOG" -expect_fail "$FAIL_LOG" run_plain_doc --no-stdlib --query "jo.List" +expect_fail "$FAIL_LOG" run_plain_doc --no-stdlib --query "NoSuchSymbol" rg -q "No documentation entries match symbol selector" "$FAIL_LOG" expect_fail "$FAIL_LOG" run_plain_doc --format json From ec51ab464121394c0b7d1e5cfd01fde0852bccc2 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 07:24:35 +0200 Subject: [PATCH 10/36] Do query on sast Signed-off-by: Fengyun Liu --- stack-lang/doc/Compiler.scala | 14 ++--- stack-lang/doc/DocQuery.scala | 112 ++++++++++++++++++++++++++++------ 2 files changed, 102 insertions(+), 24 deletions(-) diff --git a/stack-lang/doc/Compiler.scala b/stack-lang/doc/Compiler.scala index f450fa1f..f2099ba3 100644 --- a/stack-lang/doc/Compiler.scala +++ b/stack-lang/doc/Compiler.scala @@ -37,9 +37,7 @@ object Compiler: given Config = config - val queryText = query.value.trim - - if sources.isEmpty && queryText.isEmpty then + if sources.isEmpty && query.value.trim.isEmpty then println("Usage: jo doc [options]") println() println("Options:") @@ -52,7 +50,7 @@ object Compiler: println() println("Examples:") println(" jo doc lib/Core.jo lib/List.jo --out site/api") - println(" jo doc src/main.jo --out docs --title MyProject") + println(" jo doc --query jo.List") System.exit(1) Reporter.monitor(): @@ -63,6 +61,7 @@ object Compiler: val rootNameTable = new NameTable given lazyDefn: Definitions.Lazy = Definitions.Lazy(rootNameTable) val queryText = query.value.trim + val selectors = DocQuery.parse(queryText) val jsonOutput = format.value == "json" || queryText.nonEmpty def writeJson(entries: List[DocEntry]): Unit = @@ -75,7 +74,7 @@ object Compiler: Reporter.error("--out is only supported with --format html") return - val selected = DocQuery.select(DocIndex(Nil, Nil), queryText) + val selected = DocQuery.select(DocIndex(Nil, Nil), selectors) if rp.hasErrors then return writeJson(selected) return @@ -92,11 +91,12 @@ object Compiler: Reporter.error("--out is only supported with --format html") return + if queryText.nonEmpty then DocQuery.forceSymbols(selectors) val libraryUnits = - if queryText.nonEmpty then delayedUnits.forceAll() + if queryText.nonEmpty then delayedUnits.force() else Nil val index = DocModel.build(units, libraryUnits, includePrivate.value) - val selected = DocQuery.select(index, queryText) + val selected = DocQuery.select(index, selectors) if rp.hasErrors then return writeJson(selected) return diff --git a/stack-lang/doc/DocQuery.scala b/stack-lang/doc/DocQuery.scala index 2eeb885b..5e55b117 100644 --- a/stack-lang/doc/DocQuery.scala +++ b/stack-lang/doc/DocQuery.scala @@ -1,37 +1,60 @@ package doc import reporting.Reporter -import sast.Symbols.Symbol +import sast.* +import sast.Denotations.* +import sast.Symbols.* import java.nio.file.Paths import scala.collection.mutable object DocQuery: + enum Selector: + case SymbolSelector(raw: String, name: String) + case FileSelector(raw: String, file: String) + def select(index: DocIndex, rawQuery: String)(using Reporter): List[DocEntry] = - val selectors = parseSelectors(rawQuery) + select(index, parse(rawQuery)) + + def select(index: DocIndex, selectors: List[Selector])(using Reporter): List[DocEntry] = val selected = if selectors.isEmpty then index.sourceRoots else selectors.flatMap(resolveSelector(index, _)) DocModel.sortEntries(pruneDescendants(DocModel.distinctEntries(selected))) - private def parseSelectors(rawQuery: String): List[String] = - rawQuery.split(",").map(_.trim).filter(_.nonEmpty).toList + def parse(rawQuery: String): List[Selector] = + rawQuery.split(",").map(_.trim).filter(_.nonEmpty).map { selector => + if selector.startsWith("file:") then + Selector.FileSelector(selector, selector.stripPrefix("file:")) + else + val name = + if selector.endsWith(".*") then selector.stripSuffix(".*") + else selector + Selector.SymbolSelector(selector, name) + }.toList + + def forceSymbols(selectors: List[Selector])(using Definitions): Unit = + val seen = mutable.HashSet.empty[Symbol] + for selector <- selectors do + selector match + case Selector.SymbolSelector(_, name) => + resolveSastSymbols(name).foreach(forceSymbol(_, seen)) + + case Selector.FileSelector(_, _) => + () - private def resolveSelector(index: DocIndex, selector: String)(using Reporter): List[DocEntry] = - if selector.startsWith("file:") then - val file = selector.stripPrefix("file:") - val matches = index.entries.filter(entry => matchesFile(entry, file)) - if matches.isEmpty then Reporter.error(s"No documentation entries match file selector `$selector`") - matches - else - val symbolSelector = - if selector.endsWith(".*") then selector.stripSuffix(".*") - else selector + private def resolveSelector(index: DocIndex, selector: Selector)(using Reporter): List[DocEntry] = + selector match + case Selector.FileSelector(raw, file) => + val matches = index.entries.filter(entry => matchesFile(entry, file)) + if matches.isEmpty then Reporter.error(s"No documentation entries match file selector `$raw`") + matches - val matches = resolveSymbol(index, symbolSelector) - if matches.isEmpty then Reporter.error(s"No documentation entries match symbol selector `$selector`") - matches + case Selector.SymbolSelector(raw, name) => + val matches = resolveSymbol(index, name) + if matches.isEmpty then Reporter.error(s"No documentation entries match symbol selector `$raw`") + matches private def resolveSymbol(index: DocIndex, selector: String): List[DocEntry] = val exactNames = @@ -43,6 +66,61 @@ object DocQuery: else if selector.contains(".") then Nil else index.entries.filter(_.symbol.name == selector) + private def resolveSastSymbols(selector: String)(using defn: Definitions): List[Symbol] = + val paths = + if selector.contains(".") then List(selector) + else List(selector, "jo." + selector) + + distinctSymbols(paths.flatMap(path => resolvePath(path.split('.').filter(_.nonEmpty).toList))) + + private def resolvePath(parts: List[String])(using defn: Definitions): List[Symbol] = + def resolveInNameTable(table: NameTable, name: String): List[Symbol] = + table.resolve(name) ++ table.resolveAnnotation(name) + + def resolveFromSymbol(sym: Symbol, rest: List[String]): List[Symbol] = + rest match + case Nil => List(sym) + case name :: tail => + val members = + if sym.isContainer then + resolveInNameTable(sym.nameTable, name) + else + sym.info match + case classInfo: ClassInfo => classInfo.getMemberSymbol(name).toList + case _ => Nil + members.flatMap(resolveFromSymbol(_, tail)) + + parts match + case Nil => Nil + case name :: tail => + resolveInNameTable(defn.rootNameTable, name).flatMap(resolveFromSymbol(_, tail)) + + private def forceSymbol(sym: Symbol, seen: mutable.HashSet[Symbol])(using Definitions): Unit = + if !seen.contains(sym) then + seen += sym + if sym.isContainer then + sym.annotations + forceNameTable(sym.nameTable, seen) + else + sym.info + sym.annotations + if sym.isClass || sym.isInterface then + val classInfo = sym.classInfo + classInfo.fields.foreach(forceSymbol(_, seen)) + classInfo.methods.foreach(forceSymbol(_, seen)) + + private def forceNameTable(table: NameTable, seen: mutable.HashSet[Symbol])(using Definitions): Unit = + val symbols = table.terms ++ table.types ++ table.patterns ++ table.containers + symbols.foreach(forceSymbol(_, seen)) + + private def distinctSymbols(symbols: List[Symbol]): List[Symbol] = + val seen = mutable.HashSet.empty[Symbol] + symbols.filter: sym => + if seen.contains(sym) then false + else + seen += sym + true + private def matchesFile(entry: DocEntry, rawFile: String): Boolean = entry.source.exists: source => val rawNormalized = normalize(rawFile) From 93395a1dacaf5f90feed4de8ab74b7125750e0d1 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 07:25:12 +0200 Subject: [PATCH 11/36] Add exact member query test Signed-off-by: Fengyun Liu --- tests/custom/doc-query/lib-exact-member.json.check | 12 ++++++++++++ tests/custom/doc-query/test.sh | 2 ++ 2 files changed, 14 insertions(+) create mode 100644 tests/custom/doc-query/lib-exact-member.json.check diff --git a/tests/custom/doc-query/lib-exact-member.json.check b/tests/custom/doc-query/lib-exact-member.json.check new file mode 100644 index 00000000..a4204934 --- /dev/null +++ b/tests/custom/doc-query/lib-exact-member.json.check @@ -0,0 +1,12 @@ +[ + { + "name": "DocQueryAPI.Box.label", + "kind": "def", + "signature": "def label(prefix: String = \"box\"): String receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, + "visibility": "public", + "flags": ["loaded", "fun", "method"], + "annotations": [], + "doc": "Render a label." + } +] diff --git a/tests/custom/doc-query/test.sh b/tests/custom/doc-query/test.sh index 504d9483..9a473d73 100755 --- a/tests/custom/doc-query/test.sh +++ b/tests/custom/doc-query/test.sh @@ -65,6 +65,7 @@ run_json_doc --query "DocQueryAPI.Box.label" "$API_FILE" > "$DIR/exact-member.js run_json_doc --include-private --query "DocQueryAPI.hidden" "$API_FILE" > "$DIR/private.json" run_doc --query "DocQueryAPI.*,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/query-implies-json.json" run_doc --lib "$SAST_DIR" --query "DocQueryAPI.*" > "$DIR/lib-query.json" +run_doc --lib "$SAST_DIR" --query "DocQueryAPI.Box.label" > "$DIR/lib-exact-member.json" diff -u "$DIR/all.json.check" "$DIR/all.json" diff -u "$DIR/structural.json.check" "$DIR/structural.json" @@ -72,6 +73,7 @@ diff -u "$DIR/structural.json.check" "$DIR/file.json" diff -u "$DIR/structural.json.check" "$DIR/file-absolute.json" diff -u "$DIR/structural.json.check" "$DIR/query-implies-json.json" diff -u "$DIR/lib-query.json.check" "$DIR/lib-query.json" +diff -u "$DIR/lib-exact-member.json.check" "$DIR/lib-exact-member.json" diff -u "$DIR/describe.json.check" "$DIR/describe.json" diff -u "$DIR/exact-member.json.check" "$DIR/exact-member.json" diff -u "$DIR/private.json.check" "$DIR/private.json" From 5e8a31e02002ad9783db793450ffcf3e39a21e3b Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 07:59:52 +0200 Subject: [PATCH 12/36] Use grep instead rg in test Signed-off-by: Fengyun Liu --- tests/custom/doc-query/test.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/custom/doc-query/test.sh b/tests/custom/doc-query/test.sh index 9a473d73..2aa7b90f 100755 --- a/tests/custom/doc-query/test.sh +++ b/tests/custom/doc-query/test.sh @@ -79,18 +79,18 @@ diff -u "$DIR/exact-member.json.check" "$DIR/exact-member.json" diff -u "$DIR/private.json.check" "$DIR/private.json" expect_fail "$FAIL_LOG" "$PROJECT_ROOT/bin/jo" compile --doc --format yaml "$API_FILE" -rg -q "Option --format must be one of: html, json" "$FAIL_LOG" +grep -q -- "Option --format must be one of: html, json" "$FAIL_LOG" expect_fail "$FAIL_LOG" run_json_doc --out "$SAST_DIR/out" "$API_FILE" -rg -q -- "--out is only supported with --format html" "$FAIL_LOG" +grep -q -- "--out is only supported with --format html" "$FAIL_LOG" expect_fail "$FAIL_LOG" run_plain_doc --query "NoSuchSymbol" -rg -q "No documentation entries match symbol selector" "$FAIL_LOG" +grep -q -- "No documentation entries match symbol selector" "$FAIL_LOG" expect_fail "$FAIL_LOG" run_plain_doc --no-stdlib --query "NoSuchSymbol" -rg -q "No documentation entries match symbol selector" "$FAIL_LOG" +grep -q -- "No documentation entries match symbol selector" "$FAIL_LOG" expect_fail "$FAIL_LOG" run_plain_doc --format json -rg -q "Usage: jo doc" "$FAIL_LOG" +grep -q -- "Usage: jo doc" "$FAIL_LOG" echo " ✓ All tests passed for $TEST_NAME" From 117cf3275488c69aafe9365846e3c18edecfd3da Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 08:26:18 +0200 Subject: [PATCH 13/36] Simplify symbol query Signed-off-by: Fengyun Liu --- stack-lang/doc/Compiler.scala | 26 +++++++-- stack-lang/doc/DocQuery.scala | 87 ++++--------------------------- stack-lang/sast/Definitions.scala | 16 ++++++ 3 files changed, 47 insertions(+), 82 deletions(-) diff --git a/stack-lang/doc/Compiler.scala b/stack-lang/doc/Compiler.scala index f2099ba3..d9c33cd3 100644 --- a/stack-lang/doc/Compiler.scala +++ b/stack-lang/doc/Compiler.scala @@ -1,11 +1,14 @@ package doc import sast.* +import sast.Trees.FileUnit import typing.Typer import reporting.Reporter import reporting.Config +import DocQuery.Selector + import java.io.{FileOutputStream, OutputStreamWriter, PrintWriter} import java.nio.file.{Files, Path, Paths} import java.nio.charset.StandardCharsets @@ -84,23 +87,36 @@ object Compiler: if rp.hasErrors then return - given Definitions = lazyDefn.value + given defn: Definitions = lazyDefn.value if jsonOutput then if outputDir.value != outputDir.default then Reporter.error("--out is only supported with --format html") return - if queryText.nonEmpty then DocQuery.forceSymbols(selectors) + val querySymbols = selectors.flatMap: + case Selector.SymbolSelector(_, path) => defn.resolve(path) + case _: Selector.FileSelector => Nil + val libraryUnits = - if queryText.nonEmpty then delayedUnits.force() - else Nil + if queryText.isEmpty then + Nil + else + delayedUnits.forceIf: unit => + querySymbols.exists: querySymbol => + unit.owner.containedIn(querySymbol) || querySymbol.containedIn(unit.owner) + + delayedUnits.force() + val index = DocModel.build(units, libraryUnits, includePrivate.value) val selected = DocQuery.select(index, selectors) if rp.hasErrors then return writeJson(selected) - return + else + generateHtmlDoc(units) + + def generateHtmlDoc(units: List[FileUnit])(using Config, Definitions): Unit = val outputPath = Paths.get(outputDir.value) val includePrivateVal = includePrivate.value diff --git a/stack-lang/doc/DocQuery.scala b/stack-lang/doc/DocQuery.scala index 5e55b117..24fbbc7b 100644 --- a/stack-lang/doc/DocQuery.scala +++ b/stack-lang/doc/DocQuery.scala @@ -1,8 +1,6 @@ package doc import reporting.Reporter -import sast.* -import sast.Denotations.* import sast.Symbols.* import java.nio.file.Paths @@ -21,28 +19,7 @@ object DocQuery: if selectors.isEmpty then index.sourceRoots else selectors.flatMap(resolveSelector(index, _)) - DocModel.sortEntries(pruneDescendants(DocModel.distinctEntries(selected))) - - def parse(rawQuery: String): List[Selector] = - rawQuery.split(",").map(_.trim).filter(_.nonEmpty).map { selector => - if selector.startsWith("file:") then - Selector.FileSelector(selector, selector.stripPrefix("file:")) - else - val name = - if selector.endsWith(".*") then selector.stripSuffix(".*") - else selector - Selector.SymbolSelector(selector, name) - }.toList - - def forceSymbols(selectors: List[Selector])(using Definitions): Unit = - val seen = mutable.HashSet.empty[Symbol] - for selector <- selectors do - selector match - case Selector.SymbolSelector(_, name) => - resolveSastSymbols(name).foreach(forceSymbol(_, seen)) - - case Selector.FileSelector(_, _) => - () + DocModel.sortEntries(pruneDescendants(selected).distinct) private def resolveSelector(index: DocIndex, selector: Selector)(using Reporter): List[DocEntry] = selector match @@ -66,60 +43,16 @@ object DocQuery: else if selector.contains(".") then Nil else index.entries.filter(_.symbol.name == selector) - private def resolveSastSymbols(selector: String)(using defn: Definitions): List[Symbol] = - val paths = - if selector.contains(".") then List(selector) - else List(selector, "jo." + selector) - - distinctSymbols(paths.flatMap(path => resolvePath(path.split('.').filter(_.nonEmpty).toList))) - - private def resolvePath(parts: List[String])(using defn: Definitions): List[Symbol] = - def resolveInNameTable(table: NameTable, name: String): List[Symbol] = - table.resolve(name) ++ table.resolveAnnotation(name) - - def resolveFromSymbol(sym: Symbol, rest: List[String]): List[Symbol] = - rest match - case Nil => List(sym) - case name :: tail => - val members = - if sym.isContainer then - resolveInNameTable(sym.nameTable, name) - else - sym.info match - case classInfo: ClassInfo => classInfo.getMemberSymbol(name).toList - case _ => Nil - members.flatMap(resolveFromSymbol(_, tail)) - - parts match - case Nil => Nil - case name :: tail => - resolveInNameTable(defn.rootNameTable, name).flatMap(resolveFromSymbol(_, tail)) - - private def forceSymbol(sym: Symbol, seen: mutable.HashSet[Symbol])(using Definitions): Unit = - if !seen.contains(sym) then - seen += sym - if sym.isContainer then - sym.annotations - forceNameTable(sym.nameTable, seen) - else - sym.info - sym.annotations - if sym.isClass || sym.isInterface then - val classInfo = sym.classInfo - classInfo.fields.foreach(forceSymbol(_, seen)) - classInfo.methods.foreach(forceSymbol(_, seen)) - - private def forceNameTable(table: NameTable, seen: mutable.HashSet[Symbol])(using Definitions): Unit = - val symbols = table.terms ++ table.types ++ table.patterns ++ table.containers - symbols.foreach(forceSymbol(_, seen)) - - private def distinctSymbols(symbols: List[Symbol]): List[Symbol] = - val seen = mutable.HashSet.empty[Symbol] - symbols.filter: sym => - if seen.contains(sym) then false + def parse(rawQuery: String): List[Selector] = + rawQuery.split(",").map(_.trim).filter(_.nonEmpty).map { selector => + if selector.startsWith("file:") then + Selector.FileSelector(selector, selector.stripPrefix("file:")) else - seen += sym - true + val name = + if selector.endsWith(".*") then selector.stripSuffix(".*") + else selector + Selector.SymbolSelector(selector, name) + }.toList private def matchesFile(entry: DocEntry, rawFile: String): Boolean = entry.source.exists: source => diff --git a/stack-lang/sast/Definitions.scala b/stack-lang/sast/Definitions.scala index 6808e986..01ae6814 100644 --- a/stack-lang/sast/Definitions.scala +++ b/stack-lang/sast/Definitions.scala @@ -21,6 +21,8 @@ final class Definitions(private var _index: SymbolIndex) extends Definitions.Laz //---------------------------------------------------------------------------- // Name lookup + def resolve(path: String): List[Symbol] = Definitions.resolveStatic(rootNameTable, path.split('.').toList) + def resolveTerm(path: String): Symbol = resolveStatic(path.split('.').toList, SymbolKind.Term) def resolveType(path: String): Symbol = resolveStatic(path.split('.').toList, SymbolKind.Type) def resolveContainer(path: String): Symbol = resolveStatic(path.split('.').toList, SymbolKind.Container) @@ -181,3 +183,17 @@ object Definitions: case None => None + + def resolveStatic(nameTable: NameTable, parts: List[String]): List[Symbol] = + (parts: @unchecked) match + case name :: Nil => + nameTable.resolve(name) + + case name :: rest => + nameTable.resolveContainer(name) match + case Some(sym) => + val nameTable = sym.nameTable + resolveStatic(nameTable, rest) + + case None => + Nil From f2f2f6f03d6d0e98676d24ba31cb46c1cac4384a Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 08:46:57 +0200 Subject: [PATCH 14/36] Tolerate .* syntax but don't advertise it Signed-off-by: Fengyun Liu --- docs/usage/commands/compile.md | 6 +++--- docs/usage/reference/compiler-options.md | 3 +-- stack-lang/doc/Compiler.scala | 4 ++-- stack-lang/doc/DocQuery.scala | 8 ++++---- tests/custom/doc-query/test.sh | 4 ++-- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/docs/usage/commands/compile.md b/docs/usage/commands/compile.md index 1d15de70..da1f30a3 100644 --- a/docs/usage/commands/compile.md +++ b/docs/usage/commands/compile.md @@ -46,7 +46,7 @@ Experimental. |------|-------------| | `--doc` | Generate documentation instead of normal compile output | | `--format html|json` | Documentation output format. Default: `html` | -| `--query ` | Comma-separated JSON selectors, such as `MyAPI.*` or `file:src/API.jo`; implies `--format json` | +| `--query ` | Comma-separated JSON selectors, such as `MyAPI` or `file:src/API.jo`; implies `--format json` | | `--out ` | Documentation output directory | | `--title ` | Documentation title | | `--readme ` | Markdown file to use as the generated documentation home page | @@ -57,7 +57,7 @@ Experimental. `--format json`. JSON output does not use `--out`. Without `--query`, JSON output contains the public API surface from the positional source files. With `--query`, selectors are comma-separated and may -name symbols, wildcard descendants with `.*`, or source files with `file:`. +name symbols or source files with `file:`. Symbol selectors are resolved from the language default scope; for example `~` is just an ordinary symbol query. A query may omit positional source files; in that case it searches the loaded SAST libraries, including stdlib unless @@ -126,7 +126,7 @@ jo compile --doc --format json src/API.jo Emit machine-readable docs for selected symbols: ```sh -jo compile --doc --query 'MyAPI.*,jo.py.*' src/API.jo +jo compile --doc --query 'MyAPI,jo.py' src/API.jo ``` Query stdlib docs from SAST files only: diff --git a/docs/usage/reference/compiler-options.md b/docs/usage/reference/compiler-options.md index eb913451..3eb0e8ce 100644 --- a/docs/usage/reference/compiler-options.md +++ b/docs/usage/reference/compiler-options.md @@ -109,8 +109,7 @@ files. `--query` accepts comma-separated selectors: | Selector | Meaning | |----------|---------| -| `` | Exact symbol resolved from the language default scope. | -| `.*` | The symbol and its recursive members, emitted structurally. | +| `` | Symbol resolved from the language default scope. Aggregate symbols are emitted structurally with their members. | | `file:` | Queryable symbols whose source file matches the path. | When `--query` is present, positional source files are optional. With no source diff --git a/stack-lang/doc/Compiler.scala b/stack-lang/doc/Compiler.scala index d9c33cd3..60f17c33 100644 --- a/stack-lang/doc/Compiler.scala +++ b/stack-lang/doc/Compiler.scala @@ -47,7 +47,7 @@ object Compiler: println(" --out Output directory (default: docs)") println(" --title Project title for documentation") println(" --format Output format (default: html)") - println(" --query JSON selectors, e.g. jo.py.*,file:src/API.jo") + println(" --query JSON selectors, e.g. jo.py,file:src/API.jo") println(" --include-private Include private symbols") println(" --include-source Embed source code in output") println() @@ -95,7 +95,7 @@ object Compiler: return val querySymbols = selectors.flatMap: - case Selector.SymbolSelector(_, path) => defn.resolve(path) + case Selector.SymbolSelector(path) => defn.resolve(path) case _: Selector.FileSelector => Nil val libraryUnits = diff --git a/stack-lang/doc/DocQuery.scala b/stack-lang/doc/DocQuery.scala index 24fbbc7b..8a8be454 100644 --- a/stack-lang/doc/DocQuery.scala +++ b/stack-lang/doc/DocQuery.scala @@ -8,7 +8,7 @@ import scala.collection.mutable object DocQuery: enum Selector: - case SymbolSelector(raw: String, name: String) + case SymbolSelector(name: String) case FileSelector(raw: String, file: String) def select(index: DocIndex, rawQuery: String)(using Reporter): List[DocEntry] = @@ -28,9 +28,9 @@ object DocQuery: if matches.isEmpty then Reporter.error(s"No documentation entries match file selector `$raw`") matches - case Selector.SymbolSelector(raw, name) => + case Selector.SymbolSelector(name) => val matches = resolveSymbol(index, name) - if matches.isEmpty then Reporter.error(s"No documentation entries match symbol selector `$raw`") + if matches.isEmpty then Reporter.error(s"No documentation entries match symbol selector `$name`") matches private def resolveSymbol(index: DocIndex, selector: String): List[DocEntry] = @@ -51,7 +51,7 @@ object DocQuery: val name = if selector.endsWith(".*") then selector.stripSuffix(".*") else selector - Selector.SymbolSelector(selector, name) + Selector.SymbolSelector(name) }.toList private def matchesFile(entry: DocEntry, rawFile: String): Boolean = diff --git a/tests/custom/doc-query/test.sh b/tests/custom/doc-query/test.sh index 2aa7b90f..e9a01e3c 100755 --- a/tests/custom/doc-query/test.sh +++ b/tests/custom/doc-query/test.sh @@ -63,8 +63,8 @@ run_json_doc --query "file:$PROJECT_ROOT/$API_FILE" "$API_FILE" > "$DIR/file-abs run_json_doc --query "DocQueryAPI.describe" "$API_FILE" > "$DIR/describe.json" run_json_doc --query "DocQueryAPI.Box.label" "$API_FILE" > "$DIR/exact-member.json" run_json_doc --include-private --query "DocQueryAPI.hidden" "$API_FILE" > "$DIR/private.json" -run_doc --query "DocQueryAPI.*,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/query-implies-json.json" -run_doc --lib "$SAST_DIR" --query "DocQueryAPI.*" > "$DIR/lib-query.json" +run_doc --query "DocQueryAPI,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/query-implies-json.json" +run_doc --lib "$SAST_DIR" --query "DocQueryAPI" > "$DIR/lib-query.json" run_doc --lib "$SAST_DIR" --query "DocQueryAPI.Box.label" > "$DIR/lib-exact-member.json" diff -u "$DIR/all.json.check" "$DIR/all.json" From 6acb25cf7036ede8ea2776c55e5dceeba9a93bd9 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 09:08:46 +0200 Subject: [PATCH 15/36] Tolerate class member selection Signed-off-by: Fengyun Liu --- stack-lang/doc/Compiler.scala | 7 +++++-- stack-lang/doc/DocQuery.scala | 25 +++++++++++++++++++++++++ stack-lang/sast/Definitions.scala | 16 ---------------- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/stack-lang/doc/Compiler.scala b/stack-lang/doc/Compiler.scala index 60f17c33..83022307 100644 --- a/stack-lang/doc/Compiler.scala +++ b/stack-lang/doc/Compiler.scala @@ -95,8 +95,11 @@ object Compiler: return val querySymbols = selectors.flatMap: - case Selector.SymbolSelector(path) => defn.resolve(path) - case _: Selector.FileSelector => Nil + case Selector.SymbolSelector(path) => + DocQuery.resolveSymbol(defn.rootNameTable, path.split('.').filter(_.nonEmpty).toList) + + case _: Selector.FileSelector => + Nil val libraryUnits = if queryText.isEmpty then diff --git a/stack-lang/doc/DocQuery.scala b/stack-lang/doc/DocQuery.scala index 8a8be454..1f1b8ecc 100644 --- a/stack-lang/doc/DocQuery.scala +++ b/stack-lang/doc/DocQuery.scala @@ -1,6 +1,7 @@ package doc import reporting.Reporter +import sast.{ Definitions, NameTable, Flags } import sast.Symbols.* import java.nio.file.Paths @@ -11,6 +12,30 @@ object DocQuery: case SymbolSelector(name: String) case FileSelector(raw: String, file: String) + def resolveSymbol(nameTable: NameTable, parts: List[String])(using Definitions): List[Symbol] = + parts match + case Nil => Nil + + case name :: Nil => + nameTable.resolve(name) + + case name :: rest => + nameTable.resolveContainer(name) match + case Some(sym) => + val nameTable = sym.nameTable + resolveSymbol(nameTable, rest) + + case None => + rest match + case memberName :: Nil => + nameTable.resolveType(name) match + case Some(sym) if sym.isOneOf(Flags.Class | Flags.Interface) => + sym.classInfo.getMemberSymbol(memberName).toList + + case _ => Nil + + case _ => Nil + def select(index: DocIndex, rawQuery: String)(using Reporter): List[DocEntry] = select(index, parse(rawQuery)) diff --git a/stack-lang/sast/Definitions.scala b/stack-lang/sast/Definitions.scala index 01ae6814..6808e986 100644 --- a/stack-lang/sast/Definitions.scala +++ b/stack-lang/sast/Definitions.scala @@ -21,8 +21,6 @@ final class Definitions(private var _index: SymbolIndex) extends Definitions.Laz //---------------------------------------------------------------------------- // Name lookup - def resolve(path: String): List[Symbol] = Definitions.resolveStatic(rootNameTable, path.split('.').toList) - def resolveTerm(path: String): Symbol = resolveStatic(path.split('.').toList, SymbolKind.Term) def resolveType(path: String): Symbol = resolveStatic(path.split('.').toList, SymbolKind.Type) def resolveContainer(path: String): Symbol = resolveStatic(path.split('.').toList, SymbolKind.Container) @@ -183,17 +181,3 @@ object Definitions: case None => None - - def resolveStatic(nameTable: NameTable, parts: List[String]): List[Symbol] = - (parts: @unchecked) match - case name :: Nil => - nameTable.resolve(name) - - case name :: rest => - nameTable.resolveContainer(name) match - case Some(sym) => - val nameTable = sym.nameTable - resolveStatic(nameTable, rest) - - case None => - Nil From e6db24d3c6a629faa21fabfb358016fd93f2b4dd Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 09:38:06 +0200 Subject: [PATCH 16/36] Add extra file in file selector test Signed-off-by: Fengyun Liu --- tests/custom/doc-query/extra.jo | 4 ++++ tests/custom/doc-query/test.sh | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 tests/custom/doc-query/extra.jo diff --git a/tests/custom/doc-query/extra.jo b/tests/custom/doc-query/extra.jo new file mode 100644 index 00000000..6b28e34e --- /dev/null +++ b/tests/custom/doc-query/extra.jo @@ -0,0 +1,4 @@ +namespace DocQueryExtra + +//[ Extra documented value used to exercise file selector filtering. //] +def extraValue: Int = 7 diff --git a/tests/custom/doc-query/test.sh b/tests/custom/doc-query/test.sh index e9a01e3c..3d6a2786 100755 --- a/tests/custom/doc-query/test.sh +++ b/tests/custom/doc-query/test.sh @@ -7,6 +7,7 @@ PROJECT_ROOT="$(cd "$DIR/../../.." && pwd)" TEST_NAME="$(basename "$DIR")" REL_DIR="tests/custom/doc-query" API_FILE="$REL_DIR/api.jo" +EXTRA_FILE="$REL_DIR/extra.jo" SAST_DIR="${TMPDIR:-/tmp}/jo-doc-query-sast-$$" SAST_LOG="$SAST_DIR.log" FAIL_LOG="$DIR/fail.log" @@ -58,8 +59,8 @@ cd "$PROJECT_ROOT" run_json_doc "$API_FILE" > "$DIR/all.json" run_json_doc --query "DocQueryAPI.*,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/structural.json" -run_json_doc --query "file:$API_FILE" "$API_FILE" > "$DIR/file.json" -run_json_doc --query "file:$PROJECT_ROOT/$API_FILE" "$API_FILE" > "$DIR/file-absolute.json" +run_json_doc --query "file:$API_FILE" "$API_FILE" "$EXTRA_FILE" > "$DIR/file.json" +run_json_doc --query "file:$PROJECT_ROOT/$API_FILE" "$API_FILE" "$EXTRA_FILE" > "$DIR/file-absolute.json" run_json_doc --query "DocQueryAPI.describe" "$API_FILE" > "$DIR/describe.json" run_json_doc --query "DocQueryAPI.Box.label" "$API_FILE" > "$DIR/exact-member.json" run_json_doc --include-private --query "DocQueryAPI.hidden" "$API_FILE" > "$DIR/private.json" From 593f7079da669525b02f51f27518aa92ecc6c091 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 10:32:27 +0200 Subject: [PATCH 17/36] Eliminate DocIndex and DocModel Signed-off-by: Fengyun Liu --- stack-lang/doc/Compiler.scala | 22 +- stack-lang/doc/DocJsonEmitter.scala | 75 ----- stack-lang/doc/DocModel.scala | 266 --------------- stack-lang/doc/DocQuery.scala | 500 ++++++++++++++++++++++++++-- 4 files changed, 470 insertions(+), 393 deletions(-) delete mode 100644 stack-lang/doc/DocJsonEmitter.scala delete mode 100644 stack-lang/doc/DocModel.scala diff --git a/stack-lang/doc/Compiler.scala b/stack-lang/doc/Compiler.scala index 83022307..b140d465 100644 --- a/stack-lang/doc/Compiler.scala +++ b/stack-lang/doc/Compiler.scala @@ -7,8 +7,6 @@ import typing.Typer import reporting.Reporter import reporting.Config -import DocQuery.Selector - import java.io.{FileOutputStream, OutputStreamWriter, PrintWriter} import java.nio.file.{Files, Path, Paths} import java.nio.charset.StandardCharsets @@ -67,9 +65,9 @@ object Compiler: val selectors = DocQuery.parse(queryText) val jsonOutput = format.value == "json" || queryText.nonEmpty - def writeJson(entries: List[DocEntry]): Unit = + def writeJson(targets: List[DocQuery.DocTarget])(using Definitions): Unit = val out = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8)) - DocJsonEmitter.emit(entries, out) + DocQuery.emitJson(targets, includePrivate.value, out) out.flush() if jsonOutput && sources.isEmpty && Config.libPaths.value.isEmpty then @@ -77,9 +75,7 @@ object Compiler: Reporter.error("--out is only supported with --format html") return - val selected = DocQuery.select(DocIndex(Nil, Nil), selectors) - if rp.hasErrors then return - writeJson(selected) + DocQuery.reportNoMatches(selectors) return // Parse and type check @@ -94,12 +90,8 @@ object Compiler: Reporter.error("--out is only supported with --format html") return - val querySymbols = selectors.flatMap: - case Selector.SymbolSelector(path) => - DocQuery.resolveSymbol(defn.rootNameTable, path.split('.').filter(_.nonEmpty).toList) - - case _: Selector.FileSelector => - Nil + val resolvedSelectors = DocQuery.resolveSelectors(defn.rootNameTable, selectors) + val querySymbols = DocQuery.querySymbols(resolvedSelectors) val libraryUnits = if queryText.isEmpty then @@ -111,8 +103,8 @@ object Compiler: delayedUnits.force() - val index = DocModel.build(units, libraryUnits, includePrivate.value) - val selected = DocQuery.select(index, selectors) + val filteredUnits = DocQuery.filterUnits(units, libraryUnits, resolvedSelectors) + val selected = DocQuery.select(filteredUnits, resolvedSelectors, includePrivate.value) if rp.hasErrors then return writeJson(selected) diff --git a/stack-lang/doc/DocJsonEmitter.scala b/stack-lang/doc/DocJsonEmitter.scala deleted file mode 100644 index b6b436b9..00000000 --- a/stack-lang/doc/DocJsonEmitter.scala +++ /dev/null @@ -1,75 +0,0 @@ -package doc - -import sast.Constant - -import java.io.PrintWriter - -object DocJsonEmitter: - def emit(entries: List[DocEntry], out: PrintWriter): Unit = - out.println("[") - emitEntryList(entries, out, " ") - if entries.nonEmpty then out.println() - out.println("]") - - private def emitEntryList(entries: List[DocEntry], out: PrintWriter, indent: String): Unit = - var first = true - for entry <- entries do - if !first then out.println(",") - first = false - emitEntry(entry, out, indent) - - private def emitEntry(entry: DocEntry, out: PrintWriter, indent: String): Unit = - val next = indent + " " - out.println(indent + "{") - emitField("name", JsonUtil.string(entry.name), out, next) - emitField("kind", JsonUtil.string(entry.kind), out, next) - emitField("signature", JsonUtil.string(entry.signature), out, next) - emitField("source", sourceJson(entry.source), out, next) - emitField("visibility", JsonUtil.string(entry.visibility), out, next) - emitField("flags", stringArray(entry.flags), out, next) - emitField("annotations", annotationsJson(entry.annotations), out, next) - emitField("doc", entry.doc.map(JsonUtil.string).getOrElse("null"), out, next, comma = entry.views.nonEmpty || entry.members.nonEmpty) - - if entry.views.nonEmpty then - emitField("views", stringArray(entry.views), out, next, comma = entry.members.nonEmpty) - - if entry.members.nonEmpty then - out.println(next + JsonUtil.string("members") + ": [") - emitEntryList(entry.members, out, next + " ") - out.println() - out.println(next + "]") - - out.print(indent + "}") - - private def emitField(name: String, value: String, out: PrintWriter, indent: String, comma: Boolean = true): Unit = - out.print(indent) - out.print(JsonUtil.string(name)) - out.print(": ") - out.print(value) - if comma then out.print(",") - out.println() - - private def sourceJson(source: Option[SourceLoc]): String = - source match - case Some(SourceLoc(file, line)) => - s"""{ "file": ${JsonUtil.string(file)}, "line": $line }""" - case None => - "null" - - private def stringArray(values: List[String]): String = - values.map(JsonUtil.string).mkString("[", ", ", "]") - - private def annotationsJson(annotations: List[DocAnnotation]): String = - annotations.map { annot => - s"""{ "name": ${JsonUtil.string(annot.name)}, "args": ${valuesJson(annot.args)} }""" - }.mkString("[", ", ", "]") - - private def valuesJson(values: List[Constant]): String = - values.map(constantJson).mkString("[", ", ", "]") - - private def constantJson(value: Constant): String = - value match - case Constant.String(v) => JsonUtil.string(v) - case Constant.Int(v) => v.toString - case Constant.Float(v) => v.toString - case Constant.Bool(v) => v.toString diff --git a/stack-lang/doc/DocModel.scala b/stack-lang/doc/DocModel.scala deleted file mode 100644 index 6c56a3a5..00000000 --- a/stack-lang/doc/DocModel.scala +++ /dev/null @@ -1,266 +0,0 @@ -package doc - -import sast.* -import sast.Symbols.* -import sast.Trees.* -import sast.Types.* -import sast.Denotations.* -import sast.Flags - -import scala.collection.mutable - -case class SourceLoc(file: String, line: Int) - -case class DocAnnotation(name: String, args: List[Constant]) - -case class DocEntry( - symbol: Symbol, - name: String, - kind: String, - signature: String, - source: Option[SourceLoc], - visibility: String, - flags: List[String], - annotations: List[DocAnnotation], - doc: Option[String], - members: List[DocEntry] = Nil, - views: List[String] = Nil, -): - def isAggregate: Boolean = - kind == "namespace" || kind == "section" || kind == "class" - -case class DocIndex(entries: List[DocEntry], sourceRoots: List[DocEntry]) - -object DocModel: - def build(sourceUnits: List[FileUnit], libraryUnits: List[FileUnit], includePrivate: Boolean)(using Definitions): DocIndex = - val allUnits = sourceUnits ++ libraryUnits - val cache = mutable.HashMap.empty[Symbol, Option[DocEntry]] - - def visible(sym: Symbol): Boolean = - includePrivate || !sym.isPrivate - - def sourceLoc(sym: Symbol): Option[SourceLoc] = - if sym.sourcePos == null then None - else Some(SourceLoc(sym.source.file, sym.sourcePos.startLine + 1)) - - def visibility(sym: Symbol): String = - sym.visibility match - case Visibility.Default => "public" - case Visibility.Private(within) => - if within == sym.owner then "private" - else s"private[${within.fullName}]" - - def docs(sym: Symbol): Option[String] = - val lines = summon[Definitions].index.docComment(sym) - if lines.isEmpty then None else Some(lines.mkString("\n")) - - def annotations(sym: Symbol): List[DocAnnotation] = - sym.annotations.map: annot => - DocAnnotation(annot.symbol.fullName, annot.args) - - def flags(sym: Symbol): List[String] = - Flags.flagStrings(sym.flags) - - def typeParams(params: List[Symbol]): String = - if params.isEmpty then "" else params.map(_.name).mkString("[", ", ", "]") - - def symbolBase(sym: Symbol, kind: String, signature: String, members: List[DocEntry] = Nil, views: List[String] = Nil): DocEntry = - DocEntry( - symbol = sym, - name = sym.fullName, - kind = kind, - signature = signature, - source = sourceLoc(sym), - visibility = visibility(sym), - flags = flags(sym), - annotations = annotations(sym), - doc = docs(sym), - members = sortEntries(members), - views = views, - ) - - def defaultValue(value: DefaultValue): String = - value match - case DefaultValue.Lit(const) => const match - case Constant.Bool(v) => v.toString - case Constant.Int(v) => v.toString - case Constant.Float(v) => v.toString - case Constant.String(v) => JsonUtil.string(v) - case DefaultValue.Ref(sym) => sym.name - - def paramSignature(sym: Symbol, default: Option[DefaultValue] = None): String = - val base = sym.name + ": " + sym.tpe.show - default match - case Some(value) => base + " = " + defaultValue(value) - case None => base - - def autoCandidateSignature(candidate: AutoCandidate): String = - candidate.show - - def autoSignature(sym: Symbol, candidates: List[AutoCandidate]): String = - val candidateText = - if candidates.isEmpty then "" - else " with [" + candidates.map(autoCandidateSignature).mkString(", ") + "]" - paramSignature(sym) + candidateText - - def receivesSignature(procType: ProcType): String = - def showEffects(effects: List[Symbol]): String = - if effects.isEmpty then " receives none" - else " receives " + effects.map(_.name).mkString(", ") - - procType.receivesInfo match - case sym: Symbol => - summon[Definitions].index.effectEngine.getKnownEffects(sym) match - case Some(effects) => showEffects(effects) - case None => "" - case effects: List[Symbol] => - showEffects(effects) - - def procSignature(fd: FunDef): String = - val procType = fd.symbol.tpe.asProcType - val tparamText = - if fd.tparams.isEmpty then "" - else fd.tparams.map(_.name).mkString("[", ", ", "]") - - val preParamText = - if procType.preParamCount == 0 then "" - else fd.params.take(procType.preParamCount).map(paramSignature(_)).mkString("(", ", ", ")") - - val postParams = fd.params.drop(procType.preParamCount) - val defaultCount = procType.defaults.size - val postParamText = - val split = postParams.size - defaultCount - val withoutDefaults = postParams.take(split).map(paramSignature(_)) - val withDefaults = postParams.drop(split).zip(procType.defaults).map: - case (param, default) => paramSignature(param, Some(default)) - (withoutDefaults ++ withDefaults).mkString("(", ", ", ")") - - val autoText = - if fd.autos.isEmpty then "" - else fd.autos.zip(fd.candidates).map(autoSignature).mkString("(auto ", ", ", ")") - - tparamText + preParamText + postParamText + autoText + ": " + fd.resultType.tpe.show + receivesSignature(procType) - - def funSignature(fd: FunDef): String = - val sym = fd.symbol - if sym.is(Flags.Annotation) then - "annotation " + sym.name + procSignature(fd).stripSuffix(": void receives none").stripSuffix(": void") - else if sym.is(Flags.Constructor) then - "constructor" + procSignature(fd) - else - "def " + sym.name + procSignature(fd) - - def patternSignature(pd: PatDef): String = - "pattern " + pd.symbol.name + pd.symbol.tpe.asProcType.show - - def typeSignature(td: TypeDef): String = - val sym = td.symbol - val info = sym.info - val rhs = - if sym.is(Flags.Defer) then "" - else - info match - case TypeOperatorInfo(_, body, _) => " = " + body.show - case tp: Type => " = " + tp.show - case other => " = " + other.toString - "type " + sym.name + typeParams(td.tparams) + rhs - - def classSignature(cd: ClassDef): String = - val sym = cd.symbol - val prefix = - if sym.is(Flags.Object) then "object" - else if sym.isInterface then "interface" - else "class" - prefix + " " + sym.name + typeParams(cd.tparams) - - def interfaceSignature(id: InterfaceDef): String = - "interface " + id.symbol.name + typeParams(id.tparams) - - def fieldSignature(field: FieldDecl): String = - val sym = field.symbol - val prefix = if sym.isMutable then "var " else "val " - prefix + sym.name + ": " + field.tpt.tpe.show - - def entryForField(field: FieldDecl): Option[DocEntry] = - val sym = field.symbol - if !visible(sym) then None - else Some(symbolBase(sym, "field", fieldSignature(field))) - - def entryForFun(fd: FunDef): Option[DocEntry] = - val sym = fd.symbol - if !visible(sym) || sym.is(Flags.Object) then None - else - val kind = if sym.is(Flags.Constructor) then "constructor" else "def" - Some(symbolBase(sym, kind, funSignature(fd))) - - def entryForDef(defn: Def): Option[DocEntry] = - cache.getOrElseUpdate(defn.symbol, - if !visible(defn.symbol) then None - else - defn match - case pd: ParamDef => - Some(symbolBase(pd.symbol, "param", "param " + pd.name + ": " + pd.tpt.tpe.show)) - - case td: TypeDef => - Some(symbolBase(td.symbol, "type", typeSignature(td))) - - case fd: FunDef => - entryForFun(fd) - - case pd: PatDef => - if pd.resultType.tpe.isSingletonObjectType then None - else Some(symbolBase(pd.symbol, "pattern", patternSignature(pd))) - - case cd: ClassDef => - val fields = cd.vals.flatMap(entryForField) - val funs = cd.funs.flatMap(entryForFun) - val views = cd.views.map(_.tpe.show) - Some(symbolBase(cd.symbol, "class", classSignature(cd), fields ++ funs, views)) - - case id: InterfaceDef => - val methods = id.methods.flatMap(entryForFun) - Some(symbolBase(id.symbol, "class", interfaceSignature(id), methods)) - - case sec: Section => - val members = sec.defs.flatMap(entryForDef) - Some(symbolBase(sec.symbol, "section", "section " + sec.symbol.name, members)) - ) - - def namespaceEntry(sym: Symbol, units: List[FileUnit]): Option[DocEntry] = - if !visible(sym) then None - else - val members = units.flatMap(_.defs.flatMap(entryForDef)) - Some(symbolBase(sym, "namespace", "namespace " + sym.fullName, members)) - - val namespaceEntries = - allUnits.groupBy(_.owner).toList.sortBy(_._1.fullName).flatMap(namespaceEntry) - - val sourceRoots = - sourceUnits.flatMap(unit => unit.defs.flatMap(entryForDef)) - - val entries = - distinctEntries(namespaceEntries.flatMap(flattenEntry) ++ sourceRoots.flatMap(flattenEntry)) - - DocIndex(entries, sortEntries(sourceRoots)) - - def flattenEntry(entry: DocEntry): List[DocEntry] = - entry :: entry.members.flatMap(flattenEntry) - - def distinctEntries(entries: List[DocEntry]): List[DocEntry] = - val seen = mutable.HashSet.empty[Symbol] - entries.filter: entry => - if seen.contains(entry.symbol) then false - else - seen += entry.symbol - true - - def sortEntries(entries: List[DocEntry]): List[DocEntry] = - entries.map(sortEntry).sortBy(sortKey) - - private def sortEntry(entry: DocEntry): DocEntry = - entry.copy(members = sortEntries(entry.members)) - - private def sortKey(entry: DocEntry): (String, Int, String, String) = - val file = entry.source.map(_.file).getOrElse("") - val line = entry.source.map(_.line).getOrElse(0) - (file, line, entry.kind, entry.name) diff --git a/stack-lang/doc/DocQuery.scala b/stack-lang/doc/DocQuery.scala index 1f1b8ecc..5cf1b53f 100644 --- a/stack-lang/doc/DocQuery.scala +++ b/stack-lang/doc/DocQuery.scala @@ -1,9 +1,14 @@ package doc import reporting.Reporter -import sast.{ Definitions, NameTable, Flags } +import sast.* import sast.Symbols.* +import sast.Trees.* +import sast.Types.* +import sast.Denotations.* +import sast.Flags +import java.io.PrintWriter import java.nio.file.Paths import scala.collection.mutable @@ -12,6 +17,34 @@ object DocQuery: case SymbolSelector(name: String) case FileSelector(raw: String, file: String) + enum ResolvedSelector: + case SymbolSelector(name: String, symbols: List[Symbol]) + case FileSelector(raw: String, file: String) + + enum DocTarget: + case Namespace(sym: Symbol, units: List[FileUnit]) + case Definition(defn: Def) + case Field(field: FieldDecl) + + def symbol: Symbol = + this match + case Namespace(sym, _) => sym + case Definition(defn) => defn.symbol + case Field(field) => field.symbol + + def resolveSelectors(nameTable: NameTable, selectors: List[Selector])(using Definitions): List[ResolvedSelector] = + selectors.map: + case Selector.SymbolSelector(path) => + ResolvedSelector.SymbolSelector(path, resolveSymbol(nameTable, path.split('.').filter(_.nonEmpty).toList)) + + case Selector.FileSelector(raw, file) => + ResolvedSelector.FileSelector(raw, file) + + def querySymbols(selectors: List[ResolvedSelector]): List[Symbol] = + selectors.flatMap: + case ResolvedSelector.SymbolSelector(_, symbols) => symbols + case ResolvedSelector.FileSelector(_, _) => Nil + def resolveSymbol(nameTable: NameTable, parts: List[String])(using Definitions): List[Symbol] = parts match case Nil => Nil @@ -22,8 +55,7 @@ object DocQuery: case name :: rest => nameTable.resolveContainer(name) match case Some(sym) => - val nameTable = sym.nameTable - resolveSymbol(nameTable, rest) + resolveSymbol(sym.nameTable, rest) case None => rest match @@ -36,37 +68,53 @@ object DocQuery: case _ => Nil - def select(index: DocIndex, rawQuery: String)(using Reporter): List[DocEntry] = - select(index, parse(rawQuery)) + def filterUnits(sourceUnits: List[FileUnit], libraryUnits: List[FileUnit], selectors: List[ResolvedSelector])(using Reporter): List[FileUnit] = + if selectors.isEmpty then + sourceUnits + else + val allUnits = sourceUnits ++ libraryUnits + val selected = selectors.flatMap: + case ResolvedSelector.FileSelector(raw, file) => + val matches = allUnits.filter(unit => matchesFile(unit.source.file, file)) + if matches.isEmpty then Reporter.error(s"No documentation entries match file selector `$raw`") + matches + + case ResolvedSelector.SymbolSelector(_, symbols) => + allUnits.filter: unit => + symbols.exists: sym => + unit.owner.containedIn(sym) || sym.containedIn(unit.owner) + + distinctUnits(selected) - def select(index: DocIndex, selectors: List[Selector])(using Reporter): List[DocEntry] = + def select(units: List[FileUnit], selectors: List[ResolvedSelector], includePrivate: Boolean)(using Reporter, Definitions): List[DocTarget] = val selected = - if selectors.isEmpty then index.sourceRoots - else selectors.flatMap(resolveSelector(index, _)) + if selectors.isEmpty then + sourceTargets(units, includePrivate) + else + selectors.flatMap(resolveSelector(units, _, includePrivate)) - DocModel.sortEntries(pruneDescendants(selected).distinct) + pruneDescendants(mergeTargets(selected)) - private def resolveSelector(index: DocIndex, selector: Selector)(using Reporter): List[DocEntry] = - selector match - case Selector.FileSelector(raw, file) => - val matches = index.entries.filter(entry => matchesFile(entry, file)) - if matches.isEmpty then Reporter.error(s"No documentation entries match file selector `$raw`") - matches + def reportNoMatches(selectors: List[Selector])(using Reporter): Unit = + selectors.foreach: + case Selector.FileSelector(raw, _) => + Reporter.error(s"No documentation entries match file selector `$raw`") case Selector.SymbolSelector(name) => - val matches = resolveSymbol(index, name) - if matches.isEmpty then Reporter.error(s"No documentation entries match symbol selector `$name`") - matches + Reporter.error(s"No documentation entries match symbol selector `$name`") - private def resolveSymbol(index: DocIndex, selector: String): List[DocEntry] = - val exactNames = - if selector.contains(".") then List(selector) - else List(selector, "jo." + selector) + private def resolveSelector(units: List[FileUnit], selector: ResolvedSelector, includePrivate: Boolean)(using Reporter, Definitions): List[DocTarget] = + selector match + case ResolvedSelector.FileSelector(raw, file) => + val matches = units.filter(unit => matchesFile(unit.source.file, file)) + val targets = namespaceTargets(matches) + if targets.isEmpty then Reporter.error(s"No documentation entries match file selector `$raw`") + targets - val exact = index.entries.filter(entry => exactNames.contains(entry.name)) - if exact.nonEmpty then exact - else if selector.contains(".") then Nil - else index.entries.filter(_.symbol.name == selector) + case ResolvedSelector.SymbolSelector(name, symbols) => + val targets = symbols.flatMap(sym => targetForSymbol(units, sym, includePrivate)) + if targets.isEmpty then Reporter.error(s"No documentation entries match symbol selector `$name`") + targets def parse(rawQuery: String): List[Selector] = rawQuery.split(",").map(_.trim).filter(_.nonEmpty).map { selector => @@ -79,13 +127,12 @@ object DocQuery: Selector.SymbolSelector(name) }.toList - private def matchesFile(entry: DocEntry, rawFile: String): Boolean = - entry.source.exists: source => - val rawNormalized = normalize(rawFile) - val sourceNormalized = normalize(source.file) - rawFile == source.file || - rawNormalized == sourceNormalized || - absolute(rawFile) == absolute(source.file) + private def matchesFile(sourceFile: String, rawFile: String): Boolean = + val rawNormalized = normalize(rawFile) + val sourceNormalized = normalize(sourceFile) + rawFile == sourceFile || + rawNormalized == sourceNormalized || + absolute(rawFile) == absolute(sourceFile) private def normalize(path: String): String = Paths.get(path).normalize().toString @@ -93,9 +140,388 @@ object DocQuery: private def absolute(path: String): String = Paths.get(path).toAbsolutePath.normalize().toString - private def pruneDescendants(entries: List[DocEntry]): List[DocEntry] = + private def mergeTargets(targets: List[DocTarget]): List[DocTarget] = + val namespaceUnits = mutable.LinkedHashMap.empty[Symbol, mutable.ArrayBuffer[FileUnit]] + val otherTargets = mutable.LinkedHashMap.empty[Symbol, DocTarget] + + for target <- targets do + target match + case DocTarget.Namespace(sym, units) => + val existing = namespaceUnits.getOrElseUpdate(sym, mutable.ArrayBuffer.empty) + for unit <- units do + if !existing.exists(_ eq unit) then existing += unit + + case _ => + otherTargets.getOrElseUpdate(target.symbol, target) + + val namespaces = namespaceUnits.toList.map: + case (sym, units) => DocTarget.Namespace(sym, units.toList) + + namespaces ++ otherTargets.values + + private def pruneDescendants(targets: List[DocTarget]): List[DocTarget] = val selected = mutable.HashSet.empty[Symbol] - selected ++= entries.map(_.symbol) + selected ++= targets.map(_.symbol) + + targets.filterNot: target => + target.symbol.ownersIterator.exists(selected.contains) + + private def distinctUnits(units: List[FileUnit]): List[FileUnit] = + val seen = mutable.HashSet.empty[FileUnit] + units.filter: unit => + if seen.contains(unit) then false + else + seen += unit + true + + def emitJson(targets: List[DocTarget], includePrivate: Boolean, out: PrintWriter)(using Definitions): Unit = + val sortedTargets = sortTargets(targets) + out.println("[") + emitTargetList(sortedTargets, includePrivate, out, " ") + if sortedTargets.nonEmpty then out.println() + out.println("]") + + def sourceTargets(units: List[FileUnit], includePrivate: Boolean)(using Definitions): List[DocTarget] = + units.flatMap(unit => unit.defs.flatMap(targetForDef(_, includePrivate))) + + def namespaceTargets(units: List[FileUnit]): List[DocTarget] = + units.groupBy(_.owner).toList.sortBy(_._1.fullName).map: + case (owner, ownerUnits) => DocTarget.Namespace(owner, ownerUnits.sortBy(_.source.file)) + + def targetForSymbol(units: List[FileUnit], sym: Symbol, includePrivate: Boolean)(using Definitions): List[DocTarget] = + val namespaceUnits = units.filter(_.owner == sym) + if namespaceUnits.nonEmpty then + List(DocTarget.Namespace(sym, namespaceUnits)) + else + symbolTargets(units, includePrivate).get(sym).toList + + def targetForDef(defn: Def, includePrivate: Boolean)(using Definitions): Option[DocTarget] = + if !visible(defn.symbol, includePrivate) then None + else + defn match + case fd: FunDef if fd.symbol.is(Flags.Object) => + None + + case pd: PatDef if pd.resultType.tpe.isSingletonObjectType => + None + + case _ => + Some(DocTarget.Definition(defn)) + + def targetForField(field: FieldDecl, includePrivate: Boolean): Option[DocTarget] = + if visible(field.symbol, includePrivate) then Some(DocTarget.Field(field)) + else None + + def visible(sym: Symbol, includePrivate: Boolean): Boolean = + includePrivate || !sym.isPrivate + + def sortTargets(targets: List[DocTarget]): List[DocTarget] = + targets.map(sortTarget).sortBy(sortKey) + + private def symbolTargets(units: List[FileUnit], includePrivate: Boolean)(using Definitions): Map[Symbol, DocTarget] = + val targets = mutable.LinkedHashMap.empty[Symbol, DocTarget] + + def add(target: DocTarget): Unit = + targets.getOrElseUpdate(target.symbol, target) + + def addDef(defn: Def): Unit = + targetForDef(defn, includePrivate).foreach(add) + + defn match + case cd: ClassDef => + cd.vals.flatMap(targetForField(_, includePrivate)).foreach(add) + cd.funs.foreach(addDef) + + case id: InterfaceDef => + id.methods.foreach(addDef) + + case sec: Section => + sec.defs.foreach(addDef) + + case _ => + + units.foreach: unit => + unit.defs.foreach(addDef) + + targets.toMap + + private def sortTarget(target: DocTarget): DocTarget = + target match + case DocTarget.Namespace(sym, units) => + DocTarget.Namespace(sym, units.sortBy(_.source.file)) + + case _ => + target + + private def sortKey(target: DocTarget): (String, Int, String, String) = + val source = sourceLoc(target.symbol) + val file = source.map(_.file).getOrElse("") + val line = source.map(_.line).getOrElse(0) + (file, line, kind(target), target.symbol.fullName) + + private def emitTargetList(targets: List[DocTarget], includePrivate: Boolean, out: PrintWriter, indent: String)(using Definitions): Unit = + var first = true + for target <- targets do + if !first then out.println(",") + first = false + emitTarget(target, includePrivate, out, indent) + + private def emitTarget(target: DocTarget, includePrivate: Boolean, out: PrintWriter, indent: String)(using Definitions): Unit = + val sym = target.symbol + val members = memberTargets(target, includePrivate) + val views = target match + case DocTarget.Definition(cd: ClassDef) => cd.views.map(_.tpe.show) + case _ => Nil + + val next = indent + " " + out.println(indent + "{") + emitField("name", JsonUtil.string(sym.fullName), out, next) + emitField("kind", JsonUtil.string(kind(target)), out, next) + emitField("signature", JsonUtil.string(signature(target)), out, next) + emitField("source", sourceJson(sourceLoc(sym)), out, next) + emitField("visibility", JsonUtil.string(visibility(sym)), out, next) + emitField("flags", stringArray(Flags.flagStrings(sym.flags)), out, next) + emitField("annotations", annotationsJson(sym), out, next) + emitField("doc", docs(sym).map(JsonUtil.string).getOrElse("null"), out, next, comma = views.nonEmpty || members.nonEmpty) + + if views.nonEmpty then + emitField("views", stringArray(views), out, next, comma = members.nonEmpty) + + if members.nonEmpty then + out.println(next + JsonUtil.string("members") + ": [") + emitTargetList(members, includePrivate, out, next + " ") + out.println() + out.println(next + "]") + + out.print(indent + "}") + + private def memberTargets(target: DocTarget, includePrivate: Boolean)(using Definitions): List[DocTarget] = + val members = + target match + case DocTarget.Namespace(_, units) => + units.flatMap(unit => unit.defs.flatMap(targetForDef(_, includePrivate))) + + case DocTarget.Definition(sec: Section) => + sec.defs.flatMap(targetForDef(_, includePrivate)) + + case DocTarget.Definition(cd: ClassDef) => + cd.vals.flatMap(targetForField(_, includePrivate)) ++ + cd.funs.flatMap(targetForDef(_, includePrivate)) + + case DocTarget.Definition(id: InterfaceDef) => + id.methods.flatMap(targetForDef(_, includePrivate)) + + case _ => + Nil + + sortTargets(members) + + private def kind(target: DocTarget): String = + target match + case DocTarget.Namespace(_, _) => + "namespace" + + case DocTarget.Field(_) => + "field" + + case DocTarget.Definition(_: ParamDef) => + "param" + + case DocTarget.Definition(_: TypeDef) => + "type" + + case DocTarget.Definition(fd: FunDef) => + if fd.symbol.is(Flags.Constructor) then "constructor" else "def" + + case DocTarget.Definition(_: PatDef) => + "pattern" + + case DocTarget.Definition(_: ClassDef | _: InterfaceDef) => + "class" + + case DocTarget.Definition(_: Section) => + "section" + + private def signature(target: DocTarget)(using Definitions): String = + target match + case DocTarget.Namespace(sym, _) => + "namespace " + sym.fullName + + case DocTarget.Field(field) => + fieldSignature(field) + + case DocTarget.Definition(pd: ParamDef) => + "param " + pd.name + ": " + pd.tpt.tpe.show + + case DocTarget.Definition(td: TypeDef) => + typeSignature(td) + + case DocTarget.Definition(fd: FunDef) => + funSignature(fd) + + case DocTarget.Definition(pd: PatDef) => + patternSignature(pd) + + case DocTarget.Definition(cd: ClassDef) => + classSignature(cd) + + case DocTarget.Definition(id: InterfaceDef) => + interfaceSignature(id) + + case DocTarget.Definition(sec: Section) => + "section " + sec.symbol.name + + private def sourceLoc(sym: Symbol): Option[SourceLoc] = + if sym.sourcePos == null then None + else Some(SourceLoc(sym.source.file, sym.sourcePos.startLine + 1)) + + private case class SourceLoc(file: String, line: Int) + + private def sourceJson(source: Option[SourceLoc]): String = + source match + case Some(SourceLoc(file, line)) => + s"""{ "file": ${JsonUtil.string(file)}, "line": $line }""" + case None => + "null" + + private def visibility(sym: Symbol): String = + sym.visibility match + case Visibility.Default => "public" + case Visibility.Private(within) => + if within == sym.owner then "private" + else s"private[${within.fullName}]" + + private def docs(sym: Symbol)(using Definitions): Option[String] = + val lines = summon[Definitions].index.docComment(sym) + if lines.isEmpty then None else Some(lines.mkString("\n")) + + private def annotationsJson(sym: Symbol)(using Definitions): String = + sym.annotations.map { annot => + s"""{ "name": ${JsonUtil.string(annot.symbol.fullName)}, "args": ${valuesJson(annot.args)} }""" + }.mkString("[", ", ", "]") + + private def valuesJson(values: List[Constant]): String = + values.map(constantJson).mkString("[", ", ", "]") + + private def constantJson(value: Constant): String = + value match + case Constant.String(v) => JsonUtil.string(v) + case Constant.Int(v) => v.toString + case Constant.Float(v) => v.toString + case Constant.Bool(v) => v.toString + + private def stringArray(values: List[String]): String = + values.map(JsonUtil.string).mkString("[", ", ", "]") + + private def emitField(name: String, value: String, out: PrintWriter, indent: String, comma: Boolean = true): Unit = + out.print(indent) + out.print(JsonUtil.string(name)) + out.print(": ") + out.print(value) + if comma then out.print(",") + out.println() + + private def typeParams(params: List[Symbol]): String = + if params.isEmpty then "" else params.map(_.name).mkString("[", ", ", "]") + + private def defaultValue(value: DefaultValue): String = + value match + case DefaultValue.Lit(const) => const match + case Constant.Bool(v) => v.toString + case Constant.Int(v) => v.toString + case Constant.Float(v) => v.toString + case Constant.String(v) => JsonUtil.string(v) + case DefaultValue.Ref(sym) => sym.name + + private def paramSignature(sym: Symbol, default: Option[DefaultValue] = None)(using Definitions): String = + val base = sym.name + ": " + sym.tpe.show + default match + case Some(value) => base + " = " + defaultValue(value) + case None => base + + private def autoCandidateSignature(candidate: AutoCandidate)(using Definitions): String = + candidate.show + + private def autoSignature(sym: Symbol, candidates: List[AutoCandidate])(using Definitions): String = + val candidateText = + if candidates.isEmpty then "" + else " with [" + candidates.map(autoCandidateSignature).mkString(", ") + "]" + paramSignature(sym) + candidateText + + private def receivesSignature(procType: ProcType)(using Definitions): String = + def showEffects(effects: List[Symbol]): String = + if effects.isEmpty then " receives none" + else " receives " + effects.map(_.name).mkString(", ") + + procType.receivesInfo match + case sym: Symbol => + summon[Definitions].index.effectEngine.getKnownEffects(sym) match + case Some(effects) => showEffects(effects) + case None => "" + case effects: List[Symbol] => + showEffects(effects) + + private def procSignature(fd: FunDef)(using Definitions): String = + val procType = fd.symbol.tpe.asProcType + val tparamText = + if fd.tparams.isEmpty then "" + else fd.tparams.map(_.name).mkString("[", ", ", "]") + + val preParamText = + if procType.preParamCount == 0 then "" + else fd.params.take(procType.preParamCount).map(paramSignature(_)).mkString("(", ", ", ")") + + val postParams = fd.params.drop(procType.preParamCount) + val defaultCount = procType.defaults.size + val postParamText = + val split = postParams.size - defaultCount + val withoutDefaults = postParams.take(split).map(paramSignature(_)) + val withDefaults = postParams.drop(split).zip(procType.defaults).map: + case (param, default) => paramSignature(param, Some(default)) + (withoutDefaults ++ withDefaults).mkString("(", ", ", ")") + + val autoText = + if fd.autos.isEmpty then "" + else fd.autos.zip(fd.candidates).map(autoSignature).mkString("(auto ", ", ", ")") + + tparamText + preParamText + postParamText + autoText + ": " + fd.resultType.tpe.show + receivesSignature(procType) + + private def funSignature(fd: FunDef)(using Definitions): String = + val sym = fd.symbol + if sym.is(Flags.Annotation) then + "annotation " + sym.name + procSignature(fd).stripSuffix(": void receives none").stripSuffix(": void") + else if sym.is(Flags.Constructor) then + "constructor" + procSignature(fd) + else + "def " + sym.name + procSignature(fd) + + private def patternSignature(pd: PatDef)(using Definitions): String = + "pattern " + pd.symbol.name + pd.symbol.tpe.asProcType.show + + private def typeSignature(td: TypeDef)(using Definitions): String = + val sym = td.symbol + val info = sym.info + val rhs = + if sym.is(Flags.Defer) then "" + else + info match + case TypeOperatorInfo(_, body, _) => " = " + body.show + case tp: Type => " = " + tp.show + case other => " = " + other.toString + "type " + sym.name + typeParams(td.tparams) + rhs + + private def classSignature(cd: ClassDef): String = + val sym = cd.symbol + val prefix = + if sym.is(Flags.Object) then "object" + else if sym.isInterface then "interface" + else "class" + prefix + " " + sym.name + typeParams(cd.tparams) + + private def interfaceSignature(id: InterfaceDef): String = + "interface " + id.symbol.name + typeParams(id.tparams) - entries.filterNot: entry => - entry.symbol.ownersIterator.exists(selected.contains) + private def fieldSignature(field: FieldDecl)(using Definitions): String = + val sym = field.symbol + val prefix = if sym.isMutable then "var " else "val " + prefix + sym.name + ": " + field.tpt.tpe.show From d34cb37bc88e4b866d083f24cca86e47d2cdac22 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 10:46:54 +0200 Subject: [PATCH 18/36] Refactor modeling for filter Signed-off-by: Fengyun Liu --- stack-lang/doc/Compiler.scala | 13 ++-- stack-lang/doc/DocQuery.scala | 115 +++++++++++++++------------------- 2 files changed, 57 insertions(+), 71 deletions(-) diff --git a/stack-lang/doc/Compiler.scala b/stack-lang/doc/Compiler.scala index b140d465..d3da37b2 100644 --- a/stack-lang/doc/Compiler.scala +++ b/stack-lang/doc/Compiler.scala @@ -62,7 +62,6 @@ object Compiler: val rootNameTable = new NameTable given lazyDefn: Definitions.Lazy = Definitions.Lazy(rootNameTable) val queryText = query.value.trim - val selectors = DocQuery.parse(queryText) val jsonOutput = format.value == "json" || queryText.nonEmpty def writeJson(targets: List[DocQuery.DocTarget])(using Definitions): Unit = @@ -75,7 +74,7 @@ object Compiler: Reporter.error("--out is only supported with --format html") return - DocQuery.reportNoMatches(selectors) + DocQuery.reportNoMatches(queryText) return // Parse and type check @@ -90,21 +89,21 @@ object Compiler: Reporter.error("--out is only supported with --format html") return - val resolvedSelectors = DocQuery.resolveSelectors(defn.rootNameTable, selectors) - val querySymbols = DocQuery.querySymbols(resolvedSelectors) + val filter = DocQuery.parse(queryText, defn.rootNameTable) + if rp.hasErrors then return val libraryUnits = if queryText.isEmpty then Nil else delayedUnits.forceIf: unit => - querySymbols.exists: querySymbol => + filter.symbols.exists: querySymbol => unit.owner.containedIn(querySymbol) || querySymbol.containedIn(unit.owner) delayedUnits.force() - val filteredUnits = DocQuery.filterUnits(units, libraryUnits, resolvedSelectors) - val selected = DocQuery.select(filteredUnits, resolvedSelectors, includePrivate.value) + val filteredUnits = DocQuery.filterUnits(units, libraryUnits, filter) + val selected = DocQuery.select(filteredUnits, filter, includePrivate.value) if rp.hasErrors then return writeJson(selected) diff --git a/stack-lang/doc/DocQuery.scala b/stack-lang/doc/DocQuery.scala index 5cf1b53f..756c3154 100644 --- a/stack-lang/doc/DocQuery.scala +++ b/stack-lang/doc/DocQuery.scala @@ -13,13 +13,9 @@ import java.nio.file.Paths import scala.collection.mutable object DocQuery: - enum Selector: - case SymbolSelector(name: String) - case FileSelector(raw: String, file: String) - - enum ResolvedSelector: - case SymbolSelector(name: String, symbols: List[Symbol]) - case FileSelector(raw: String, file: String) + case class Filter(files: List[String], symbols: List[Symbol]): + def isEmpty: Boolean = + files.isEmpty && symbols.isEmpty enum DocTarget: case Namespace(sym: Symbol, units: List[FileUnit]) @@ -32,19 +28,6 @@ object DocQuery: case Definition(defn) => defn.symbol case Field(field) => field.symbol - def resolveSelectors(nameTable: NameTable, selectors: List[Selector])(using Definitions): List[ResolvedSelector] = - selectors.map: - case Selector.SymbolSelector(path) => - ResolvedSelector.SymbolSelector(path, resolveSymbol(nameTable, path.split('.').filter(_.nonEmpty).toList)) - - case Selector.FileSelector(raw, file) => - ResolvedSelector.FileSelector(raw, file) - - def querySymbols(selectors: List[ResolvedSelector]): List[Symbol] = - selectors.flatMap: - case ResolvedSelector.SymbolSelector(_, symbols) => symbols - case ResolvedSelector.FileSelector(_, _) => Nil - def resolveSymbol(nameTable: NameTable, parts: List[String])(using Definitions): List[Symbol] = parts match case Nil => Nil @@ -68,64 +51,68 @@ object DocQuery: case _ => Nil - def filterUnits(sourceUnits: List[FileUnit], libraryUnits: List[FileUnit], selectors: List[ResolvedSelector])(using Reporter): List[FileUnit] = - if selectors.isEmpty then + def filterUnits(sourceUnits: List[FileUnit], libraryUnits: List[FileUnit], filter: Filter)(using Reporter): List[FileUnit] = + if filter.isEmpty then sourceUnits else val allUnits = sourceUnits ++ libraryUnits - val selected = selectors.flatMap: - case ResolvedSelector.FileSelector(raw, file) => - val matches = allUnits.filter(unit => matchesFile(unit.source.file, file)) - if matches.isEmpty then Reporter.error(s"No documentation entries match file selector `$raw`") - matches + val fileUnits = filter.files.flatMap: file => + val matches = allUnits.filter(unit => matchesFile(unit.source.file, file)) + if matches.isEmpty then Reporter.error(s"No documentation entries match file selector `file:$file`") + matches - case ResolvedSelector.SymbolSelector(_, symbols) => - allUnits.filter: unit => - symbols.exists: sym => - unit.owner.containedIn(sym) || sym.containedIn(unit.owner) + val symbolUnits = + allUnits.filter: unit => + filter.symbols.exists: sym => + unit.owner.containedIn(sym) || sym.containedIn(unit.owner) - distinctUnits(selected) + distinctUnits(fileUnits ++ symbolUnits) - def select(units: List[FileUnit], selectors: List[ResolvedSelector], includePrivate: Boolean)(using Reporter, Definitions): List[DocTarget] = + def select(units: List[FileUnit], filter: Filter, includePrivate: Boolean)(using Reporter, Definitions): List[DocTarget] = val selected = - if selectors.isEmpty then + if filter.isEmpty then sourceTargets(units, includePrivate) else - selectors.flatMap(resolveSelector(units, _, includePrivate)) + val fileTargets = filter.files.flatMap: file => + val matches = units.filter(unit => matchesFile(unit.source.file, file)) + val targets = namespaceTargets(matches) + if targets.isEmpty then Reporter.error(s"No documentation entries match file selector `file:$file`") + targets + + val symbolTargets = filter.symbols.flatMap: sym => + val targets = targetForSymbol(units, sym, includePrivate) + if targets.isEmpty then Reporter.error(s"No documentation entries match symbol selector `${sym.fullName}`") + targets + + fileTargets ++ symbolTargets pruneDescendants(mergeTargets(selected)) - def reportNoMatches(selectors: List[Selector])(using Reporter): Unit = - selectors.foreach: - case Selector.FileSelector(raw, _) => - Reporter.error(s"No documentation entries match file selector `$raw`") - - case Selector.SymbolSelector(name) => - Reporter.error(s"No documentation entries match symbol selector `$name`") - - private def resolveSelector(units: List[FileUnit], selector: ResolvedSelector, includePrivate: Boolean)(using Reporter, Definitions): List[DocTarget] = - selector match - case ResolvedSelector.FileSelector(raw, file) => - val matches = units.filter(unit => matchesFile(unit.source.file, file)) - val targets = namespaceTargets(matches) - if targets.isEmpty then Reporter.error(s"No documentation entries match file selector `$raw`") - targets - - case ResolvedSelector.SymbolSelector(name, symbols) => - val targets = symbols.flatMap(sym => targetForSymbol(units, sym, includePrivate)) - if targets.isEmpty then Reporter.error(s"No documentation entries match symbol selector `$name`") - targets - - def parse(rawQuery: String): List[Selector] = - rawQuery.split(",").map(_.trim).filter(_.nonEmpty).map { selector => + def reportNoMatches(rawQuery: String)(using Reporter): Unit = + rawQuery.split(",").map(_.trim).filter(_.nonEmpty).foreach: selector => if selector.startsWith("file:") then - Selector.FileSelector(selector, selector.stripPrefix("file:")) + Reporter.error(s"No documentation entries match file selector `$selector`") else - val name = - if selector.endsWith(".*") then selector.stripSuffix(".*") - else selector - Selector.SymbolSelector(name) - }.toList + Reporter.error(s"No documentation entries match symbol selector `${symbolName(selector)}`") + + def parse(rawQuery: String, nameTable: NameTable)(using Reporter, Definitions): Filter = + val files = mutable.ArrayBuffer.empty[String] + val symbols = mutable.ArrayBuffer.empty[Symbol] + + rawQuery.split(",").map(_.trim).filter(_.nonEmpty).foreach: selector => + if selector.startsWith("file:") then + files += selector.stripPrefix("file:") + else + val name = symbolName(selector) + val resolved = resolveSymbol(nameTable, name.split('.').toList) + if resolved.isEmpty then Reporter.error(s"No documentation entries match symbol selector `$name`") + else symbols ++= resolved + + Filter(files.toList, symbols.toList) + + private def symbolName(selector: String): String = + if selector.endsWith(".*") then selector.stripSuffix(".*") + else selector private def matchesFile(sourceFile: String, rawFile: String): Boolean = val rawNormalized = normalize(rawFile) From c07669c3e7207db68ad455deca2e99967044c56a Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 11:35:58 +0200 Subject: [PATCH 19/36] Remove DocTarget Signed-off-by: Fengyun Liu --- stack-lang/doc/Compiler.scala | 7 +- stack-lang/doc/DocQuery.scala | 432 +++++++++++++++++++++------------- 2 files changed, 268 insertions(+), 171 deletions(-) diff --git a/stack-lang/doc/Compiler.scala b/stack-lang/doc/Compiler.scala index d3da37b2..12e374a6 100644 --- a/stack-lang/doc/Compiler.scala +++ b/stack-lang/doc/Compiler.scala @@ -64,9 +64,9 @@ object Compiler: val queryText = query.value.trim val jsonOutput = format.value == "json" || queryText.nonEmpty - def writeJson(targets: List[DocQuery.DocTarget])(using Definitions): Unit = + def writeJson(units: List[FileUnit], filter: DocQuery.Filter)(using Definitions): Unit = val out = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8)) - DocQuery.emitJson(targets, includePrivate.value, out) + DocQuery.emitJson(units, filter, includePrivate.value, out) out.flush() if jsonOutput && sources.isEmpty && Config.libPaths.value.isEmpty then @@ -103,9 +103,8 @@ object Compiler: delayedUnits.force() val filteredUnits = DocQuery.filterUnits(units, libraryUnits, filter) - val selected = DocQuery.select(filteredUnits, filter, includePrivate.value) if rp.hasErrors then return - writeJson(selected) + writeJson(filteredUnits, filter) else generateHtmlDoc(units) diff --git a/stack-lang/doc/DocQuery.scala b/stack-lang/doc/DocQuery.scala index 756c3154..cb14f908 100644 --- a/stack-lang/doc/DocQuery.scala +++ b/stack-lang/doc/DocQuery.scala @@ -17,17 +17,6 @@ object DocQuery: def isEmpty: Boolean = files.isEmpty && symbols.isEmpty - enum DocTarget: - case Namespace(sym: Symbol, units: List[FileUnit]) - case Definition(defn: Def) - case Field(field: FieldDecl) - - def symbol: Symbol = - this match - case Namespace(sym, _) => sym - case Definition(defn) => defn.symbol - case Field(field) => field.symbol - def resolveSymbol(nameTable: NameTable, parts: List[String])(using Definitions): List[Symbol] = parts match case Nil => Nil @@ -68,26 +57,6 @@ object DocQuery: distinctUnits(fileUnits ++ symbolUnits) - def select(units: List[FileUnit], filter: Filter, includePrivate: Boolean)(using Reporter, Definitions): List[DocTarget] = - val selected = - if filter.isEmpty then - sourceTargets(units, includePrivate) - else - val fileTargets = filter.files.flatMap: file => - val matches = units.filter(unit => matchesFile(unit.source.file, file)) - val targets = namespaceTargets(matches) - if targets.isEmpty then Reporter.error(s"No documentation entries match file selector `file:$file`") - targets - - val symbolTargets = filter.symbols.flatMap: sym => - val targets = targetForSymbol(units, sym, includePrivate) - if targets.isEmpty then Reporter.error(s"No documentation entries match symbol selector `${sym.fullName}`") - targets - - fileTargets ++ symbolTargets - - pruneDescendants(mergeTargets(selected)) - def reportNoMatches(rawQuery: String)(using Reporter): Unit = rawQuery.split(",").map(_.trim).filter(_.nonEmpty).foreach: selector => if selector.startsWith("file:") then @@ -127,32 +96,6 @@ object DocQuery: private def absolute(path: String): String = Paths.get(path).toAbsolutePath.normalize().toString - private def mergeTargets(targets: List[DocTarget]): List[DocTarget] = - val namespaceUnits = mutable.LinkedHashMap.empty[Symbol, mutable.ArrayBuffer[FileUnit]] - val otherTargets = mutable.LinkedHashMap.empty[Symbol, DocTarget] - - for target <- targets do - target match - case DocTarget.Namespace(sym, units) => - val existing = namespaceUnits.getOrElseUpdate(sym, mutable.ArrayBuffer.empty) - for unit <- units do - if !existing.exists(_ eq unit) then existing += unit - - case _ => - otherTargets.getOrElseUpdate(target.symbol, target) - - val namespaces = namespaceUnits.toList.map: - case (sym, units) => DocTarget.Namespace(sym, units.toList) - - namespaces ++ otherTargets.values - - private def pruneDescendants(targets: List[DocTarget]): List[DocTarget] = - val selected = mutable.HashSet.empty[Symbol] - selected ++= targets.map(_.symbol) - - targets.filterNot: target => - target.symbol.ownersIterator.exists(selected.contains) - private def distinctUnits(units: List[FileUnit]): List[FileUnit] = val seen = mutable.HashSet.empty[FileUnit] units.filter: unit => @@ -161,110 +104,283 @@ object DocQuery: seen += unit true - def emitJson(targets: List[DocTarget], includePrivate: Boolean, out: PrintWriter)(using Definitions): Unit = - val sortedTargets = sortTargets(targets) + def emitJson(units: List[FileUnit], filter: Filter, includePrivate: Boolean, out: PrintWriter)(using Reporter, Definitions): Unit = + val trimmedUnits = trimUnits(units, filter, includePrivate) + val sortedTargets = sortRoots(jsonRoots(trimmedUnits, filter)) out.println("[") - emitTargetList(sortedTargets, includePrivate, out, " ") + emitRootList(sortedTargets, out, " ") if sortedTargets.nonEmpty then out.println() out.println("]") - def sourceTargets(units: List[FileUnit], includePrivate: Boolean)(using Definitions): List[DocTarget] = - units.flatMap(unit => unit.defs.flatMap(targetForDef(_, includePrivate))) + private def trimUnits(units: List[FileUnit], filter: Filter, includePrivate: Boolean)(using Reporter, Definitions): List[FileUnit] = + if filter.isEmpty then + return units.flatMap: unit => + val defs = visibleDefs(unit.defs, includePrivate) + if defs.isEmpty then None else Some(unit.copy(defs = defs)) + + val requested = filter.symbols.toSet + val matched = mutable.HashSet.empty[Symbol] + val trimmed = units.flatMap: unit => + if emitsNamespace(unit, filter) then + if requested.contains(unit.owner) then matched += unit.owner + val defs = visibleDefs(unit.defs, includePrivate) + markMatchesInDefs(defs, requested, includePrivate, matched) + Some(unit.copy(defs = defs)) + else + val defs = trimDefs(unit.defs, requested, includePrivate, matched) + if defs.isEmpty then None else Some(unit.copy(defs = defs)) + + for sym <- filter.symbols do + val matchedByAncestor = + sym.ownersIterator.exists(matched.contains) + if !matched.contains(sym) && !matchedByAncestor then + Reporter.error(s"No documentation entries match symbol selector `${sym.fullName}`") + + distinctUnits(trimmed) + + private def trimDefs( + defs: List[Def], + requested: Set[Symbol], + includePrivate: Boolean, + matched: mutable.Set[Symbol], + )(using Definitions): List[Def] = + defs.flatMap: defn => + docSymbolForDef(defn, includePrivate) match + case Some(sym) if requested.contains(sym) => + matched += sym + visibleDef(defn, includePrivate).toList + + case Some(_) => + defn match + case cd: ClassDef => + trimClass(cd, requested, includePrivate, matched) + + case id: InterfaceDef => + trimInterface(id, requested, includePrivate, matched) + + case sec: Section => + trimSection(sec, requested, includePrivate, matched) + + case _ => + Nil + + case None => + Nil - def namespaceTargets(units: List[FileUnit]): List[DocTarget] = - units.groupBy(_.owner).toList.sortBy(_._1.fullName).map: - case (owner, ownerUnits) => DocTarget.Namespace(owner, ownerUnits.sortBy(_.source.file)) + private def trimClass( + cd: ClassDef, + requested: Set[Symbol], + includePrivate: Boolean, + matched: mutable.Set[Symbol], + )(using Definitions): List[Def] = + val fields = cd.vals.filter: field => + val keep = requested.contains(field.symbol) && visible(field.symbol, includePrivate) + if keep then matched += field.symbol + keep + + val methods = cd.funs.filter: fun => + docSymbolForDef(fun, includePrivate) match + case Some(sym) if requested.contains(sym) => + matched += sym + true + case _ => + false - def targetForSymbol(units: List[FileUnit], sym: Symbol, includePrivate: Boolean)(using Definitions): List[DocTarget] = - val namespaceUnits = units.filter(_.owner == sym) - if namespaceUnits.nonEmpty then - List(DocTarget.Namespace(sym, namespaceUnits)) + if fields.isEmpty && methods.isEmpty then + Nil + else if fields.isEmpty && methods.size == 1 then + methods else - symbolTargets(units, includePrivate).get(sym).toList + List(ClassDef(cd.symbol, cd.self, cd.tparams, fields, methods, cd.views)(cd.annots, cd.span)) + + private def trimInterface( + id: InterfaceDef, + requested: Set[Symbol], + includePrivate: Boolean, + matched: mutable.Set[Symbol], + )(using Definitions): List[Def] = + val methods = id.methods.filter: fun => + docSymbolForDef(fun, includePrivate) match + case Some(sym) if requested.contains(sym) => + matched += sym + true + case _ => + false - def targetForDef(defn: Def, includePrivate: Boolean)(using Definitions): Option[DocTarget] = - if !visible(defn.symbol, includePrivate) then None + if methods.isEmpty then + Nil + else if methods.size == 1 then + methods else - defn match - case fd: FunDef if fd.symbol.is(Flags.Object) => - None - - case pd: PatDef if pd.resultType.tpe.isSingletonObjectType => - None + List(InterfaceDef(id.symbol, id.self, id.tparams, methods)(id.annots, id.span)) + + private def trimSection( + sec: Section, + requested: Set[Symbol], + includePrivate: Boolean, + matched: mutable.Set[Symbol], + )(using Definitions): List[Def] = + val defs = trimDefs(sec.defs, requested, includePrivate, matched) + if defs.isEmpty then + Nil + else if defs.size == 1 then + defs + else + List(Section(sec.symbol, defs)(sec.annots, sec.span)) - case _ => - Some(DocTarget.Definition(defn)) + private def visibleDefs(defs: List[Def], includePrivate: Boolean)(using Definitions): List[Def] = + defs.flatMap(visibleDef(_, includePrivate)) - def targetForField(field: FieldDecl, includePrivate: Boolean): Option[DocTarget] = - if visible(field.symbol, includePrivate) then Some(DocTarget.Field(field)) - else None + private def visibleDef(defn: Def, includePrivate: Boolean)(using Definitions): Option[Def] = + docSymbolForDef(defn, includePrivate).map: _ => + defn match + case cd: ClassDef => + val fields = cd.vals.filter(field => visible(field.symbol, includePrivate)) + val methods = visibleFunDefs(cd.funs, includePrivate) + ClassDef(cd.symbol, cd.self, cd.tparams, fields, methods, cd.views)(cd.annots, cd.span) - def visible(sym: Symbol, includePrivate: Boolean): Boolean = - includePrivate || !sym.isPrivate + case id: InterfaceDef => + val methods = visibleFunDefs(id.methods, includePrivate) + InterfaceDef(id.symbol, id.self, id.tparams, methods)(id.annots, id.span) - def sortTargets(targets: List[DocTarget]): List[DocTarget] = - targets.map(sortTarget).sortBy(sortKey) + case sec: Section => + Section(sec.symbol, visibleDefs(sec.defs, includePrivate))(sec.annots, sec.span) - private def symbolTargets(units: List[FileUnit], includePrivate: Boolean)(using Definitions): Map[Symbol, DocTarget] = - val targets = mutable.LinkedHashMap.empty[Symbol, DocTarget] + case _ => + defn - def add(target: DocTarget): Unit = - targets.getOrElseUpdate(target.symbol, target) + private def visibleFunDefs(defs: List[FunDef], includePrivate: Boolean)(using Definitions): List[FunDef] = + defs.filter(fun => docSymbolForDef(fun, includePrivate).isDefined) - def addDef(defn: Def): Unit = - targetForDef(defn, includePrivate).foreach(add) + private def markMatchesInDefs( + defs: List[Def], + requested: Set[Symbol], + includePrivate: Boolean, + matched: mutable.Set[Symbol], + )(using Definitions): Unit = + defs.foreach: defn => + docSymbolForDef(defn, includePrivate).foreach: sym => + if requested.contains(sym) then matched += sym defn match case cd: ClassDef => - cd.vals.flatMap(targetForField(_, includePrivate)).foreach(add) - cd.funs.foreach(addDef) + cd.vals.foreach: field => + if requested.contains(field.symbol) && visible(field.symbol, includePrivate) then + matched += field.symbol + markMatchesInDefs(cd.funs, requested, includePrivate, matched) case id: InterfaceDef => - id.methods.foreach(addDef) + markMatchesInDefs(id.methods, requested, includePrivate, matched) case sec: Section => - sec.defs.foreach(addDef) + markMatchesInDefs(sec.defs, requested, includePrivate, matched) case _ => - units.foreach: unit => - unit.defs.foreach(addDef) + private def docSymbolForDef(defn: Def, includePrivate: Boolean)(using Definitions): Option[Symbol] = + if !visible(defn.symbol, includePrivate) then None + else + defn match + case fd: FunDef if fd.symbol.is(Flags.Object) => + None + + case pd: PatDef if pd.resultType.tpe.isSingletonObjectType => + None - targets.toMap + case _ => + Some(defn.symbol) - private def sortTarget(target: DocTarget): DocTarget = - target match - case DocTarget.Namespace(sym, units) => - DocTarget.Namespace(sym, units.sortBy(_.source.file)) + def visible(sym: Symbol, includePrivate: Boolean): Boolean = + includePrivate || !sym.isPrivate - case _ => - target + private def emitsNamespace(unit: FileUnit, filter: Filter): Boolean = + !filter.isEmpty && ( + filter.symbols.contains(unit.owner) || + filter.files.exists(file => matchesFile(unit.source.file, file)) + ) - private def sortKey(target: DocTarget): (String, Int, String, String) = - val source = sourceLoc(target.symbol) + private def jsonRoots(units: List[FileUnit], filter: Filter): List[FileUnit | Def] = + if filter.isEmpty then + units.flatMap(_.defs) + else + val namespaces = mutable.LinkedHashMap.empty[Symbol, mutable.ArrayBuffer[FileUnit]] + val defs = mutable.ArrayBuffer.empty[Def] + + for unit <- units do + if emitsNamespace(unit, filter) then + namespaces.getOrElseUpdate(unit.owner, mutable.ArrayBuffer.empty) += unit + else + defs ++= unit.defs + + val namespaceRoots: List[FileUnit | Def] = namespaces.toList.map: + case (owner, ownerUnits) => + val sorted = ownerUnits.toList.sortBy(_.source.file) + FileUnit(owner, sorted.flatMap(_.imports).distinct, sorted.flatMap(_.defs), sorted.head.source) + + namespaceRoots ++ defs.toList.map(defn => defn: FileUnit | Def) + + private def sortRoots(roots: List[FileUnit | Def]): List[FileUnit | Def] = + roots.sortBy: + case unit: FileUnit => sortKey(unit.owner, "namespace") + case defn: Def => sortKey(defn.symbol, kind(defn)) + + private def sortMembers(members: List[Def | FieldDecl]): List[Def | FieldDecl] = + members.sortBy: + case defn: Def => sortKey(defn.symbol, kind(defn)) + case field: FieldDecl => sortKey(field.symbol, "field") + + private def sortKey(sym: Symbol, kind: String): (String, Int, String, String) = + val source = sourceLoc(sym) val file = source.map(_.file).getOrElse("") val line = source.map(_.line).getOrElse(0) - (file, line, kind(target), target.symbol.fullName) + (file, line, kind, sym.fullName) - private def emitTargetList(targets: List[DocTarget], includePrivate: Boolean, out: PrintWriter, indent: String)(using Definitions): Unit = + private def emitRootList(roots: List[FileUnit | Def], out: PrintWriter, indent: String)(using Definitions): Unit = var first = true - for target <- targets do + for root <- roots do if !first then out.println(",") first = false - emitTarget(target, includePrivate, out, indent) + root match + case unit: FileUnit => emitNamespace(unit, out, indent) + case defn: Def => emitDef(defn, out, indent) - private def emitTarget(target: DocTarget, includePrivate: Boolean, out: PrintWriter, indent: String)(using Definitions): Unit = - val sym = target.symbol - val members = memberTargets(target, includePrivate) - val views = target match - case DocTarget.Definition(cd: ClassDef) => cd.views.map(_.tpe.show) + private def emitMemberList(members: List[Def | FieldDecl], out: PrintWriter, indent: String)(using Definitions): Unit = + var first = true + for member <- members do + if !first then out.println(",") + first = false + member match + case defn: Def => emitDef(defn, out, indent) + case field: FieldDecl => emitFieldDecl(field, out, indent) + + private def emitNamespace(unit: FileUnit, out: PrintWriter, indent: String)(using Definitions): Unit = + val members = sortMembers(unit.defs.map(defn => defn: Def | FieldDecl)) + emitSymbol(unit.owner, "namespace", "namespace " + unit.owner.fullName, Nil, members, out, indent) + + private def emitDef(defn: Def, out: PrintWriter, indent: String)(using Definitions): Unit = + val members = sortMembers(memberNodes(defn)) + val views = defn match + case cd: ClassDef => cd.views.map(_.tpe.show) case _ => Nil - + emitSymbol(defn.symbol, kind(defn), signature(defn), views, members, out, indent) + + private def emitFieldDecl(field: FieldDecl, out: PrintWriter, indent: String)(using Definitions): Unit = + emitSymbol(field.symbol, "field", fieldSignature(field), Nil, Nil, out, indent) + + private def emitSymbol( + sym: Symbol, + kind: String, + signature: String, + views: List[String], + members: List[Def | FieldDecl], + out: PrintWriter, + indent: String, + )(using Definitions): Unit = val next = indent + " " out.println(indent + "{") emitField("name", JsonUtil.string(sym.fullName), out, next) - emitField("kind", JsonUtil.string(kind(target)), out, next) - emitField("signature", JsonUtil.string(signature(target)), out, next) + emitField("kind", JsonUtil.string(kind), out, next) + emitField("signature", JsonUtil.string(signature), out, next) emitField("source", sourceJson(sourceLoc(sym)), out, next) emitField("visibility", JsonUtil.string(visibility(sym)), out, next) emitField("flags", stringArray(Flags.flagStrings(sym.flags)), out, next) @@ -276,86 +392,68 @@ object DocQuery: if members.nonEmpty then out.println(next + JsonUtil.string("members") + ": [") - emitTargetList(members, includePrivate, out, next + " ") + emitMemberList(members, out, next + " ") out.println() out.println(next + "]") out.print(indent + "}") - private def memberTargets(target: DocTarget, includePrivate: Boolean)(using Definitions): List[DocTarget] = - val members = - target match - case DocTarget.Namespace(_, units) => - units.flatMap(unit => unit.defs.flatMap(targetForDef(_, includePrivate))) + private def memberNodes(defn: Def): List[Def | FieldDecl] = + defn match + case sec: Section => + sec.defs.map(defn => defn: Def | FieldDecl) - case DocTarget.Definition(sec: Section) => - sec.defs.flatMap(targetForDef(_, includePrivate)) + case cd: ClassDef => + cd.vals.map(field => field: Def | FieldDecl) ++ + cd.funs.map(fun => fun: Def | FieldDecl) - case DocTarget.Definition(cd: ClassDef) => - cd.vals.flatMap(targetForField(_, includePrivate)) ++ - cd.funs.flatMap(targetForDef(_, includePrivate)) + case id: InterfaceDef => + id.methods.map(fun => fun: Def | FieldDecl) - case DocTarget.Definition(id: InterfaceDef) => - id.methods.flatMap(targetForDef(_, includePrivate)) - - case _ => - Nil - - sortTargets(members) - - private def kind(target: DocTarget): String = - target match - case DocTarget.Namespace(_, _) => - "namespace" - - case DocTarget.Field(_) => - "field" + case _ => + Nil - case DocTarget.Definition(_: ParamDef) => + private def kind(defn: Def): String = + defn match + case _: ParamDef => "param" - case DocTarget.Definition(_: TypeDef) => + case _: TypeDef => "type" - case DocTarget.Definition(fd: FunDef) => + case fd: FunDef => if fd.symbol.is(Flags.Constructor) then "constructor" else "def" - case DocTarget.Definition(_: PatDef) => + case _: PatDef => "pattern" - case DocTarget.Definition(_: ClassDef | _: InterfaceDef) => + case _: ClassDef | _: InterfaceDef => "class" - case DocTarget.Definition(_: Section) => + case _: Section => "section" - private def signature(target: DocTarget)(using Definitions): String = - target match - case DocTarget.Namespace(sym, _) => - "namespace " + sym.fullName - - case DocTarget.Field(field) => - fieldSignature(field) - - case DocTarget.Definition(pd: ParamDef) => + private def signature(defn: Def)(using Definitions): String = + defn match + case pd: ParamDef => "param " + pd.name + ": " + pd.tpt.tpe.show - case DocTarget.Definition(td: TypeDef) => + case td: TypeDef => typeSignature(td) - case DocTarget.Definition(fd: FunDef) => + case fd: FunDef => funSignature(fd) - case DocTarget.Definition(pd: PatDef) => + case pd: PatDef => patternSignature(pd) - case DocTarget.Definition(cd: ClassDef) => + case cd: ClassDef => classSignature(cd) - case DocTarget.Definition(id: InterfaceDef) => + case id: InterfaceDef => interfaceSignature(id) - case DocTarget.Definition(sec: Section) => + case sec: Section => "section " + sec.symbol.name private def sourceLoc(sym: Symbol): Option[SourceLoc] = From a3ede72c2d5bdaa76b372da36e7e7792c8a9a693 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Mon, 27 Jul 2026 11:36:28 +0200 Subject: [PATCH 20/36] Test query field and interface member Signed-off-by: Fengyun Liu --- tests/custom/doc-query/exact-field.json.check | 24 +++++++++++++++++++ .../doc-query/interface-member.json.check | 12 ++++++++++ tests/custom/doc-query/test.sh | 4 ++++ 3 files changed, 40 insertions(+) create mode 100644 tests/custom/doc-query/exact-field.json.check create mode 100644 tests/custom/doc-query/interface-member.json.check diff --git a/tests/custom/doc-query/exact-field.json.check b/tests/custom/doc-query/exact-field.json.check new file mode 100644 index 00000000..5aa8cbad --- /dev/null +++ b/tests/custom/doc-query/exact-field.json.check @@ -0,0 +1,24 @@ +[ + { + "name": "DocQueryAPI.Box", + "kind": "class", + "signature": "class Box", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 42 }, + "visibility": "public", + "flags": ["class"], + "annotations": [], + "doc": "A small documented class.", + "members": [ + { + "name": "DocQueryAPI.Box.name", + "kind": "field", + "signature": "val name: String", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 44 }, + "visibility": "public", + "flags": ["field"], + "annotations": [], + "doc": "Stored display name." + } + ] + } +] diff --git a/tests/custom/doc-query/interface-member.json.check b/tests/custom/doc-query/interface-member.json.check new file mode 100644 index 00000000..b218ba44 --- /dev/null +++ b/tests/custom/doc-query/interface-member.json.check @@ -0,0 +1,12 @@ +[ + { + "name": "DocQueryAPI.FileLike.readText", + "kind": "def", + "signature": "def readText(path: String, limit: Int = 10): String receives none", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, + "visibility": "public", + "flags": ["defer", "fun", "method"], + "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], + "doc": "Read a text file with an optional limit." + } +] diff --git a/tests/custom/doc-query/test.sh b/tests/custom/doc-query/test.sh index 3d6a2786..478b2b52 100755 --- a/tests/custom/doc-query/test.sh +++ b/tests/custom/doc-query/test.sh @@ -63,6 +63,8 @@ run_json_doc --query "file:$API_FILE" "$API_FILE" "$EXTRA_FILE" > "$DIR/file.jso run_json_doc --query "file:$PROJECT_ROOT/$API_FILE" "$API_FILE" "$EXTRA_FILE" > "$DIR/file-absolute.json" run_json_doc --query "DocQueryAPI.describe" "$API_FILE" > "$DIR/describe.json" run_json_doc --query "DocQueryAPI.Box.label" "$API_FILE" > "$DIR/exact-member.json" +run_json_doc --query "DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/interface-member.json" +run_json_doc --query "DocQueryAPI.Box.name" "$API_FILE" > "$DIR/exact-field.json" run_json_doc --include-private --query "DocQueryAPI.hidden" "$API_FILE" > "$DIR/private.json" run_doc --query "DocQueryAPI,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/query-implies-json.json" run_doc --lib "$SAST_DIR" --query "DocQueryAPI" > "$DIR/lib-query.json" @@ -77,6 +79,8 @@ diff -u "$DIR/lib-query.json.check" "$DIR/lib-query.json" diff -u "$DIR/lib-exact-member.json.check" "$DIR/lib-exact-member.json" diff -u "$DIR/describe.json.check" "$DIR/describe.json" diff -u "$DIR/exact-member.json.check" "$DIR/exact-member.json" +diff -u "$DIR/interface-member.json.check" "$DIR/interface-member.json" +diff -u "$DIR/exact-field.json.check" "$DIR/exact-field.json" diff -u "$DIR/private.json.check" "$DIR/private.json" expect_fail "$FAIL_LOG" "$PROJECT_ROOT/bin/jo" compile --doc --format yaml "$API_FILE" From 745f1f7a60ee565a75eae485ad3fd2f4555b6ab9 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 16:23:09 +0200 Subject: [PATCH 21/36] Fix query of class member Signed-off-by: Fengyun Liu --- stack-lang/doc/DocQuery.scala | 21 ++++++++++--------- .../doc-query/companion-member.json.check | 12 +++++++++++ tests/custom/doc-query/companion.jo | 13 ++++++++++++ tests/custom/doc-query/test.sh | 3 +++ 4 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 tests/custom/doc-query/companion-member.json.check create mode 100644 tests/custom/doc-query/companion.jo diff --git a/stack-lang/doc/DocQuery.scala b/stack-lang/doc/DocQuery.scala index cb14f908..aa8179e4 100644 --- a/stack-lang/doc/DocQuery.scala +++ b/stack-lang/doc/DocQuery.scala @@ -25,20 +25,21 @@ object DocQuery: nameTable.resolve(name) case name :: rest => - nameTable.resolveContainer(name) match - case Some(sym) => + val containerMatches = + nameTable.resolveContainer(name).toList.flatMap: sym => resolveSymbol(sym.nameTable, rest) - case None => - rest match - case memberName :: Nil => - nameTable.resolveType(name) match - case Some(sym) if sym.isOneOf(Flags.Class | Flags.Interface) => - sym.classInfo.getMemberSymbol(memberName).toList + val memberMatches = + rest match + case memberName :: Nil => + nameTable.resolveType(name).toList.flatMap: sym => + if sym.isOneOf(Flags.Class | Flags.Interface) then + sym.classInfo.getMemberSymbol(memberName).toList + else Nil - case _ => Nil + case _ => Nil - case _ => Nil + (containerMatches ++ memberMatches).distinct def filterUnits(sourceUnits: List[FileUnit], libraryUnits: List[FileUnit], filter: Filter)(using Reporter): List[FileUnit] = if filter.isEmpty then diff --git a/tests/custom/doc-query/companion-member.json.check b/tests/custom/doc-query/companion-member.json.check new file mode 100644 index 00000000..58120e54 --- /dev/null +++ b/tests/custom/doc-query/companion-member.json.check @@ -0,0 +1,12 @@ +[ + { + "name": "DocQueryCompanion.Value.label", + "kind": "def", + "signature": "def label: String receives none", + "source": { "file": "tests/custom/doc-query/companion.jo", "line": 6 }, + "visibility": "public", + "flags": ["fun", "method"], + "annotations": [], + "doc": "Return the value label." + } +] diff --git a/tests/custom/doc-query/companion.jo b/tests/custom/doc-query/companion.jo new file mode 100644 index 00000000..3ff32021 --- /dev/null +++ b/tests/custom/doc-query/companion.jo @@ -0,0 +1,13 @@ +namespace DocQueryCompanion + +//[ A value with instance operations. //] +class Value + //[ Return the value label. //] + def label: String = "value" +end + +//[ Companion operations for values. //] +section Value + //[ Create a value. //] + def create: Value = new Value() +end diff --git a/tests/custom/doc-query/test.sh b/tests/custom/doc-query/test.sh index 478b2b52..255a59af 100755 --- a/tests/custom/doc-query/test.sh +++ b/tests/custom/doc-query/test.sh @@ -8,6 +8,7 @@ TEST_NAME="$(basename "$DIR")" REL_DIR="tests/custom/doc-query" API_FILE="$REL_DIR/api.jo" EXTRA_FILE="$REL_DIR/extra.jo" +COMPANION_FILE="$REL_DIR/companion.jo" SAST_DIR="${TMPDIR:-/tmp}/jo-doc-query-sast-$$" SAST_LOG="$SAST_DIR.log" FAIL_LOG="$DIR/fail.log" @@ -65,6 +66,7 @@ run_json_doc --query "DocQueryAPI.describe" "$API_FILE" > "$DIR/describe.json" run_json_doc --query "DocQueryAPI.Box.label" "$API_FILE" > "$DIR/exact-member.json" run_json_doc --query "DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/interface-member.json" run_json_doc --query "DocQueryAPI.Box.name" "$API_FILE" > "$DIR/exact-field.json" +run_json_doc --query "DocQueryCompanion.Value.label" "$COMPANION_FILE" > "$DIR/companion-member.json" run_json_doc --include-private --query "DocQueryAPI.hidden" "$API_FILE" > "$DIR/private.json" run_doc --query "DocQueryAPI,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/query-implies-json.json" run_doc --lib "$SAST_DIR" --query "DocQueryAPI" > "$DIR/lib-query.json" @@ -81,6 +83,7 @@ diff -u "$DIR/describe.json.check" "$DIR/describe.json" diff -u "$DIR/exact-member.json.check" "$DIR/exact-member.json" diff -u "$DIR/interface-member.json.check" "$DIR/interface-member.json" diff -u "$DIR/exact-field.json.check" "$DIR/exact-field.json" +diff -u "$DIR/companion-member.json.check" "$DIR/companion-member.json" diff -u "$DIR/private.json.check" "$DIR/private.json" expect_fail "$FAIL_LOG" "$PROJECT_ROOT/bin/jo" compile --doc --format yaml "$API_FILE" From 10eb1680728b60b88f651bc2fdcb08b86fc64b34 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 16:30:02 +0200 Subject: [PATCH 22/36] Force loading sasts that match file selectors Signed-off-by: Fengyun Liu --- stack-lang/doc/Compiler.scala | 1 + stack-lang/doc/DocQuery.scala | 5 +++++ stack-lang/pickle/Decoder.scala | 2 +- stack-lang/pickle/LazyFileUnit.scala | 1 + tests/custom/doc-query/companion-member.json.check | 2 +- tests/custom/doc-query/test.sh | 2 ++ 6 files changed, 11 insertions(+), 2 deletions(-) diff --git a/stack-lang/doc/Compiler.scala b/stack-lang/doc/Compiler.scala index 12e374a6..27cf9ba5 100644 --- a/stack-lang/doc/Compiler.scala +++ b/stack-lang/doc/Compiler.scala @@ -97,6 +97,7 @@ object Compiler: Nil else delayedUnits.forceIf: unit => + filter.selectsFile(unit.sourceFile) || filter.symbols.exists: querySymbol => unit.owner.containedIn(querySymbol) || querySymbol.containedIn(unit.owner) diff --git a/stack-lang/doc/DocQuery.scala b/stack-lang/doc/DocQuery.scala index aa8179e4..b52e208b 100644 --- a/stack-lang/doc/DocQuery.scala +++ b/stack-lang/doc/DocQuery.scala @@ -17,6 +17,9 @@ object DocQuery: def isEmpty: Boolean = files.isEmpty && symbols.isEmpty + def selectsFile(sourceFile: String): Boolean = + files.exists(file => matchesFile(sourceFile, file)) + def resolveSymbol(nameTable: NameTable, parts: List[String])(using Definitions): List[Symbol] = parts match case Nil => Nil @@ -89,6 +92,8 @@ object DocQuery: val sourceNormalized = normalize(sourceFile) rawFile == sourceFile || rawNormalized == sourceNormalized || + (Paths.get(rawFile).getParent == null && + Paths.get(sourceFile).getFileName == Paths.get(rawFile).getFileName) || absolute(rawFile) == absolute(sourceFile) private def normalize(path: String): String = diff --git a/stack-lang/pickle/Decoder.scala b/stack-lang/pickle/Decoder.scala index 34b4db0a..2f293822 100644 --- a/stack-lang/pickle/Decoder.scala +++ b/stack-lang/pickle/Decoder.scala @@ -185,7 +185,7 @@ object Decoder: FileUnit(owner, imports, members, source) - LazyFileUnit(owner, state.accessed, delayed) + LazyFileUnit(owner, source.file, state.accessed, delayed) /** Decode all imports for a file unit */ private def decodeImports diff --git a/stack-lang/pickle/LazyFileUnit.scala b/stack-lang/pickle/LazyFileUnit.scala index a56345f4..eb862855 100644 --- a/stack-lang/pickle/LazyFileUnit.scala +++ b/stack-lang/pickle/LazyFileUnit.scala @@ -17,6 +17,7 @@ import scala.collection.mutable */ class LazyFileUnit( val owner: Symbol, + val sourceFile: String, private val accessed: java.util.concurrent.atomic.AtomicBoolean, private val delayed: () => FileUnit ): diff --git a/tests/custom/doc-query/companion-member.json.check b/tests/custom/doc-query/companion-member.json.check index 58120e54..14bf0778 100644 --- a/tests/custom/doc-query/companion-member.json.check +++ b/tests/custom/doc-query/companion-member.json.check @@ -2,7 +2,7 @@ { "name": "DocQueryCompanion.Value.label", "kind": "def", - "signature": "def label: String receives none", + "signature": "def label(): String receives none", "source": { "file": "tests/custom/doc-query/companion.jo", "line": 6 }, "visibility": "public", "flags": ["fun", "method"], diff --git a/tests/custom/doc-query/test.sh b/tests/custom/doc-query/test.sh index 255a59af..2fe4939c 100755 --- a/tests/custom/doc-query/test.sh +++ b/tests/custom/doc-query/test.sh @@ -71,6 +71,7 @@ run_json_doc --include-private --query "DocQueryAPI.hidden" "$API_FILE" > "$DIR/ run_doc --query "DocQueryAPI,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/query-implies-json.json" run_doc --lib "$SAST_DIR" --query "DocQueryAPI" > "$DIR/lib-query.json" run_doc --lib "$SAST_DIR" --query "DocQueryAPI.Box.label" > "$DIR/lib-exact-member.json" +run_doc --lib "$SAST_DIR" --query "file:api.jo" > "$DIR/lib-file-query.json" diff -u "$DIR/all.json.check" "$DIR/all.json" diff -u "$DIR/structural.json.check" "$DIR/structural.json" @@ -78,6 +79,7 @@ diff -u "$DIR/structural.json.check" "$DIR/file.json" diff -u "$DIR/structural.json.check" "$DIR/file-absolute.json" diff -u "$DIR/structural.json.check" "$DIR/query-implies-json.json" diff -u "$DIR/lib-query.json.check" "$DIR/lib-query.json" +diff -u "$DIR/lib-query.json.check" "$DIR/lib-file-query.json" diff -u "$DIR/lib-exact-member.json.check" "$DIR/lib-exact-member.json" diff -u "$DIR/describe.json.check" "$DIR/describe.json" diff -u "$DIR/exact-member.json.check" "$DIR/exact-member.json" From 78924140e84b0f9643995b994d7894fd4a2f7cd8 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 16:47:39 +0200 Subject: [PATCH 23/36] Decouple --query from --doc Signed-off-by: Fengyun Liu --- stack-lang/cli/Main.scala | 34 +++- stack-lang/doc/Compiler.scala | 61 +----- stack-lang/query/Compiler.scala | 65 +++++++ stack-lang/{doc => query}/JsonUtil.scala | 2 +- .../{doc/DocQuery.scala => query/Query.scala} | 4 +- tests/custom/doc-query/all.json.check | 180 ------------------ tests/custom/doc-query/private.json.check | 12 -- tests/custom/doc-query/test.sh | 55 ++---- 8 files changed, 118 insertions(+), 295 deletions(-) create mode 100644 stack-lang/query/Compiler.scala rename stack-lang/{doc => query}/JsonUtil.scala (97%) rename stack-lang/{doc/DocQuery.scala => query/Query.scala} (99%) delete mode 100644 tests/custom/doc-query/all.json.check delete mode 100644 tests/custom/doc-query/private.json.check diff --git a/stack-lang/cli/Main.scala b/stack-lang/cli/Main.scala index c7d26149..db0ae041 100644 --- a/stack-lang/cli/Main.scala +++ b/stack-lang/cli/Main.scala @@ -120,6 +120,7 @@ object Main: case Some(backend) => backend match case Backend.Doc => doc.Compiler.main(flags.args) + case Backend.Query => query.Compiler.main(flags.args) case Backend.Ruby => ruby.Compiler.main(flags.args) case Backend.Python => python.Compiler.main(flags.args) case Backend.JS => js.Compiler.main(flags.args) @@ -203,6 +204,7 @@ object Main: enum Backend: case Doc + case Query case Ruby case Python case JS @@ -220,30 +222,47 @@ object Main: var remaining = List.empty[String] var i = 0 + def selectBackend(next: Backend): Unit = + backend match + case None => + backend = Some(next) + case Some(Backend.Doc) if next == Backend.Query => + backend = Some(Backend.Query) + case Some(Backend.Query) if next == Backend.Doc => + case Some(current) if current == next => + case Some(current) => + System.err.println(s"Error: conflicting compile backends: ${current.toString.toLowerCase} and ${next.toString.toLowerCase}") + System.exit(1) + while i < args.length do args(i) match case "--doc" => - backend = Some(Backend.Doc) + selectBackend(Backend.Doc) + i += 1 + + case "--query" => + selectBackend(Backend.Query) + remaining = remaining :+ args(i) i += 1 case "--ruby" => - backend = Some(Backend.Ruby) + selectBackend(Backend.Ruby) i += 1 case "--python" => - backend = Some(Backend.Python) + selectBackend(Backend.Python) i += 1 case "--js" => - backend = Some(Backend.JS) + selectBackend(Backend.JS) i += 1 case "--stack" => - backend = Some(Backend.LinuxX86Stack) + selectBackend(Backend.LinuxX86Stack) i += 1 case "--reg" => - backend = Some(Backend.LinuxX86Reg) + selectBackend(Backend.LinuxX86Reg) i += 1 case other => @@ -274,7 +293,8 @@ object Main: | jo versions use Switch the active compiler version | jo versions remove Remove an installed compiler version | jo compile [options] Compile application or library - | jo compile --doc [options] [files...] Generate documentation from source files or queried SAST libraries + | jo compile --query [files...] Query documentation from source files or SAST libraries + | jo compile --doc [options] Generate HTML documentation from source files | jo doc [module] Generate module documentation | jo help Show this help message | diff --git a/stack-lang/doc/Compiler.scala b/stack-lang/doc/Compiler.scala index 27cf9ba5..cd3438c7 100644 --- a/stack-lang/doc/Compiler.scala +++ b/stack-lang/doc/Compiler.scala @@ -2,7 +2,6 @@ package doc import sast.* import sast.Trees.FileUnit - import typing.Typer import reporting.Reporter import reporting.Config @@ -18,18 +17,9 @@ object Compiler: val readme: Config.StringSetting = Config.StringSetting("--readme", "", "markdown file to use as home page") val includePrivate: Config.BooleanSetting = Config.BooleanSetting("--include-private", false, "include private symbols") val includeSource: Config.BooleanSetting = Config.BooleanSetting("--include-source", false, "embed source code") - val format: Config.StringSetting = DocFormatSetting("--format", "html", "documentation output format: html or json") - val query: Config.StringSetting = Config.StringSetting("--query", "", "comma-separated documentation selectors for JSON output") val docOptions: List[cli.OptionParser.Setting[?]] = - outputDir :: title :: readme :: includePrivate :: includeSource :: format :: query :: Config.commonOptions - - class DocFormatSetting(flag: String, default: String, desc: String) - extends Config.StringSetting(flag, default, desc): - override def validate()(using cf: Config, rp: Reporter): Unit = - val fmt = value - if fmt != "html" && fmt != "json" then - rp.error(s"Option $flag must be one of: html, json") + outputDir :: title :: readme :: includePrivate :: includeSource :: Config.commonOptions def main(args: Array[String]): Unit = given Reporter = Reporter.createReporter() @@ -38,20 +28,17 @@ object Compiler: given Config = config - if sources.isEmpty && query.value.trim.isEmpty then + if sources.isEmpty then println("Usage: jo doc [options]") println() println("Options:") println(" --out Output directory (default: docs)") println(" --title Project title for documentation") - println(" --format Output format (default: html)") - println(" --query JSON selectors, e.g. jo.py,file:src/API.jo") println(" --include-private Include private symbols") println(" --include-source Embed source code in output") println() println("Examples:") println(" jo doc lib/Core.jo lib/List.jo --out site/api") - println(" jo doc --query jo.List") System.exit(1) Reporter.monitor(): @@ -61,54 +48,14 @@ object Compiler: def compile(sources: List[String])(using rp: Reporter, config: Config): Unit = val rootNameTable = new NameTable given lazyDefn: Definitions.Lazy = Definitions.Lazy(rootNameTable) - val queryText = query.value.trim - val jsonOutput = format.value == "json" || queryText.nonEmpty - - def writeJson(units: List[FileUnit], filter: DocQuery.Filter)(using Definitions): Unit = - val out = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8)) - DocQuery.emitJson(units, filter, includePrivate.value, out) - out.flush() - - if jsonOutput && sources.isEmpty && Config.libPaths.value.isEmpty then - if outputDir.value != outputDir.default then - Reporter.error("--out is only supported with --format html") - return - - DocQuery.reportNoMatches(queryText) - return // Parse and type check - val (units, delayedUnits) = sources |> Typer.parseStep |> Typer.typeStep + val (units, _) = sources |> Typer.parseStep |> Typer.typeStep if rp.hasErrors then return given defn: Definitions = lazyDefn.value - - if jsonOutput then - if outputDir.value != outputDir.default then - Reporter.error("--out is only supported with --format html") - return - - val filter = DocQuery.parse(queryText, defn.rootNameTable) - if rp.hasErrors then return - - val libraryUnits = - if queryText.isEmpty then - Nil - else - delayedUnits.forceIf: unit => - filter.selectsFile(unit.sourceFile) || - filter.symbols.exists: querySymbol => - unit.owner.containedIn(querySymbol) || querySymbol.containedIn(unit.owner) - - delayedUnits.force() - - val filteredUnits = DocQuery.filterUnits(units, libraryUnits, filter) - if rp.hasErrors then return - writeJson(filteredUnits, filter) - - else - generateHtmlDoc(units) + generateHtmlDoc(units) def generateHtmlDoc(units: List[FileUnit])(using Config, Definitions): Unit = val outputPath = Paths.get(outputDir.value) diff --git a/stack-lang/query/Compiler.scala b/stack-lang/query/Compiler.scala new file mode 100644 index 00000000..27ed2c19 --- /dev/null +++ b/stack-lang/query/Compiler.scala @@ -0,0 +1,65 @@ +package query + +import sast.* +import sast.Trees.FileUnit + +import typing.Typer +import reporting.{Config, Reporter} + +import java.io.{OutputStreamWriter, PrintWriter} +import java.nio.charset.StandardCharsets + +object Compiler: + val selectors: Config.StringSetting = + Config.StringSetting("--query", "", "comma-separated documentation selectors") + + val queryOptions: List[cli.OptionParser.Setting[?]] = + selectors :: Config.commonOptions + + def main(args: Array[String]): Unit = + given Reporter = Reporter.createReporter() + + val (config, sources) = cli.OptionParser.parseConfig(args, queryOptions) + given Config = config + + val queryText = selectors.value.trim + if queryText.isEmpty then + println("Usage: jo compile --query [files...] [options]") + println() + println("Options:") + println(" --query Select symbols or source files, e.g. jo.List.map,file:Byte.jo") + System.exit(1) + + Reporter.monitor(): + compile(queryText, sources) + + def compile(queryText: String, sources: List[String])(using rp: Reporter, config: Config): Unit = + val rootNameTable = new NameTable + given lazyDefn: Definitions.Lazy = Definitions.Lazy(rootNameTable) + + if sources.isEmpty && Config.libPaths.value.isEmpty then + Query.reportNoMatches(queryText) + return + + val (units, delayedUnits) = sources |> Typer.parseStep |> Typer.typeStep + if rp.hasErrors then return + + given defn: Definitions = lazyDefn.value + val filter = Query.parse(queryText, defn.rootNameTable) + if rp.hasErrors then return + + delayedUnits.forceIf: unit => + filter.selectsFile(unit.sourceFile) || + filter.symbols.exists: querySymbol => + unit.owner.containedIn(querySymbol) || querySymbol.containedIn(unit.owner) + + val libraryUnits = delayedUnits.force() + val filteredUnits = Query.filterUnits(units, libraryUnits, filter) + if rp.hasErrors then return + + writeJson(filteredUnits, filter) + + private def writeJson(units: List[FileUnit], filter: Query.Filter)(using Reporter, Definitions): Unit = + val out = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8)) + Query.emitJson(units, filter, false, out) + out.flush() diff --git a/stack-lang/doc/JsonUtil.scala b/stack-lang/query/JsonUtil.scala similarity index 97% rename from stack-lang/doc/JsonUtil.scala rename to stack-lang/query/JsonUtil.scala index 857ac291..f1fa8afb 100644 --- a/stack-lang/doc/JsonUtil.scala +++ b/stack-lang/query/JsonUtil.scala @@ -1,4 +1,4 @@ -package doc +package query import java.io.PrintWriter diff --git a/stack-lang/doc/DocQuery.scala b/stack-lang/query/Query.scala similarity index 99% rename from stack-lang/doc/DocQuery.scala rename to stack-lang/query/Query.scala index b52e208b..82270dab 100644 --- a/stack-lang/doc/DocQuery.scala +++ b/stack-lang/query/Query.scala @@ -1,4 +1,4 @@ -package doc +package query import reporting.Reporter import sast.* @@ -12,7 +12,7 @@ import java.io.PrintWriter import java.nio.file.Paths import scala.collection.mutable -object DocQuery: +object Query: case class Filter(files: List[String], symbols: List[Symbol]): def isEmpty: Boolean = files.isEmpty && symbols.isEmpty diff --git a/tests/custom/doc-query/all.json.check b/tests/custom/doc-query/all.json.check deleted file mode 100644 index dafc12aa..00000000 --- a/tests/custom/doc-query/all.json.check +++ /dev/null @@ -1,180 +0,0 @@ -[ - { - "name": "DocQueryAPI.defaultLimit", - "kind": "param", - "signature": "param defaultLimit: Int", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 4 }, - "visibility": "public", - "flags": ["context"], - "annotations": [], - "doc": "Default limit used by context-aware helpers." - }, - { - "name": "DocQueryAPI.Path", - "kind": "type", - "signature": "type Path = String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 7 }, - "visibility": "public", - "flags": [], - "annotations": [], - "doc": "File path alias used in host calls." - }, - { - "name": "DocQueryAPI.Positive", - "kind": "pattern", - "signature": "pattern Positive(value: Int): Partial[Int] receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 10 }, - "visibility": "public", - "flags": ["fun"], - "annotations": [], - "doc": "Match positive integers." - }, - { - "name": "DocQueryAPI.HostOps", - "kind": "section", - "signature": "section HostOps", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 14 }, - "visibility": "public", - "flags": ["section"], - "annotations": [], - "doc": "Host-backed operations grouped in a section.", - "members": [ - { - "name": "DocQueryAPI.HostOps.remoteSize", - "kind": "def", - "signature": "def remoteSize(path: Path): Int receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 16 }, - "visibility": "public", - "flags": ["defer", "fun"], - "annotations": [], - "doc": "Return a remote file size." - }, - { - "name": "DocQueryAPI.HostOps.normalizeLimit", - "kind": "def", - "signature": "def normalizeLimit(value: Int): Int receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 19 }, - "visibility": "public", - "flags": ["fun"], - "annotations": [], - "doc": "Clamp a caller supplied limit." - } - ] - }, - { - "name": "DocQueryAPI.describe", - "kind": "def", - "signature": "def describe[T](value: T)(auto show: Show[T] with [[T].toString]): String receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, - "visibility": "public", - "flags": ["fun"], - "annotations": [], - "doc": "Render a value with auto formatting and context." - }, - { - "name": "DocQueryAPI.Singleton", - "kind": "class", - "signature": "object Singleton", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 28 }, - "visibility": "public", - "flags": ["object", "class"], - "annotations": [], - "doc": "Shared singleton helpers.", - "members": [ - { - "name": "DocQueryAPI.Singleton.", - "kind": "constructor", - "signature": "constructor(): Singleton receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 28 }, - "visibility": "public", - "flags": ["fun", "method", "constructor"], - "annotations": [], - "doc": null - }, - { - "name": "DocQueryAPI.Singleton.label", - "kind": "def", - "signature": "def label(): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 30 }, - "visibility": "public", - "flags": ["fun", "method"], - "annotations": [], - "doc": "Return the singleton label." - } - ] - }, - { - "name": "DocQueryAPI.FileLike", - "kind": "class", - "signature": "interface FileLike", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 35 }, - "visibility": "public", - "flags": ["interface"], - "annotations": [{ "name": "jo.py.interop", "args": [] }], - "doc": "File operations exposed to a Python-backed host.", - "members": [ - { - "name": "DocQueryAPI.FileLike.readText", - "kind": "def", - "signature": "def readText(path: String, limit: Int = 10): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, - "visibility": "public", - "flags": ["defer", "fun", "method"], - "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], - "doc": "Read a text file with an optional limit." - } - ] - }, - { - "name": "DocQueryAPI.Box", - "kind": "class", - "signature": "class Box", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 42 }, - "visibility": "public", - "flags": ["class"], - "annotations": [], - "doc": "A small documented class.", - "members": [ - { - "name": "DocQueryAPI.Box.name", - "kind": "field", - "signature": "val name: String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 44 }, - "visibility": "public", - "flags": ["field"], - "annotations": [], - "doc": "Stored display name." - }, - { - "name": "DocQueryAPI.Box.", - "kind": "constructor", - "signature": "constructor(name: String): Box receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 47 }, - "visibility": "public", - "flags": ["fun", "method", "constructor"], - "annotations": [], - "doc": "Create a box with a name." - }, - { - "name": "DocQueryAPI.Box.label", - "kind": "def", - "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, - "visibility": "public", - "flags": ["fun", "method"], - "annotations": [], - "doc": "Render a label." - } - ] - }, - { - "name": "DocQueryAPI.helper", - "kind": "def", - "signature": "def helper(value: Int): Int receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 55 }, - "visibility": "public", - "flags": ["fun"], - "annotations": [], - "doc": "Public helper." - } -] diff --git a/tests/custom/doc-query/private.json.check b/tests/custom/doc-query/private.json.check deleted file mode 100644 index 2607bee3..00000000 --- a/tests/custom/doc-query/private.json.check +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "name": "DocQueryAPI.hidden", - "kind": "def", - "signature": "def hidden(): Int receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 58 }, - "visibility": "private", - "flags": ["fun"], - "annotations": [], - "doc": "Hidden helper." - } -] diff --git a/tests/custom/doc-query/test.sh b/tests/custom/doc-query/test.sh index 2fe4939c..cea64489 100755 --- a/tests/custom/doc-query/test.sh +++ b/tests/custom/doc-query/test.sh @@ -33,16 +33,12 @@ echo "Testing $TEST_NAME" cleanup -run_doc() { - "$PROJECT_ROOT/bin/jo" compile --doc --use-runtime-api python "$@" +run_query() { + "$PROJECT_ROOT/bin/jo" compile --use-runtime-api python "$@" } -run_plain_doc() { - "$PROJECT_ROOT/bin/jo" compile --doc "$@" -} - -run_json_doc() { - "$PROJECT_ROOT/bin/jo" compile --doc --format json --use-runtime-api python "$@" +run_plain_query() { + "$PROJECT_ROOT/bin/jo" compile "$@" } expect_fail() { @@ -58,22 +54,19 @@ cd "$PROJECT_ROOT" "$PROJECT_ROOT/bin/jo" compile --sast "$SAST_DIR" --use-runtime-api python "$API_FILE" > "$SAST_LOG" -run_json_doc "$API_FILE" > "$DIR/all.json" -run_json_doc --query "DocQueryAPI.*,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/structural.json" -run_json_doc --query "file:$API_FILE" "$API_FILE" "$EXTRA_FILE" > "$DIR/file.json" -run_json_doc --query "file:$PROJECT_ROOT/$API_FILE" "$API_FILE" "$EXTRA_FILE" > "$DIR/file-absolute.json" -run_json_doc --query "DocQueryAPI.describe" "$API_FILE" > "$DIR/describe.json" -run_json_doc --query "DocQueryAPI.Box.label" "$API_FILE" > "$DIR/exact-member.json" -run_json_doc --query "DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/interface-member.json" -run_json_doc --query "DocQueryAPI.Box.name" "$API_FILE" > "$DIR/exact-field.json" -run_json_doc --query "DocQueryCompanion.Value.label" "$COMPANION_FILE" > "$DIR/companion-member.json" -run_json_doc --include-private --query "DocQueryAPI.hidden" "$API_FILE" > "$DIR/private.json" -run_doc --query "DocQueryAPI,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/query-implies-json.json" -run_doc --lib "$SAST_DIR" --query "DocQueryAPI" > "$DIR/lib-query.json" -run_doc --lib "$SAST_DIR" --query "DocQueryAPI.Box.label" > "$DIR/lib-exact-member.json" -run_doc --lib "$SAST_DIR" --query "file:api.jo" > "$DIR/lib-file-query.json" - -diff -u "$DIR/all.json.check" "$DIR/all.json" +run_query --query "DocQueryAPI.*,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/structural.json" +run_query --query "file:$API_FILE" "$API_FILE" "$EXTRA_FILE" > "$DIR/file.json" +run_query --query "file:$PROJECT_ROOT/$API_FILE" "$API_FILE" "$EXTRA_FILE" > "$DIR/file-absolute.json" +run_query --query "DocQueryAPI.describe" "$API_FILE" > "$DIR/describe.json" +run_query --query "DocQueryAPI.Box.label" "$API_FILE" > "$DIR/exact-member.json" +run_query --query "DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/interface-member.json" +run_query --query "DocQueryAPI.Box.name" "$API_FILE" > "$DIR/exact-field.json" +run_query --query "DocQueryCompanion.Value.label" "$COMPANION_FILE" > "$DIR/companion-member.json" +run_query --query "DocQueryAPI,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/query-implies-json.json" +run_query --lib "$SAST_DIR" --query "DocQueryAPI" > "$DIR/lib-query.json" +run_query --lib "$SAST_DIR" --query "DocQueryAPI.Box.label" > "$DIR/lib-exact-member.json" +run_query --lib "$SAST_DIR" --query "file:api.jo" > "$DIR/lib-file-query.json" + diff -u "$DIR/structural.json.check" "$DIR/structural.json" diff -u "$DIR/structural.json.check" "$DIR/file.json" diff -u "$DIR/structural.json.check" "$DIR/file-absolute.json" @@ -86,21 +79,11 @@ diff -u "$DIR/exact-member.json.check" "$DIR/exact-member.json" diff -u "$DIR/interface-member.json.check" "$DIR/interface-member.json" diff -u "$DIR/exact-field.json.check" "$DIR/exact-field.json" diff -u "$DIR/companion-member.json.check" "$DIR/companion-member.json" -diff -u "$DIR/private.json.check" "$DIR/private.json" -expect_fail "$FAIL_LOG" "$PROJECT_ROOT/bin/jo" compile --doc --format yaml "$API_FILE" -grep -q -- "Option --format must be one of: html, json" "$FAIL_LOG" - -expect_fail "$FAIL_LOG" run_json_doc --out "$SAST_DIR/out" "$API_FILE" -grep -q -- "--out is only supported with --format html" "$FAIL_LOG" - -expect_fail "$FAIL_LOG" run_plain_doc --query "NoSuchSymbol" +expect_fail "$FAIL_LOG" run_plain_query --query "NoSuchSymbol" grep -q -- "No documentation entries match symbol selector" "$FAIL_LOG" -expect_fail "$FAIL_LOG" run_plain_doc --no-stdlib --query "NoSuchSymbol" +expect_fail "$FAIL_LOG" run_plain_query --no-stdlib --query "NoSuchSymbol" grep -q -- "No documentation entries match symbol selector" "$FAIL_LOG" -expect_fail "$FAIL_LOG" run_plain_doc --format json -grep -q -- "Usage: jo doc" "$FAIL_LOG" - echo " ✓ All tests passed for $TEST_NAME" From e8229fd574e842b38f75c4ff1427ba2f805a2335 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 17:00:10 +0200 Subject: [PATCH 24/36] Update docs Signed-off-by: Fengyun Liu --- docs/usage/commands/compile.md | 56 ++++++++++++++---------- docs/usage/reference/compiler-options.md | 27 +++++++----- 2 files changed, 50 insertions(+), 33 deletions(-) diff --git a/docs/usage/commands/compile.md b/docs/usage/commands/compile.md index da1f30a3..a6b9ebaa 100644 --- a/docs/usage/commands/compile.md +++ b/docs/usage/commands/compile.md @@ -12,10 +12,12 @@ jo compile [--sast ] ... [--lib ]... jo compile --python|--ruby|--js [--sast ] ... \ [--lib ]... [--link-lib ]... [--link =]... -o -# Generate documentation from source files (experimental) -jo compile --doc [--format html|json] [--query ] [--out ] \ - [--title ] [--readme ] [--include-private] \ - [--include-source] [...] +# Generate HTML documentation from source files (experimental) +jo compile --doc [--out ] [--title ] [--readme ] \ + [--include-private] [--include-source] ... + +# Query API information as JSON (experimental) +jo compile --query [...] [--lib ]... ``` Without a backend flag, the compiler type-checks only. With a backend flag, it produces an executable or script. `--sast ` is optional in both cases — if present, `.sast` files are written to `` alongside the primary output. @@ -44,24 +46,34 @@ Experimental. | Flag | Description | |------|-------------| -| `--doc` | Generate documentation instead of normal compile output | -| `--format html|json` | Documentation output format. Default: `html` | -| `--query ` | Comma-separated JSON selectors, such as `MyAPI` or `file:src/API.jo`; implies `--format json` | +| `--doc` | Generate HTML documentation instead of normal compile output | | `--out ` | Documentation output directory | | `--title ` | Documentation title | | `--readme ` | Markdown file to use as the generated documentation home page | | `--include-private` | Include private symbols | | `--include-source` | Embed source code in output | -`--format json` writes a JSON array to stdout. `--query` implies -`--format json`. JSON output does not use `--out`. -Without `--query`, JSON output contains the public API surface from the -positional source files. With `--query`, selectors are comma-separated and may -name symbols or source files with `file:`. -Symbol selectors are resolved from the language default scope; for example `~` -is just an ordinary symbol query. A query may omit positional source files; in -that case it searches the loaded SAST libraries, including stdlib unless -`--no-stdlib` is set and any `--lib` paths. +### Query + +Experimental. + +| Flag | Description | +|------|-------------| +| `--query ` | Query comma-separated symbols or source files and write a JSON array to stdout | + +Selectors may name symbols or source files with `file:`. +Symbol selectors are dot-separated names resolved from the root namespace, such +as `jo.List.map`. Positional source files are optional. With no source files, +the query searches the loaded SAST libraries, including stdlib, any `--lib` +paths, and the runtime API selected by `--use-runtime-api`. + +A file selector may use the recorded source path, an absolute path, or a basename: + +```sh +jo compile --query file:Byte.jo +``` + +Query output includes public symbols only. ### App compilation @@ -117,22 +129,22 @@ jo compile --doc lib/Core.jo lib/List.jo \ --title "Jo Standard Library" ``` -Emit machine-readable docs for a source file: +Query selected symbols from source: ```sh -jo compile --doc --format json src/API.jo +jo compile --query 'MyAPI,jo.py' src/API.jo ``` -Emit machine-readable docs for selected symbols: +Query stdlib API information from SAST files only: ```sh -jo compile --doc --query 'MyAPI,jo.py' src/API.jo +jo compile --query jo.List.map ``` -Query stdlib docs from SAST files only: +Query a standard-library source file by basename: ```sh -jo compile --doc --query jo.List +jo compile --query file:Byte.jo ``` ## Notes diff --git a/docs/usage/reference/compiler-options.md b/docs/usage/reference/compiler-options.md index 3eb0e8ce..a5429abc 100644 --- a/docs/usage/reference/compiler-options.md +++ b/docs/usage/reference/compiler-options.md @@ -28,11 +28,13 @@ Backend selectors are handled before the shared compiler options are parsed. | `--ruby` | Compile a Ruby application. | | `--python` | Compile a Python application. | | `--js` | Compile a JavaScript application. Experimental. | +| `--doc` | Generate HTML documentation from source files. Experimental. | +| `--query ` | Query API information and write JSON to stdout. Experimental. | ## Common Options Common options are accepted by type-check-only compilation, app backend compilation, -and experimental documentation generation. +documentation generation, and API queries. | Option | Form | Description | |--------|------|-------------| @@ -94,26 +96,29 @@ above plus these documentation-specific options. | Option | Form | Description | |--------|------|-------------| -| `--format ` | single value | Documentation output format. Default: `html`. | -| `--query ` | comma list | Select symbols or source files for JSON documentation output. Implies `--format json`. | | `--out ` | single value | Documentation output directory. Default: `docs`. | | `--title ` | single value | Documentation title. Default: `API Documentation`. | | `--readme ` | single value | Markdown file to use as the generated documentation home page. | | `--include-private` | flag | Include private symbols. | | `--include-source` | flag | Embed source code in the generated documentation. | -`--format json` writes a bare JSON array to stdout and rejects `--out`. -`--query` implies `--format json`. -Without `--query`, it emits the public API surface from the positional source -files. `--query` accepts comma-separated selectors: +Documentation output is always HTML. + +## Query Options + +`jo compile --query ` is an experimental API-query backend. It +accepts common compiler options and optional positional source files, and writes +a bare JSON array to stdout. Selectors are comma-separated: | Selector | Meaning | |----------|---------| -| `` | Symbol resolved from the language default scope. Aggregate symbols are emitted structurally with their members. | -| `file:` | Queryable symbols whose source file matches the path. | +| `` | Dot-separated symbol name resolved from the root namespace, such as `jo.List.map`. Aggregate symbols are emitted structurally with their members. | +| `file:` | Public symbols whose recorded source file matches the normalized path, absolute path, or basename. | -When `--query` is present, positional source files are optional. With no source -files, the query searches the loaded SAST libraries: stdlib by default, runtime +Positional source files are optional. With no source files, the query searches +the loaded SAST libraries: stdlib by default, runtime API libraries selected by `--use-runtime-api`, and any `--lib` paths. +Query output includes public symbols only. + Kind-qualified selectors such as `def:` are reserved for future use. From ee6a6bf935771e9848a688ccf015de26de23d9de Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 17:10:06 +0200 Subject: [PATCH 25/36] Refactor code Signed-off-by: Fengyun Liu --- stack-lang/cli/Main.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/stack-lang/cli/Main.scala b/stack-lang/cli/Main.scala index db0ae041..8d1e8f25 100644 --- a/stack-lang/cli/Main.scala +++ b/stack-lang/cli/Main.scala @@ -226,12 +226,10 @@ object Main: backend match case None => backend = Some(next) - case Some(Backend.Doc) if next == Backend.Query => - backend = Some(Backend.Query) - case Some(Backend.Query) if next == Backend.Doc => - case Some(current) if current == next => case Some(current) => - System.err.println(s"Error: conflicting compile backends: ${current.toString.toLowerCase} and ${next.toString.toLowerCase}") + System.err.println( + s"Error: compile backend already selected: ${current.toString.toLowerCase}; cannot select ${next.toString.toLowerCase}" + ) System.exit(1) while i < args.length do From d4b122a13ad01bf46e0bb5f328b9c495f584bcc8 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 17:18:04 +0200 Subject: [PATCH 26/36] Avoid using jo.py in examples Signed-off-by: Fengyun Liu --- docs/usage/commands/compile.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/commands/compile.md b/docs/usage/commands/compile.md index a6b9ebaa..2f020205 100644 --- a/docs/usage/commands/compile.md +++ b/docs/usage/commands/compile.md @@ -132,7 +132,7 @@ jo compile --doc lib/Core.jo lib/List.jo \ Query selected symbols from source: ```sh -jo compile --query 'MyAPI,jo.py' src/API.jo +jo compile --query 'MyAPI,jo.List' src/API.jo ``` Query stdlib API information from SAST files only: From b0e8e1cea19f1a7d3155cbe69c2f927e8ec62560 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 17:28:06 +0200 Subject: [PATCH 27/36] Output end line of a symbol Signed-off-by: Fengyun Liu --- stack-lang/query/Query.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stack-lang/query/Query.scala b/stack-lang/query/Query.scala index 82270dab..3b82fb35 100644 --- a/stack-lang/query/Query.scala +++ b/stack-lang/query/Query.scala @@ -464,14 +464,14 @@ object Query: private def sourceLoc(sym: Symbol): Option[SourceLoc] = if sym.sourcePos == null then None - else Some(SourceLoc(sym.source.file, sym.sourcePos.startLine + 1)) + else Some(SourceLoc(sym.source.file, sym.sourcePos.startLine + 1, sym.sourcePos.endLine + 1)) - private case class SourceLoc(file: String, line: Int) + private case class SourceLoc(file: String, line: Int, end: Int) private def sourceJson(source: Option[SourceLoc]): String = source match - case Some(SourceLoc(file, line)) => - s"""{ "file": ${JsonUtil.string(file)}, "line": $line }""" + case Some(SourceLoc(file, line, end)) => + s"""{ "file": ${JsonUtil.string(file)}, "line": $line, "end": $end }""" case None => "null" From 0c760a4d24dda69183e44291c278cbd6ae9fa16b Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 17:35:14 +0200 Subject: [PATCH 28/36] Add end line to source Signed-off-by: Fengyun Liu --- stack-lang/query/Query.scala | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/stack-lang/query/Query.scala b/stack-lang/query/Query.scala index 3b82fb35..b08c3ede 100644 --- a/stack-lang/query/Query.scala +++ b/stack-lang/query/Query.scala @@ -7,6 +7,7 @@ import sast.Trees.* import sast.Types.* import sast.Denotations.* import sast.Flags +import ast.Positions.Span import java.io.PrintWriter import java.nio.file.Paths @@ -361,17 +362,17 @@ object Query: private def emitNamespace(unit: FileUnit, out: PrintWriter, indent: String)(using Definitions): Unit = val members = sortMembers(unit.defs.map(defn => defn: Def | FieldDecl)) - emitSymbol(unit.owner, "namespace", "namespace " + unit.owner.fullName, Nil, members, out, indent) + emitSymbol(unit.owner, "namespace", "namespace " + unit.owner.fullName, Nil, members, sourceLoc(unit.owner), out, indent) private def emitDef(defn: Def, out: PrintWriter, indent: String)(using Definitions): Unit = val members = sortMembers(memberNodes(defn)) val views = defn match case cd: ClassDef => cd.views.map(_.tpe.show) case _ => Nil - emitSymbol(defn.symbol, kind(defn), signature(defn), views, members, out, indent) + emitSymbol(defn.symbol, kind(defn), signature(defn), views, members, sourceLoc(defn.symbol, defn.span), out, indent) private def emitFieldDecl(field: FieldDecl, out: PrintWriter, indent: String)(using Definitions): Unit = - emitSymbol(field.symbol, "field", fieldSignature(field), Nil, Nil, out, indent) + emitSymbol(field.symbol, "field", fieldSignature(field), Nil, Nil, sourceLoc(field.symbol, field.span), out, indent) private def emitSymbol( sym: Symbol, @@ -379,6 +380,7 @@ object Query: signature: String, views: List[String], members: List[Def | FieldDecl], + source: Option[SourceLoc], out: PrintWriter, indent: String, )(using Definitions): Unit = @@ -387,7 +389,7 @@ object Query: emitField("name", JsonUtil.string(sym.fullName), out, next) emitField("kind", JsonUtil.string(kind), out, next) emitField("signature", JsonUtil.string(signature), out, next) - emitField("source", sourceJson(sourceLoc(sym)), out, next) + emitField("source", sourceJson(source), out, next) emitField("visibility", JsonUtil.string(visibility(sym)), out, next) emitField("flags", stringArray(Flags.flagStrings(sym.flags)), out, next) emitField("annotations", annotationsJson(sym), out, next) @@ -466,6 +468,12 @@ object Query: if sym.sourcePos == null then None else Some(SourceLoc(sym.source.file, sym.sourcePos.startLine + 1, sym.sourcePos.endLine + 1)) + private def sourceLoc(sym: Symbol, span: Span): Option[SourceLoc] = + if sym.sourcePos == null then None + else + val pos = span.toPos(using sym.source) + Some(SourceLoc(sym.source.file, pos.startLine + 1, pos.endLine + 1)) + private case class SourceLoc(file: String, line: Int, end: Int) private def sourceJson(source: Option[SourceLoc]): String = From f4ba159c0cc15014633ab7bf677b7466d5a4bf2d Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 17:35:30 +0200 Subject: [PATCH 29/36] Update check files Signed-off-by: Fengyun Liu --- .../doc-query/companion-member.json.check | 2 +- tests/custom/doc-query/describe.json.check | 2 +- tests/custom/doc-query/exact-field.json.check | 4 +-- .../custom/doc-query/exact-member.json.check | 2 +- .../doc-query/interface-member.json.check | 2 +- .../doc-query/lib-exact-member.json.check | 2 +- tests/custom/doc-query/lib-query.json.check | 34 +++++++++--------- tests/custom/doc-query/structural.json.check | 36 +++++++++---------- 8 files changed, 42 insertions(+), 42 deletions(-) diff --git a/tests/custom/doc-query/companion-member.json.check b/tests/custom/doc-query/companion-member.json.check index 14bf0778..336c79de 100644 --- a/tests/custom/doc-query/companion-member.json.check +++ b/tests/custom/doc-query/companion-member.json.check @@ -3,7 +3,7 @@ "name": "DocQueryCompanion.Value.label", "kind": "def", "signature": "def label(): String receives none", - "source": { "file": "tests/custom/doc-query/companion.jo", "line": 6 }, + "source": { "file": "tests/custom/doc-query/companion.jo", "line": 6, "end": 6 }, "visibility": "public", "flags": ["fun", "method"], "annotations": [], diff --git a/tests/custom/doc-query/describe.json.check b/tests/custom/doc-query/describe.json.check index 0c3c8e20..569d98ab 100644 --- a/tests/custom/doc-query/describe.json.check +++ b/tests/custom/doc-query/describe.json.check @@ -3,7 +3,7 @@ "name": "DocQueryAPI.describe", "kind": "def", "signature": "def describe[T](value: T)(auto show: Show[T] with [[T].toString]): String receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24, "end": 25 }, "visibility": "public", "flags": ["fun"], "annotations": [], diff --git a/tests/custom/doc-query/exact-field.json.check b/tests/custom/doc-query/exact-field.json.check index 5aa8cbad..0abcd92b 100644 --- a/tests/custom/doc-query/exact-field.json.check +++ b/tests/custom/doc-query/exact-field.json.check @@ -3,7 +3,7 @@ "name": "DocQueryAPI.Box", "kind": "class", "signature": "class Box", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 42 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 42, "end": 51 }, "visibility": "public", "flags": ["class"], "annotations": [], @@ -13,7 +13,7 @@ "name": "DocQueryAPI.Box.name", "kind": "field", "signature": "val name: String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 44 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 44, "end": 44 }, "visibility": "public", "flags": ["field"], "annotations": [], diff --git a/tests/custom/doc-query/exact-member.json.check b/tests/custom/doc-query/exact-member.json.check index 369b7dcf..733cf5e3 100644 --- a/tests/custom/doc-query/exact-member.json.check +++ b/tests/custom/doc-query/exact-member.json.check @@ -3,7 +3,7 @@ "name": "DocQueryAPI.Box.label", "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 51, "end": 51 }, "visibility": "public", "flags": ["fun", "method"], "annotations": [], diff --git a/tests/custom/doc-query/interface-member.json.check b/tests/custom/doc-query/interface-member.json.check index b218ba44..affdb7c8 100644 --- a/tests/custom/doc-query/interface-member.json.check +++ b/tests/custom/doc-query/interface-member.json.check @@ -3,7 +3,7 @@ "name": "DocQueryAPI.FileLike.readText", "kind": "def", "signature": "def readText(path: String, limit: Int = 10): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 38, "end": 38 }, "visibility": "public", "flags": ["defer", "fun", "method"], "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], diff --git a/tests/custom/doc-query/lib-exact-member.json.check b/tests/custom/doc-query/lib-exact-member.json.check index a4204934..cc888c3a 100644 --- a/tests/custom/doc-query/lib-exact-member.json.check +++ b/tests/custom/doc-query/lib-exact-member.json.check @@ -3,7 +3,7 @@ "name": "DocQueryAPI.Box.label", "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 51, "end": 51 }, "visibility": "public", "flags": ["loaded", "fun", "method"], "annotations": [], diff --git a/tests/custom/doc-query/lib-query.json.check b/tests/custom/doc-query/lib-query.json.check index 6d3497e9..572b91e7 100644 --- a/tests/custom/doc-query/lib-query.json.check +++ b/tests/custom/doc-query/lib-query.json.check @@ -13,7 +13,7 @@ "name": "DocQueryAPI.defaultLimit", "kind": "param", "signature": "param defaultLimit: Int", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 4 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 4, "end": 4 }, "visibility": "public", "flags": ["context"], "annotations": [], @@ -23,7 +23,7 @@ "name": "DocQueryAPI.Path", "kind": "type", "signature": "type Path = String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 7 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 7, "end": 7 }, "visibility": "public", "flags": [], "annotations": [], @@ -33,7 +33,7 @@ "name": "DocQueryAPI.Positive", "kind": "pattern", "signature": "pattern Positive(value: Int): Partial[Int] receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 10 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 10, "end": 11 }, "visibility": "public", "flags": ["fun"], "annotations": [], @@ -43,7 +43,7 @@ "name": "DocQueryAPI.HostOps", "kind": "section", "signature": "section HostOps", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 14 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 14, "end": 20 }, "visibility": "public", "flags": ["section"], "annotations": [], @@ -53,7 +53,7 @@ "name": "DocQueryAPI.HostOps.remoteSize", "kind": "def", "signature": "def remoteSize(path: Path): Int receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 16 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 16, "end": 16 }, "visibility": "public", "flags": ["defer", "loaded", "fun"], "annotations": [], @@ -63,7 +63,7 @@ "name": "DocQueryAPI.HostOps.normalizeLimit", "kind": "def", "signature": "def normalizeLimit(value: Int): Int receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 19 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 19, "end": 20 }, "visibility": "public", "flags": ["loaded", "fun"], "annotations": [], @@ -75,7 +75,7 @@ "name": "DocQueryAPI.describe", "kind": "def", "signature": "def describe[T](value: T)(auto show: Show[T] with [[T].toString]): String receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24, "end": 25 }, "visibility": "public", "flags": ["loaded", "fun"], "annotations": [], @@ -85,7 +85,7 @@ "name": "DocQueryAPI.Singleton", "kind": "class", "signature": "object Singleton", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 28 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 28, "end": 30 }, "visibility": "public", "flags": ["object", "class"], "annotations": [], @@ -95,7 +95,7 @@ "name": "DocQueryAPI.Singleton.", "kind": "constructor", "signature": "constructor(): Singleton receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 28 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 28, "end": 30 }, "visibility": "public", "flags": ["loaded", "fun", "method", "constructor"], "annotations": [], @@ -105,7 +105,7 @@ "name": "DocQueryAPI.Singleton.label", "kind": "def", "signature": "def label(): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 30 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 30, "end": 30 }, "visibility": "public", "flags": ["loaded", "fun", "method"], "annotations": [], @@ -117,7 +117,7 @@ "name": "DocQueryAPI.FileLike", "kind": "class", "signature": "interface FileLike", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 35 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 35, "end": 38 }, "visibility": "public", "flags": ["interface"], "annotations": [{ "name": "jo.py.interop", "args": [] }], @@ -127,7 +127,7 @@ "name": "DocQueryAPI.FileLike.readText", "kind": "def", "signature": "def readText(path: String, limit: Int = 10): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 38, "end": 38 }, "visibility": "public", "flags": ["defer", "loaded", "fun", "method"], "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], @@ -139,7 +139,7 @@ "name": "DocQueryAPI.Box", "kind": "class", "signature": "class Box", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 42 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 42, "end": 51 }, "visibility": "public", "flags": ["class"], "annotations": [], @@ -149,7 +149,7 @@ "name": "DocQueryAPI.Box.name", "kind": "field", "signature": "val name: String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 44 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 44, "end": 44 }, "visibility": "public", "flags": ["field"], "annotations": [], @@ -159,7 +159,7 @@ "name": "DocQueryAPI.Box.", "kind": "constructor", "signature": "constructor(name: String): Box receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 47 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 47, "end": 48 }, "visibility": "public", "flags": ["loaded", "fun", "method", "constructor"], "annotations": [], @@ -169,7 +169,7 @@ "name": "DocQueryAPI.Box.label", "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 51, "end": 51 }, "visibility": "public", "flags": ["loaded", "fun", "method"], "annotations": [], @@ -181,7 +181,7 @@ "name": "DocQueryAPI.helper", "kind": "def", "signature": "def helper(value: Int): Int receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 55 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 55, "end": 55 }, "visibility": "public", "flags": ["loaded", "fun"], "annotations": [], diff --git a/tests/custom/doc-query/structural.json.check b/tests/custom/doc-query/structural.json.check index 23b9fd49..d5979843 100644 --- a/tests/custom/doc-query/structural.json.check +++ b/tests/custom/doc-query/structural.json.check @@ -3,7 +3,7 @@ "name": "DocQueryAPI", "kind": "namespace", "signature": "namespace DocQueryAPI", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 1 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 1, "end": 1 }, "visibility": "public", "flags": ["namespace"], "annotations": [], @@ -13,7 +13,7 @@ "name": "DocQueryAPI.defaultLimit", "kind": "param", "signature": "param defaultLimit: Int", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 4 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 4, "end": 4 }, "visibility": "public", "flags": ["context"], "annotations": [], @@ -23,7 +23,7 @@ "name": "DocQueryAPI.Path", "kind": "type", "signature": "type Path = String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 7 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 7, "end": 7 }, "visibility": "public", "flags": [], "annotations": [], @@ -33,7 +33,7 @@ "name": "DocQueryAPI.Positive", "kind": "pattern", "signature": "pattern Positive(value: Int): Partial[Int] receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 10 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 10, "end": 11 }, "visibility": "public", "flags": ["fun"], "annotations": [], @@ -43,7 +43,7 @@ "name": "DocQueryAPI.HostOps", "kind": "section", "signature": "section HostOps", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 14 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 14, "end": 20 }, "visibility": "public", "flags": ["section"], "annotations": [], @@ -53,7 +53,7 @@ "name": "DocQueryAPI.HostOps.remoteSize", "kind": "def", "signature": "def remoteSize(path: Path): Int receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 16 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 16, "end": 16 }, "visibility": "public", "flags": ["defer", "fun"], "annotations": [], @@ -63,7 +63,7 @@ "name": "DocQueryAPI.HostOps.normalizeLimit", "kind": "def", "signature": "def normalizeLimit(value: Int): Int receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 19 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 19, "end": 20 }, "visibility": "public", "flags": ["fun"], "annotations": [], @@ -75,7 +75,7 @@ "name": "DocQueryAPI.describe", "kind": "def", "signature": "def describe[T](value: T)(auto show: Show[T] with [[T].toString]): String receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24, "end": 25 }, "visibility": "public", "flags": ["fun"], "annotations": [], @@ -85,7 +85,7 @@ "name": "DocQueryAPI.Singleton", "kind": "class", "signature": "object Singleton", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 28 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 28, "end": 30 }, "visibility": "public", "flags": ["object", "class"], "annotations": [], @@ -95,7 +95,7 @@ "name": "DocQueryAPI.Singleton.", "kind": "constructor", "signature": "constructor(): Singleton receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 28 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 28, "end": 30 }, "visibility": "public", "flags": ["fun", "method", "constructor"], "annotations": [], @@ -105,7 +105,7 @@ "name": "DocQueryAPI.Singleton.label", "kind": "def", "signature": "def label(): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 30 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 30, "end": 30 }, "visibility": "public", "flags": ["fun", "method"], "annotations": [], @@ -117,7 +117,7 @@ "name": "DocQueryAPI.FileLike", "kind": "class", "signature": "interface FileLike", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 35 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 35, "end": 38 }, "visibility": "public", "flags": ["interface"], "annotations": [{ "name": "jo.py.interop", "args": [] }], @@ -127,7 +127,7 @@ "name": "DocQueryAPI.FileLike.readText", "kind": "def", "signature": "def readText(path: String, limit: Int = 10): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 38 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 38, "end": 38 }, "visibility": "public", "flags": ["defer", "fun", "method"], "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], @@ -139,7 +139,7 @@ "name": "DocQueryAPI.Box", "kind": "class", "signature": "class Box", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 42 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 42, "end": 51 }, "visibility": "public", "flags": ["class"], "annotations": [], @@ -149,7 +149,7 @@ "name": "DocQueryAPI.Box.name", "kind": "field", "signature": "val name: String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 44 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 44, "end": 44 }, "visibility": "public", "flags": ["field"], "annotations": [], @@ -159,7 +159,7 @@ "name": "DocQueryAPI.Box.", "kind": "constructor", "signature": "constructor(name: String): Box receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 47 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 47, "end": 48 }, "visibility": "public", "flags": ["fun", "method", "constructor"], "annotations": [], @@ -169,7 +169,7 @@ "name": "DocQueryAPI.Box.label", "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 51 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 51, "end": 51 }, "visibility": "public", "flags": ["fun", "method"], "annotations": [], @@ -181,7 +181,7 @@ "name": "DocQueryAPI.helper", "kind": "def", "signature": "def helper(value: Int): Int receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 55 }, + "source": { "file": "tests/custom/doc-query/api.jo", "line": 55, "end": 55 }, "visibility": "public", "flags": ["fun"], "annotations": [], From 183e8efd4b0ae0d0cbdd978725a27b398ff9d49f Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 17:57:52 +0200 Subject: [PATCH 30/36] Doc for --fields Signed-off-by: Fengyun Liu --- docs/usage/commands/compile.md | 7 +++++++ docs/usage/reference/compiler-options.md | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/docs/usage/commands/compile.md b/docs/usage/commands/compile.md index 2f020205..f2cc55e8 100644 --- a/docs/usage/commands/compile.md +++ b/docs/usage/commands/compile.md @@ -60,6 +60,7 @@ Experimental. | Flag | Description | |------|-------------| | `--query ` | Query comma-separated symbols or source files and write a JSON array to stdout | +| `--fields ` | Select comma-separated query output fields (default: `name,signature,doc`) | Selectors may name symbols or source files with `file:`. Symbol selectors are dot-separated names resolved from the root namespace, such @@ -141,6 +142,12 @@ Query stdlib API information from SAST files only: jo compile --query jo.List.map ``` +Select output fields: + +```sh +jo compile --query jo.List.map --fields name,signature,source,doc +``` + Query a standard-library source file by basename: ```sh diff --git a/docs/usage/reference/compiler-options.md b/docs/usage/reference/compiler-options.md index a5429abc..999c7d2d 100644 --- a/docs/usage/reference/compiler-options.md +++ b/docs/usage/reference/compiler-options.md @@ -120,5 +120,9 @@ the loaded SAST libraries: stdlib by default, runtime API libraries selected by `--use-runtime-api`, and any `--lib` paths. Query output includes public symbols only. +Each symbol includes `name`, `signature`, and `doc` by default. Use +`--fields` to select any combination of `name`, `kind`, `signature`, `source`, +`visibility`, `flags`, `annotations`, and `doc`. Field order in the option does +not affect JSON output order. Classes with views include `views` automatically. Kind-qualified selectors such as `def:` are reserved for future use. From d904132fde8a9d83fac58d43cd2e39599b10dea7 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 18:01:26 +0200 Subject: [PATCH 31/36] Support --fields option Signed-off-by: Fengyun Liu --- stack-lang/query/Compiler.scala | 17 ++++--- stack-lang/query/Query.scala | 88 +++++++++++++++++++++++---------- 2 files changed, 72 insertions(+), 33 deletions(-) diff --git a/stack-lang/query/Compiler.scala b/stack-lang/query/Compiler.scala index 27ed2c19..d33e4b08 100644 --- a/stack-lang/query/Compiler.scala +++ b/stack-lang/query/Compiler.scala @@ -13,8 +13,11 @@ object Compiler: val selectors: Config.StringSetting = Config.StringSetting("--query", "", "comma-separated documentation selectors") + val fields: Config.StringSetting = + Config.StringSetting("--fields", "name,signature,doc", "comma-separated query output fields") + val queryOptions: List[cli.OptionParser.Setting[?]] = - selectors :: Config.commonOptions + selectors :: fields :: Config.commonOptions def main(args: Array[String]): Unit = given Reporter = Reporter.createReporter() @@ -28,12 +31,14 @@ object Compiler: println() println("Options:") println(" --query Select symbols or source files, e.g. jo.List.map,file:Byte.jo") + println(" --fields Select output fields (default: name,signature,doc)") System.exit(1) Reporter.monitor(): - compile(queryText, sources) + val selectedFields = Query.parseFields(fields.value) + if !summon[Reporter].hasErrors then compile(queryText, selectedFields, sources) - def compile(queryText: String, sources: List[String])(using rp: Reporter, config: Config): Unit = + def compile(queryText: String, fields: Set[String], sources: List[String])(using rp: Reporter, config: Config): Unit = val rootNameTable = new NameTable given lazyDefn: Definitions.Lazy = Definitions.Lazy(rootNameTable) @@ -57,9 +62,9 @@ object Compiler: val filteredUnits = Query.filterUnits(units, libraryUnits, filter) if rp.hasErrors then return - writeJson(filteredUnits, filter) + writeJson(filteredUnits, filter, fields) - private def writeJson(units: List[FileUnit], filter: Query.Filter)(using Reporter, Definitions): Unit = + private def writeJson(units: List[FileUnit], filter: Query.Filter, fields: Set[String])(using Reporter, Definitions): Unit = val out = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8)) - Query.emitJson(units, filter, false, out) + Query.emitJson(units, filter, fields, false, out) out.flush() diff --git a/stack-lang/query/Query.scala b/stack-lang/query/Query.scala index b08c3ede..716f50c8 100644 --- a/stack-lang/query/Query.scala +++ b/stack-lang/query/Query.scala @@ -14,6 +14,21 @@ import java.nio.file.Paths import scala.collection.mutable object Query: + private val outputFields = + List("name", "kind", "signature", "source", "visibility", "flags", "annotations", "doc") + + private val availableFields = outputFields.mkString(",") + + def parseFields(rawFields: String)(using Reporter): Set[String] = + val fields = rawFields.split(",").map(_.trim).filter(_.nonEmpty).toSet + if fields.isEmpty then + Reporter.error(s"Option --fields requires at least one field. Available fields: $availableFields") + + for field <- fields.diff(outputFields.toSet).toList.sorted do + Reporter.error(s"Unknown query field: $field. Available fields: $availableFields") + + fields + case class Filter(files: List[String], symbols: List[Symbol]): def isEmpty: Boolean = files.isEmpty && symbols.isEmpty @@ -111,11 +126,17 @@ object Query: seen += unit true - def emitJson(units: List[FileUnit], filter: Filter, includePrivate: Boolean, out: PrintWriter)(using Reporter, Definitions): Unit = + def emitJson( + units: List[FileUnit], + filter: Filter, + fields: Set[String], + includePrivate: Boolean, + out: PrintWriter, + )(using Reporter, Definitions): Unit = val trimmedUnits = trimUnits(units, filter, includePrivate) val sortedTargets = sortRoots(jsonRoots(trimmedUnits, filter)) out.println("[") - emitRootList(sortedTargets, out, " ") + emitRootList(sortedTargets, fields, out, " ") if sortedTargets.nonEmpty then out.println() out.println("]") @@ -342,37 +363,47 @@ object Query: val line = source.map(_.line).getOrElse(0) (file, line, kind, sym.fullName) - private def emitRootList(roots: List[FileUnit | Def], out: PrintWriter, indent: String)(using Definitions): Unit = + private def emitRootList( + roots: List[FileUnit | Def], + fields: Set[String], + out: PrintWriter, + indent: String, + )(using Definitions): Unit = var first = true for root <- roots do if !first then out.println(",") first = false root match - case unit: FileUnit => emitNamespace(unit, out, indent) - case defn: Def => emitDef(defn, out, indent) + case unit: FileUnit => emitNamespace(unit, fields, out, indent) + case defn: Def => emitDef(defn, fields, out, indent) - private def emitMemberList(members: List[Def | FieldDecl], out: PrintWriter, indent: String)(using Definitions): Unit = + private def emitMemberList( + members: List[Def | FieldDecl], + fields: Set[String], + out: PrintWriter, + indent: String, + )(using Definitions): Unit = var first = true for member <- members do if !first then out.println(",") first = false member match - case defn: Def => emitDef(defn, out, indent) - case field: FieldDecl => emitFieldDecl(field, out, indent) + case defn: Def => emitDef(defn, fields, out, indent) + case field: FieldDecl => emitFieldDecl(field, fields, out, indent) - private def emitNamespace(unit: FileUnit, out: PrintWriter, indent: String)(using Definitions): Unit = + private def emitNamespace(unit: FileUnit, fields: Set[String], out: PrintWriter, indent: String)(using Definitions): Unit = val members = sortMembers(unit.defs.map(defn => defn: Def | FieldDecl)) - emitSymbol(unit.owner, "namespace", "namespace " + unit.owner.fullName, Nil, members, sourceLoc(unit.owner), out, indent) + emitSymbol(unit.owner, "namespace", "namespace " + unit.owner.fullName, Nil, members, sourceLoc(unit.owner), fields, out, indent) - private def emitDef(defn: Def, out: PrintWriter, indent: String)(using Definitions): Unit = + private def emitDef(defn: Def, fields: Set[String], out: PrintWriter, indent: String)(using Definitions): Unit = val members = sortMembers(memberNodes(defn)) val views = defn match case cd: ClassDef => cd.views.map(_.tpe.show) case _ => Nil - emitSymbol(defn.symbol, kind(defn), signature(defn), views, members, sourceLoc(defn.symbol, defn.span), out, indent) + emitSymbol(defn.symbol, kind(defn), signature(defn), views, members, sourceLoc(defn.symbol, defn.span), fields, out, indent) - private def emitFieldDecl(field: FieldDecl, out: PrintWriter, indent: String)(using Definitions): Unit = - emitSymbol(field.symbol, "field", fieldSignature(field), Nil, Nil, sourceLoc(field.symbol, field.span), out, indent) + private def emitFieldDecl(field: FieldDecl, fields: Set[String], out: PrintWriter, indent: String)(using Definitions): Unit = + emitSymbol(field.symbol, "field", fieldSignature(field), Nil, Nil, sourceLoc(field.symbol, field.span), fields, out, indent) private def emitSymbol( sym: Symbol, @@ -381,26 +412,29 @@ object Query: views: List[String], members: List[Def | FieldDecl], source: Option[SourceLoc], + fields: Set[String], out: PrintWriter, indent: String, )(using Definitions): Unit = val next = indent + " " out.println(indent + "{") - emitField("name", JsonUtil.string(sym.fullName), out, next) - emitField("kind", JsonUtil.string(kind), out, next) - emitField("signature", JsonUtil.string(signature), out, next) - emitField("source", sourceJson(source), out, next) - emitField("visibility", JsonUtil.string(visibility(sym)), out, next) - emitField("flags", stringArray(Flags.flagStrings(sym.flags)), out, next) - emitField("annotations", annotationsJson(sym), out, next) - emitField("doc", docs(sym).map(JsonUtil.string).getOrElse("null"), out, next, comma = views.nonEmpty || members.nonEmpty) - - if views.nonEmpty then - emitField("views", stringArray(views), out, next, comma = members.nonEmpty) + val entries = mutable.ArrayBuffer.empty[(String, String)] + if fields.contains("name") then entries += "name" -> JsonUtil.string(sym.fullName) + if fields.contains("kind") then entries += "kind" -> JsonUtil.string(kind) + if fields.contains("signature") then entries += "signature" -> JsonUtil.string(signature) + if fields.contains("source") then entries += "source" -> sourceJson(source) + if fields.contains("visibility") then entries += "visibility" -> JsonUtil.string(visibility(sym)) + if fields.contains("flags") then entries += "flags" -> stringArray(Flags.flagStrings(sym.flags)) + if fields.contains("annotations") then entries += "annotations" -> annotationsJson(sym) + if fields.contains("doc") then entries += "doc" -> docs(sym).map(JsonUtil.string).getOrElse("null") + if views.nonEmpty then entries += "views" -> stringArray(views) + + for ((name, value), index) <- entries.zipWithIndex do + emitField(name, value, out, next, comma = index < entries.size - 1 || members.nonEmpty) if members.nonEmpty then out.println(next + JsonUtil.string("members") + ": [") - emitMemberList(members, out, next + " ") + emitMemberList(members, fields, out, next + " ") out.println() out.println(next + "]") @@ -512,7 +546,7 @@ object Query: private def stringArray(values: List[String]): String = values.map(JsonUtil.string).mkString("[", ", ", "]") - private def emitField(name: String, value: String, out: PrintWriter, indent: String, comma: Boolean = true): Unit = + private def emitField(name: String, value: String, out: PrintWriter, indent: String, comma: Boolean): Unit = out.print(indent) out.print(JsonUtil.string(name)) out.print(": ") From d65b045c20a0653cf0ee8dd89ca38cf8b8768ddc Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 18:02:08 +0200 Subject: [PATCH 32/36] Update check files Signed-off-by: Fengyun Liu --- .../doc-query/companion-member.json.check | 5 -- tests/custom/doc-query/companion.jo | 7 ++ tests/custom/doc-query/describe.json.check | 5 -- tests/custom/doc-query/exact-field.json.check | 10 --- .../custom/doc-query/exact-member.json.check | 5 -- tests/custom/doc-query/fields.json.check | 6 ++ .../doc-query/interface-member.json.check | 5 -- .../doc-query/lib-exact-member.json.check | 5 -- tests/custom/doc-query/lib-query.json.check | 90 ------------------- tests/custom/doc-query/structural.json.check | 90 ------------------- tests/custom/doc-query/test.sh | 15 ++++ 11 files changed, 28 insertions(+), 215 deletions(-) create mode 100644 tests/custom/doc-query/fields.json.check diff --git a/tests/custom/doc-query/companion-member.json.check b/tests/custom/doc-query/companion-member.json.check index 336c79de..3962c3ee 100644 --- a/tests/custom/doc-query/companion-member.json.check +++ b/tests/custom/doc-query/companion-member.json.check @@ -1,12 +1,7 @@ [ { "name": "DocQueryCompanion.Value.label", - "kind": "def", "signature": "def label(): String receives none", - "source": { "file": "tests/custom/doc-query/companion.jo", "line": 6, "end": 6 }, - "visibility": "public", - "flags": ["fun", "method"], - "annotations": [], "doc": "Return the value label." } ] diff --git a/tests/custom/doc-query/companion.jo b/tests/custom/doc-query/companion.jo index 3ff32021..08cf8216 100644 --- a/tests/custom/doc-query/companion.jo +++ b/tests/custom/doc-query/companion.jo @@ -11,3 +11,10 @@ section Value //[ Create a value. //] def create: Value = new Value() end + +interface Tagged +end + +class TaggedValue + view Tagged +end diff --git a/tests/custom/doc-query/describe.json.check b/tests/custom/doc-query/describe.json.check index 569d98ab..4e73a074 100644 --- a/tests/custom/doc-query/describe.json.check +++ b/tests/custom/doc-query/describe.json.check @@ -1,12 +1,7 @@ [ { "name": "DocQueryAPI.describe", - "kind": "def", "signature": "def describe[T](value: T)(auto show: Show[T] with [[T].toString]): String receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24, "end": 25 }, - "visibility": "public", - "flags": ["fun"], - "annotations": [], "doc": "Render a value with auto formatting and context." } ] diff --git a/tests/custom/doc-query/exact-field.json.check b/tests/custom/doc-query/exact-field.json.check index 0abcd92b..8d3ccf25 100644 --- a/tests/custom/doc-query/exact-field.json.check +++ b/tests/custom/doc-query/exact-field.json.check @@ -1,22 +1,12 @@ [ { "name": "DocQueryAPI.Box", - "kind": "class", "signature": "class Box", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 42, "end": 51 }, - "visibility": "public", - "flags": ["class"], - "annotations": [], "doc": "A small documented class.", "members": [ { "name": "DocQueryAPI.Box.name", - "kind": "field", "signature": "val name: String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 44, "end": 44 }, - "visibility": "public", - "flags": ["field"], - "annotations": [], "doc": "Stored display name." } ] diff --git a/tests/custom/doc-query/exact-member.json.check b/tests/custom/doc-query/exact-member.json.check index 733cf5e3..e44c8f43 100644 --- a/tests/custom/doc-query/exact-member.json.check +++ b/tests/custom/doc-query/exact-member.json.check @@ -1,12 +1,7 @@ [ { "name": "DocQueryAPI.Box.label", - "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 51, "end": 51 }, - "visibility": "public", - "flags": ["fun", "method"], - "annotations": [], "doc": "Render a label." } ] diff --git a/tests/custom/doc-query/fields.json.check b/tests/custom/doc-query/fields.json.check new file mode 100644 index 00000000..9b3f62bd --- /dev/null +++ b/tests/custom/doc-query/fields.json.check @@ -0,0 +1,6 @@ +[ + { + "name": "DocQueryAPI.describe", + "source": { "file": "tests/custom/doc-query/api.jo", "line": 24, "end": 25 } + } +] diff --git a/tests/custom/doc-query/interface-member.json.check b/tests/custom/doc-query/interface-member.json.check index affdb7c8..781b2683 100644 --- a/tests/custom/doc-query/interface-member.json.check +++ b/tests/custom/doc-query/interface-member.json.check @@ -1,12 +1,7 @@ [ { "name": "DocQueryAPI.FileLike.readText", - "kind": "def", "signature": "def readText(path: String, limit: Int = 10): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 38, "end": 38 }, - "visibility": "public", - "flags": ["defer", "fun", "method"], - "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], "doc": "Read a text file with an optional limit." } ] diff --git a/tests/custom/doc-query/lib-exact-member.json.check b/tests/custom/doc-query/lib-exact-member.json.check index cc888c3a..e44c8f43 100644 --- a/tests/custom/doc-query/lib-exact-member.json.check +++ b/tests/custom/doc-query/lib-exact-member.json.check @@ -1,12 +1,7 @@ [ { "name": "DocQueryAPI.Box.label", - "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 51, "end": 51 }, - "visibility": "public", - "flags": ["loaded", "fun", "method"], - "annotations": [], "doc": "Render a label." } ] diff --git a/tests/custom/doc-query/lib-query.json.check b/tests/custom/doc-query/lib-query.json.check index 572b91e7..79762498 100644 --- a/tests/custom/doc-query/lib-query.json.check +++ b/tests/custom/doc-query/lib-query.json.check @@ -1,190 +1,100 @@ [ { "name": "DocQueryAPI", - "kind": "namespace", "signature": "namespace DocQueryAPI", - "source": null, - "visibility": "public", - "flags": ["namespace"], - "annotations": [], "doc": null, "members": [ { "name": "DocQueryAPI.defaultLimit", - "kind": "param", "signature": "param defaultLimit: Int", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 4, "end": 4 }, - "visibility": "public", - "flags": ["context"], - "annotations": [], "doc": "Default limit used by context-aware helpers." }, { "name": "DocQueryAPI.Path", - "kind": "type", "signature": "type Path = String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 7, "end": 7 }, - "visibility": "public", - "flags": [], - "annotations": [], "doc": "File path alias used in host calls." }, { "name": "DocQueryAPI.Positive", - "kind": "pattern", "signature": "pattern Positive(value: Int): Partial[Int] receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 10, "end": 11 }, - "visibility": "public", - "flags": ["fun"], - "annotations": [], "doc": "Match positive integers." }, { "name": "DocQueryAPI.HostOps", - "kind": "section", "signature": "section HostOps", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 14, "end": 20 }, - "visibility": "public", - "flags": ["section"], - "annotations": [], "doc": "Host-backed operations grouped in a section.", "members": [ { "name": "DocQueryAPI.HostOps.remoteSize", - "kind": "def", "signature": "def remoteSize(path: Path): Int receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 16, "end": 16 }, - "visibility": "public", - "flags": ["defer", "loaded", "fun"], - "annotations": [], "doc": "Return a remote file size." }, { "name": "DocQueryAPI.HostOps.normalizeLimit", - "kind": "def", "signature": "def normalizeLimit(value: Int): Int receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 19, "end": 20 }, - "visibility": "public", - "flags": ["loaded", "fun"], - "annotations": [], "doc": "Clamp a caller supplied limit." } ] }, { "name": "DocQueryAPI.describe", - "kind": "def", "signature": "def describe[T](value: T)(auto show: Show[T] with [[T].toString]): String receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24, "end": 25 }, - "visibility": "public", - "flags": ["loaded", "fun"], - "annotations": [], "doc": "Render a value with auto formatting and context." }, { "name": "DocQueryAPI.Singleton", - "kind": "class", "signature": "object Singleton", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 28, "end": 30 }, - "visibility": "public", - "flags": ["object", "class"], - "annotations": [], "doc": "Shared singleton helpers.", "members": [ { "name": "DocQueryAPI.Singleton.", - "kind": "constructor", "signature": "constructor(): Singleton receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 28, "end": 30 }, - "visibility": "public", - "flags": ["loaded", "fun", "method", "constructor"], - "annotations": [], "doc": null }, { "name": "DocQueryAPI.Singleton.label", - "kind": "def", "signature": "def label(): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 30, "end": 30 }, - "visibility": "public", - "flags": ["loaded", "fun", "method"], - "annotations": [], "doc": "Return the singleton label." } ] }, { "name": "DocQueryAPI.FileLike", - "kind": "class", "signature": "interface FileLike", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 35, "end": 38 }, - "visibility": "public", - "flags": ["interface"], - "annotations": [{ "name": "jo.py.interop", "args": [] }], "doc": "File operations exposed to a Python-backed host.", "members": [ { "name": "DocQueryAPI.FileLike.readText", - "kind": "def", "signature": "def readText(path: String, limit: Int = 10): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 38, "end": 38 }, - "visibility": "public", - "flags": ["defer", "loaded", "fun", "method"], - "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], "doc": "Read a text file with an optional limit." } ] }, { "name": "DocQueryAPI.Box", - "kind": "class", "signature": "class Box", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 42, "end": 51 }, - "visibility": "public", - "flags": ["class"], - "annotations": [], "doc": "A small documented class.", "members": [ { "name": "DocQueryAPI.Box.name", - "kind": "field", "signature": "val name: String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 44, "end": 44 }, - "visibility": "public", - "flags": ["field"], - "annotations": [], "doc": "Stored display name." }, { "name": "DocQueryAPI.Box.", - "kind": "constructor", "signature": "constructor(name: String): Box receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 47, "end": 48 }, - "visibility": "public", - "flags": ["loaded", "fun", "method", "constructor"], - "annotations": [], "doc": "Create a box with a name." }, { "name": "DocQueryAPI.Box.label", - "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 51, "end": 51 }, - "visibility": "public", - "flags": ["loaded", "fun", "method"], - "annotations": [], "doc": "Render a label." } ] }, { "name": "DocQueryAPI.helper", - "kind": "def", "signature": "def helper(value: Int): Int receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 55, "end": 55 }, - "visibility": "public", - "flags": ["loaded", "fun"], - "annotations": [], "doc": "Public helper." } ] diff --git a/tests/custom/doc-query/structural.json.check b/tests/custom/doc-query/structural.json.check index d5979843..79762498 100644 --- a/tests/custom/doc-query/structural.json.check +++ b/tests/custom/doc-query/structural.json.check @@ -1,190 +1,100 @@ [ { "name": "DocQueryAPI", - "kind": "namespace", "signature": "namespace DocQueryAPI", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 1, "end": 1 }, - "visibility": "public", - "flags": ["namespace"], - "annotations": [], "doc": null, "members": [ { "name": "DocQueryAPI.defaultLimit", - "kind": "param", "signature": "param defaultLimit: Int", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 4, "end": 4 }, - "visibility": "public", - "flags": ["context"], - "annotations": [], "doc": "Default limit used by context-aware helpers." }, { "name": "DocQueryAPI.Path", - "kind": "type", "signature": "type Path = String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 7, "end": 7 }, - "visibility": "public", - "flags": [], - "annotations": [], "doc": "File path alias used in host calls." }, { "name": "DocQueryAPI.Positive", - "kind": "pattern", "signature": "pattern Positive(value: Int): Partial[Int] receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 10, "end": 11 }, - "visibility": "public", - "flags": ["fun"], - "annotations": [], "doc": "Match positive integers." }, { "name": "DocQueryAPI.HostOps", - "kind": "section", "signature": "section HostOps", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 14, "end": 20 }, - "visibility": "public", - "flags": ["section"], - "annotations": [], "doc": "Host-backed operations grouped in a section.", "members": [ { "name": "DocQueryAPI.HostOps.remoteSize", - "kind": "def", "signature": "def remoteSize(path: Path): Int receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 16, "end": 16 }, - "visibility": "public", - "flags": ["defer", "fun"], - "annotations": [], "doc": "Return a remote file size." }, { "name": "DocQueryAPI.HostOps.normalizeLimit", - "kind": "def", "signature": "def normalizeLimit(value: Int): Int receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 19, "end": 20 }, - "visibility": "public", - "flags": ["fun"], - "annotations": [], "doc": "Clamp a caller supplied limit." } ] }, { "name": "DocQueryAPI.describe", - "kind": "def", "signature": "def describe[T](value: T)(auto show: Show[T] with [[T].toString]): String receives defaultLimit", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24, "end": 25 }, - "visibility": "public", - "flags": ["fun"], - "annotations": [], "doc": "Render a value with auto formatting and context." }, { "name": "DocQueryAPI.Singleton", - "kind": "class", "signature": "object Singleton", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 28, "end": 30 }, - "visibility": "public", - "flags": ["object", "class"], - "annotations": [], "doc": "Shared singleton helpers.", "members": [ { "name": "DocQueryAPI.Singleton.", - "kind": "constructor", "signature": "constructor(): Singleton receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 28, "end": 30 }, - "visibility": "public", - "flags": ["fun", "method", "constructor"], - "annotations": [], "doc": null }, { "name": "DocQueryAPI.Singleton.label", - "kind": "def", "signature": "def label(): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 30, "end": 30 }, - "visibility": "public", - "flags": ["fun", "method"], - "annotations": [], "doc": "Return the singleton label." } ] }, { "name": "DocQueryAPI.FileLike", - "kind": "class", "signature": "interface FileLike", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 35, "end": 38 }, - "visibility": "public", - "flags": ["interface"], - "annotations": [{ "name": "jo.py.interop", "args": [] }], "doc": "File operations exposed to a Python-backed host.", "members": [ { "name": "DocQueryAPI.FileLike.readText", - "kind": "def", "signature": "def readText(path: String, limit: Int = 10): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 38, "end": 38 }, - "visibility": "public", - "flags": ["defer", "fun", "method"], - "annotations": [{ "name": "jo.py.targetName", "args": ["read_text"] }], "doc": "Read a text file with an optional limit." } ] }, { "name": "DocQueryAPI.Box", - "kind": "class", "signature": "class Box", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 42, "end": 51 }, - "visibility": "public", - "flags": ["class"], - "annotations": [], "doc": "A small documented class.", "members": [ { "name": "DocQueryAPI.Box.name", - "kind": "field", "signature": "val name: String", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 44, "end": 44 }, - "visibility": "public", - "flags": ["field"], - "annotations": [], "doc": "Stored display name." }, { "name": "DocQueryAPI.Box.", - "kind": "constructor", "signature": "constructor(name: String): Box receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 47, "end": 48 }, - "visibility": "public", - "flags": ["fun", "method", "constructor"], - "annotations": [], "doc": "Create a box with a name." }, { "name": "DocQueryAPI.Box.label", - "kind": "def", "signature": "def label(prefix: String = \"box\"): String receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 51, "end": 51 }, - "visibility": "public", - "flags": ["fun", "method"], - "annotations": [], "doc": "Render a label." } ] }, { "name": "DocQueryAPI.helper", - "kind": "def", "signature": "def helper(value: Int): Int receives none", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 55, "end": 55 }, - "visibility": "public", - "flags": ["fun"], - "annotations": [], "doc": "Public helper." } ] diff --git a/tests/custom/doc-query/test.sh b/tests/custom/doc-query/test.sh index cea64489..3cf9334d 100755 --- a/tests/custom/doc-query/test.sh +++ b/tests/custom/doc-query/test.sh @@ -58,10 +58,12 @@ run_query --query "DocQueryAPI.*,DocQueryAPI.FileLike.readText" "$API_FILE" > "$ run_query --query "file:$API_FILE" "$API_FILE" "$EXTRA_FILE" > "$DIR/file.json" run_query --query "file:$PROJECT_ROOT/$API_FILE" "$API_FILE" "$EXTRA_FILE" > "$DIR/file-absolute.json" run_query --query "DocQueryAPI.describe" "$API_FILE" > "$DIR/describe.json" +run_query --query "DocQueryAPI.describe" --fields source,name "$API_FILE" > "$DIR/fields.json" run_query --query "DocQueryAPI.Box.label" "$API_FILE" > "$DIR/exact-member.json" run_query --query "DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/interface-member.json" run_query --query "DocQueryAPI.Box.name" "$API_FILE" > "$DIR/exact-field.json" run_query --query "DocQueryCompanion.Value.label" "$COMPANION_FILE" > "$DIR/companion-member.json" +run_query --query "DocQueryCompanion.TaggedValue" "$COMPANION_FILE" > "$DIR/class-views.json" run_query --query "DocQueryAPI,DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/query-implies-json.json" run_query --lib "$SAST_DIR" --query "DocQueryAPI" > "$DIR/lib-query.json" run_query --lib "$SAST_DIR" --query "DocQueryAPI.Box.label" > "$DIR/lib-exact-member.json" @@ -75,10 +77,12 @@ diff -u "$DIR/lib-query.json.check" "$DIR/lib-query.json" diff -u "$DIR/lib-query.json.check" "$DIR/lib-file-query.json" diff -u "$DIR/lib-exact-member.json.check" "$DIR/lib-exact-member.json" diff -u "$DIR/describe.json.check" "$DIR/describe.json" +diff -u "$DIR/fields.json.check" "$DIR/fields.json" diff -u "$DIR/exact-member.json.check" "$DIR/exact-member.json" diff -u "$DIR/interface-member.json.check" "$DIR/interface-member.json" diff -u "$DIR/exact-field.json.check" "$DIR/exact-field.json" diff -u "$DIR/companion-member.json.check" "$DIR/companion-member.json" +diff -u "$DIR/class-views.json.check" "$DIR/class-views.json" expect_fail "$FAIL_LOG" run_plain_query --query "NoSuchSymbol" grep -q -- "No documentation entries match symbol selector" "$FAIL_LOG" @@ -86,4 +90,15 @@ grep -q -- "No documentation entries match symbol selector" "$FAIL_LOG" expect_fail "$FAIL_LOG" run_plain_query --no-stdlib --query "NoSuchSymbol" grep -q -- "No documentation entries match symbol selector" "$FAIL_LOG" +expect_fail "$FAIL_LOG" run_query --query "DocQueryAPI.describe" --fields unknown "$API_FILE" +grep -q -- "Unknown query field: unknown" "$FAIL_LOG" +grep -q -- "Available fields: name,kind,signature,source,visibility,flags,annotations,doc" "$FAIL_LOG" + +expect_fail "$FAIL_LOG" run_query --query "DocQueryAPI.describe" --fields views "$API_FILE" +grep -q -- "Unknown query field: views" "$FAIL_LOG" + +expect_fail "$FAIL_LOG" run_query --query "DocQueryAPI.describe" --fields , "$API_FILE" +grep -q -- "Option --fields requires at least one field" "$FAIL_LOG" +grep -q -- "Available fields: name,kind,signature,source,visibility,flags,annotations,doc" "$FAIL_LOG" + echo " ✓ All tests passed for $TEST_NAME" From ef526719875a9843b2bdde5e1e62eecbdd3a4b74 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 18:08:08 +0200 Subject: [PATCH 33/36] Refactor code Signed-off-by: Fengyun Liu --- stack-lang/query/Query.scala | 99 ++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 54 deletions(-) diff --git a/stack-lang/query/Query.scala b/stack-lang/query/Query.scala index 716f50c8..a4254865 100644 --- a/stack-lang/query/Query.scala +++ b/stack-lang/query/Query.scala @@ -19,6 +19,9 @@ object Query: private val availableFields = outputFields.mkString(",") + private case class EmitContext(fields: Set[String], out: PrintWriter, indent: String): + def indented: EmitContext = copy(indent = indent + " ") + def parseFields(rawFields: String)(using Reporter): Set[String] = val fields = rawFields.split(",").map(_.trim).filter(_.nonEmpty).toSet if fields.isEmpty then @@ -135,8 +138,9 @@ object Query: )(using Reporter, Definitions): Unit = val trimmedUnits = trimUnits(units, filter, includePrivate) val sortedTargets = sortRoots(jsonRoots(trimmedUnits, filter)) + given EmitContext = EmitContext(fields, out, " ") out.println("[") - emitRootList(sortedTargets, fields, out, " ") + emitRootList(sortedTargets) if sortedTargets.nonEmpty then out.println() out.println("]") @@ -363,47 +367,37 @@ object Query: val line = source.map(_.line).getOrElse(0) (file, line, kind, sym.fullName) - private def emitRootList( - roots: List[FileUnit | Def], - fields: Set[String], - out: PrintWriter, - indent: String, - )(using Definitions): Unit = + private def emitRootList(roots: List[FileUnit | Def])(using ctx: EmitContext, defn: Definitions): Unit = var first = true for root <- roots do - if !first then out.println(",") + if !first then ctx.out.println(",") first = false root match - case unit: FileUnit => emitNamespace(unit, fields, out, indent) - case defn: Def => emitDef(defn, fields, out, indent) + case unit: FileUnit => emitNamespace(unit) + case tree: Def => emitDef(tree) - private def emitMemberList( - members: List[Def | FieldDecl], - fields: Set[String], - out: PrintWriter, - indent: String, - )(using Definitions): Unit = + private def emitMemberList(members: List[Def | FieldDecl])(using ctx: EmitContext, defn: Definitions): Unit = var first = true for member <- members do - if !first then out.println(",") + if !first then ctx.out.println(",") first = false member match - case defn: Def => emitDef(defn, fields, out, indent) - case field: FieldDecl => emitFieldDecl(field, fields, out, indent) + case tree: Def => emitDef(tree) + case field: FieldDecl => emitFieldDecl(field) - private def emitNamespace(unit: FileUnit, fields: Set[String], out: PrintWriter, indent: String)(using Definitions): Unit = + private def emitNamespace(unit: FileUnit)(using EmitContext, Definitions): Unit = val members = sortMembers(unit.defs.map(defn => defn: Def | FieldDecl)) - emitSymbol(unit.owner, "namespace", "namespace " + unit.owner.fullName, Nil, members, sourceLoc(unit.owner), fields, out, indent) + emitSymbol(unit.owner, "namespace", "namespace " + unit.owner.fullName, Nil, members, sourceLoc(unit.owner)) - private def emitDef(defn: Def, fields: Set[String], out: PrintWriter, indent: String)(using Definitions): Unit = - val members = sortMembers(memberNodes(defn)) - val views = defn match + private def emitDef(tree: Def)(using EmitContext, Definitions): Unit = + val members = sortMembers(memberNodes(tree)) + val views = tree match case cd: ClassDef => cd.views.map(_.tpe.show) case _ => Nil - emitSymbol(defn.symbol, kind(defn), signature(defn), views, members, sourceLoc(defn.symbol, defn.span), fields, out, indent) + emitSymbol(tree.symbol, kind(tree), signature(tree), views, members, sourceLoc(tree.symbol, tree.span)) - private def emitFieldDecl(field: FieldDecl, fields: Set[String], out: PrintWriter, indent: String)(using Definitions): Unit = - emitSymbol(field.symbol, "field", fieldSignature(field), Nil, Nil, sourceLoc(field.symbol, field.span), fields, out, indent) + private def emitFieldDecl(field: FieldDecl)(using EmitContext, Definitions): Unit = + emitSymbol(field.symbol, "field", fieldSignature(field), Nil, Nil, sourceLoc(field.symbol, field.span)) private def emitSymbol( sym: Symbol, @@ -412,33 +406,30 @@ object Query: views: List[String], members: List[Def | FieldDecl], source: Option[SourceLoc], - fields: Set[String], - out: PrintWriter, - indent: String, - )(using Definitions): Unit = - val next = indent + " " - out.println(indent + "{") + )(using ctx: EmitContext, defn: Definitions): Unit = + val nested = ctx.indented + ctx.out.println(ctx.indent + "{") val entries = mutable.ArrayBuffer.empty[(String, String)] - if fields.contains("name") then entries += "name" -> JsonUtil.string(sym.fullName) - if fields.contains("kind") then entries += "kind" -> JsonUtil.string(kind) - if fields.contains("signature") then entries += "signature" -> JsonUtil.string(signature) - if fields.contains("source") then entries += "source" -> sourceJson(source) - if fields.contains("visibility") then entries += "visibility" -> JsonUtil.string(visibility(sym)) - if fields.contains("flags") then entries += "flags" -> stringArray(Flags.flagStrings(sym.flags)) - if fields.contains("annotations") then entries += "annotations" -> annotationsJson(sym) - if fields.contains("doc") then entries += "doc" -> docs(sym).map(JsonUtil.string).getOrElse("null") + if ctx.fields.contains("name") then entries += "name" -> JsonUtil.string(sym.fullName) + if ctx.fields.contains("kind") then entries += "kind" -> JsonUtil.string(kind) + if ctx.fields.contains("signature") then entries += "signature" -> JsonUtil.string(signature) + if ctx.fields.contains("source") then entries += "source" -> sourceJson(source) + if ctx.fields.contains("visibility") then entries += "visibility" -> JsonUtil.string(visibility(sym)) + if ctx.fields.contains("flags") then entries += "flags" -> stringArray(Flags.flagStrings(sym.flags)) + if ctx.fields.contains("annotations") then entries += "annotations" -> annotationsJson(sym) + if ctx.fields.contains("doc") then entries += "doc" -> docs(sym).map(JsonUtil.string).getOrElse("null") if views.nonEmpty then entries += "views" -> stringArray(views) for ((name, value), index) <- entries.zipWithIndex do - emitField(name, value, out, next, comma = index < entries.size - 1 || members.nonEmpty) + emitField(name, value, comma = index < entries.size - 1 || members.nonEmpty)(using nested) if members.nonEmpty then - out.println(next + JsonUtil.string("members") + ": [") - emitMemberList(members, fields, out, next + " ") - out.println() - out.println(next + "]") + ctx.out.println(nested.indent + JsonUtil.string("members") + ": [") + emitMemberList(members)(using nested.indented, defn) + ctx.out.println() + ctx.out.println(nested.indent + "]") - out.print(indent + "}") + ctx.out.print(ctx.indent + "}") private def memberNodes(defn: Def): List[Def | FieldDecl] = defn match @@ -546,13 +537,13 @@ object Query: private def stringArray(values: List[String]): String = values.map(JsonUtil.string).mkString("[", ", ", "]") - private def emitField(name: String, value: String, out: PrintWriter, indent: String, comma: Boolean): Unit = - out.print(indent) - out.print(JsonUtil.string(name)) - out.print(": ") - out.print(value) - if comma then out.print(",") - out.println() + private def emitField(name: String, value: String, comma: Boolean)(using ctx: EmitContext): Unit = + ctx.out.print(ctx.indent) + ctx.out.print(JsonUtil.string(name)) + ctx.out.print(": ") + ctx.out.print(value) + if comma then ctx.out.print(",") + ctx.out.println() private def typeParams(params: List[Symbol]): String = if params.isEmpty then "" else params.map(_.name).mkString("[", ", ", "]") From 86fd5577a5653a4816f36e88327fa4a42304871e Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 18:08:57 +0200 Subject: [PATCH 34/36] Add missing check file Signed-off-by: Fengyun Liu --- tests/custom/doc-query/class-views.json.check | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/custom/doc-query/class-views.json.check diff --git a/tests/custom/doc-query/class-views.json.check b/tests/custom/doc-query/class-views.json.check new file mode 100644 index 00000000..e192dc00 --- /dev/null +++ b/tests/custom/doc-query/class-views.json.check @@ -0,0 +1,15 @@ +[ + { + "name": "DocQueryCompanion.TaggedValue", + "signature": "class TaggedValue", + "doc": null, + "views": ["Tagged"], + "members": [ + { + "name": "DocQueryCompanion.TaggedValue.", + "signature": "constructor(): TaggedValue receives none", + "doc": null + } + ] + } +] From cc785fdc06641006fa42efba4735b8d874d484b2 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 18:10:47 +0200 Subject: [PATCH 35/36] Fix signature for constructor Signed-off-by: Fengyun Liu --- stack-lang/query/Query.scala | 2 +- tests/custom/doc-query/class-views.json.check | 2 +- tests/custom/doc-query/lib-query.json.check | 4 ++-- tests/custom/doc-query/structural.json.check | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/stack-lang/query/Query.scala b/stack-lang/query/Query.scala index a4254865..3fc9a7e9 100644 --- a/stack-lang/query/Query.scala +++ b/stack-lang/query/Query.scala @@ -615,7 +615,7 @@ object Query: if sym.is(Flags.Annotation) then "annotation " + sym.name + procSignature(fd).stripSuffix(": void receives none").stripSuffix(": void") else if sym.is(Flags.Constructor) then - "constructor" + procSignature(fd) + "def " + sym.owner.name + procSignature(fd) else "def " + sym.name + procSignature(fd) diff --git a/tests/custom/doc-query/class-views.json.check b/tests/custom/doc-query/class-views.json.check index e192dc00..dece76ad 100644 --- a/tests/custom/doc-query/class-views.json.check +++ b/tests/custom/doc-query/class-views.json.check @@ -7,7 +7,7 @@ "members": [ { "name": "DocQueryCompanion.TaggedValue.", - "signature": "constructor(): TaggedValue receives none", + "signature": "def TaggedValue(): TaggedValue receives none", "doc": null } ] diff --git a/tests/custom/doc-query/lib-query.json.check b/tests/custom/doc-query/lib-query.json.check index 79762498..28cc3c03 100644 --- a/tests/custom/doc-query/lib-query.json.check +++ b/tests/custom/doc-query/lib-query.json.check @@ -48,7 +48,7 @@ "members": [ { "name": "DocQueryAPI.Singleton.", - "signature": "constructor(): Singleton receives none", + "signature": "def Singleton(): Singleton receives none", "doc": null }, { @@ -82,7 +82,7 @@ }, { "name": "DocQueryAPI.Box.", - "signature": "constructor(name: String): Box receives none", + "signature": "def Box(name: String): Box receives none", "doc": "Create a box with a name." }, { diff --git a/tests/custom/doc-query/structural.json.check b/tests/custom/doc-query/structural.json.check index 79762498..28cc3c03 100644 --- a/tests/custom/doc-query/structural.json.check +++ b/tests/custom/doc-query/structural.json.check @@ -48,7 +48,7 @@ "members": [ { "name": "DocQueryAPI.Singleton.", - "signature": "constructor(): Singleton receives none", + "signature": "def Singleton(): Singleton receives none", "doc": null }, { @@ -82,7 +82,7 @@ }, { "name": "DocQueryAPI.Box.", - "signature": "constructor(name: String): Box receives none", + "signature": "def Box(name: String): Box receives none", "doc": "Create a box with a name." }, { From c57a8bb77f9564bd3e9b1dbe2a40f005bfd3c914 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Wed, 29 Jul 2026 22:18:12 +0200 Subject: [PATCH 36/36] Rename field source to loc Signed-off-by: Fengyun Liu --- docs/usage/commands/compile.md | 2 +- docs/usage/reference/compiler-options.md | 2 +- stack-lang/query/Query.scala | 4 ++-- tests/custom/doc-query/fields.json.check | 2 +- tests/custom/doc-query/test.sh | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/usage/commands/compile.md b/docs/usage/commands/compile.md index f2cc55e8..e0b3400e 100644 --- a/docs/usage/commands/compile.md +++ b/docs/usage/commands/compile.md @@ -145,7 +145,7 @@ jo compile --query jo.List.map Select output fields: ```sh -jo compile --query jo.List.map --fields name,signature,source,doc +jo compile --query jo.List.map --fields name,signature,loc,doc ``` Query a standard-library source file by basename: diff --git a/docs/usage/reference/compiler-options.md b/docs/usage/reference/compiler-options.md index 999c7d2d..e12f61f6 100644 --- a/docs/usage/reference/compiler-options.md +++ b/docs/usage/reference/compiler-options.md @@ -121,7 +121,7 @@ API libraries selected by `--use-runtime-api`, and any `--lib` paths. Query output includes public symbols only. Each symbol includes `name`, `signature`, and `doc` by default. Use -`--fields` to select any combination of `name`, `kind`, `signature`, `source`, +`--fields` to select any combination of `name`, `kind`, `signature`, `loc`, `visibility`, `flags`, `annotations`, and `doc`. Field order in the option does not affect JSON output order. Classes with views include `views` automatically. diff --git a/stack-lang/query/Query.scala b/stack-lang/query/Query.scala index 3fc9a7e9..66227d2e 100644 --- a/stack-lang/query/Query.scala +++ b/stack-lang/query/Query.scala @@ -15,7 +15,7 @@ import scala.collection.mutable object Query: private val outputFields = - List("name", "kind", "signature", "source", "visibility", "flags", "annotations", "doc") + List("name", "kind", "signature", "loc", "visibility", "flags", "annotations", "doc") private val availableFields = outputFields.mkString(",") @@ -413,7 +413,7 @@ object Query: if ctx.fields.contains("name") then entries += "name" -> JsonUtil.string(sym.fullName) if ctx.fields.contains("kind") then entries += "kind" -> JsonUtil.string(kind) if ctx.fields.contains("signature") then entries += "signature" -> JsonUtil.string(signature) - if ctx.fields.contains("source") then entries += "source" -> sourceJson(source) + if ctx.fields.contains("loc") then entries += "loc" -> sourceJson(source) if ctx.fields.contains("visibility") then entries += "visibility" -> JsonUtil.string(visibility(sym)) if ctx.fields.contains("flags") then entries += "flags" -> stringArray(Flags.flagStrings(sym.flags)) if ctx.fields.contains("annotations") then entries += "annotations" -> annotationsJson(sym) diff --git a/tests/custom/doc-query/fields.json.check b/tests/custom/doc-query/fields.json.check index 9b3f62bd..85330bf4 100644 --- a/tests/custom/doc-query/fields.json.check +++ b/tests/custom/doc-query/fields.json.check @@ -1,6 +1,6 @@ [ { "name": "DocQueryAPI.describe", - "source": { "file": "tests/custom/doc-query/api.jo", "line": 24, "end": 25 } + "loc": { "file": "tests/custom/doc-query/api.jo", "line": 24, "end": 25 } } ] diff --git a/tests/custom/doc-query/test.sh b/tests/custom/doc-query/test.sh index 3cf9334d..8dd8110e 100755 --- a/tests/custom/doc-query/test.sh +++ b/tests/custom/doc-query/test.sh @@ -58,7 +58,7 @@ run_query --query "DocQueryAPI.*,DocQueryAPI.FileLike.readText" "$API_FILE" > "$ run_query --query "file:$API_FILE" "$API_FILE" "$EXTRA_FILE" > "$DIR/file.json" run_query --query "file:$PROJECT_ROOT/$API_FILE" "$API_FILE" "$EXTRA_FILE" > "$DIR/file-absolute.json" run_query --query "DocQueryAPI.describe" "$API_FILE" > "$DIR/describe.json" -run_query --query "DocQueryAPI.describe" --fields source,name "$API_FILE" > "$DIR/fields.json" +run_query --query "DocQueryAPI.describe" --fields loc,name "$API_FILE" > "$DIR/fields.json" run_query --query "DocQueryAPI.Box.label" "$API_FILE" > "$DIR/exact-member.json" run_query --query "DocQueryAPI.FileLike.readText" "$API_FILE" > "$DIR/interface-member.json" run_query --query "DocQueryAPI.Box.name" "$API_FILE" > "$DIR/exact-field.json" @@ -92,13 +92,13 @@ grep -q -- "No documentation entries match symbol selector" "$FAIL_LOG" expect_fail "$FAIL_LOG" run_query --query "DocQueryAPI.describe" --fields unknown "$API_FILE" grep -q -- "Unknown query field: unknown" "$FAIL_LOG" -grep -q -- "Available fields: name,kind,signature,source,visibility,flags,annotations,doc" "$FAIL_LOG" +grep -q -- "Available fields: name,kind,signature,loc,visibility,flags,annotations,doc" "$FAIL_LOG" expect_fail "$FAIL_LOG" run_query --query "DocQueryAPI.describe" --fields views "$API_FILE" grep -q -- "Unknown query field: views" "$FAIL_LOG" expect_fail "$FAIL_LOG" run_query --query "DocQueryAPI.describe" --fields , "$API_FILE" grep -q -- "Option --fields requires at least one field" "$FAIL_LOG" -grep -q -- "Available fields: name,kind,signature,source,visibility,flags,annotations,doc" "$FAIL_LOG" +grep -q -- "Available fields: name,kind,signature,loc,visibility,flags,annotations,doc" "$FAIL_LOG" echo " ✓ All tests passed for $TEST_NAME"