From 63615bbbcec40e17360130f1469ec9ce012f90fb Mon Sep 17 00:00:00 2001 From: David Tooke Date: Fri, 9 Oct 2026 17:56:37 -0400 Subject: [PATCH 1/4] Publish the command-line guide: a page per command, every example run docs/guide/command-line is the command-line section of the Relix programming guide, laid out as the engine's manuals are: a README.md index of chapters, and index.json as the machine-readable index. Hand-written pages cover the command and where scripts come from, input binding (-i, formats, standard input, headings, unbounded inputs, -a), output formats, the catalog (discovery, the overlay, trust, relixrc), secrets and profiles, and pipelines with jq, awk, find and xargs. The command reference, commands/*.md, is generated by CommandPages from the picocli model --help and the man pages come from, followed by each command's hand-written examples in docs/command-examples. ./gradlew commandPages rewrites them; GuideStructureTest fails while they are stale, while the index misses a page, or while a link to a page or a heading, here or in the engine's manuals, is broken. GuideExamplesTest runs every shell block over a copy of the fixture under src/test/resources/docs-fixture, the ~/shop the pages describe, and compares standard output and the exit status. A plain relix command runs in the test's JVM; anything else runs through bash, with a relix shim that treats a stage's streams as a terminal when they are the example's own, as a reader's would be. The guard was probed by mutation: a changed row, a changed or unshown exit status, a dropped output block, a broken link or heading, an unlisted page, a stale command page, an unrun bash block, a change to the table renderer and a change to the harness itself each fail it. ./gradlew docsArchive builds relix--docs.zip, which the release workflow attaches to each GitHub Release for the site to render. Fixes #16 --- .github/workflows/release.yml | 9 +- CONTRIBUTING.md | 21 + README.md | 4 + build.gradle | 37 ++ docs/command-examples/bundle.md | 10 + docs/command-examples/catalog.md | 44 ++ docs/command-examples/check.md | 30 ++ docs/command-examples/completion.md | 14 + docs/command-examples/connectors.md | 22 + docs/command-examples/doc.md | 38 ++ docs/command-examples/drivers.md | 20 + docs/command-examples/explain.md | 20 + docs/command-examples/fmt.md | 61 +++ docs/command-examples/help.md | 12 + docs/command-examples/ir.md | 67 +++ docs/command-examples/optimize.md | 39 ++ docs/command-examples/provenance.md | 54 +++ docs/command-examples/relix.md | 57 +++ docs/command-examples/run.md | 40 ++ docs/command-examples/trace.md | 24 + docs/command-examples/version.md | 11 + docs/design.md | 7 + docs/guide/command-line/README.md | 65 +++ docs/guide/command-line/catalog.md | 328 ++++++++++++++ docs/guide/command-line/commands/bundle.md | 38 ++ docs/guide/command-line/commands/catalog.md | 195 ++++++++ docs/guide/command-line/commands/check.md | 70 +++ .../guide/command-line/commands/completion.md | 39 ++ .../guide/command-line/commands/connectors.md | 82 ++++ docs/guide/command-line/commands/doc.md | 75 ++++ docs/guide/command-line/commands/drivers.md | 79 ++++ docs/guide/command-line/commands/explain.md | 50 +++ docs/guide/command-line/commands/fmt.md | 96 ++++ docs/guide/command-line/commands/help.md | 30 ++ docs/guide/command-line/commands/ir.md | 95 ++++ docs/guide/command-line/commands/optimize.md | 68 +++ .../guide/command-line/commands/provenance.md | 86 ++++ docs/guide/command-line/commands/relix.md | 155 +++++++ docs/guide/command-line/commands/run.md | 78 ++++ docs/guide/command-line/commands/trace.md | 53 +++ docs/guide/command-line/commands/version.md | 23 + docs/guide/command-line/getting-started.md | 218 +++++++++ docs/guide/command-line/index.json | 28 ++ docs/guide/command-line/inputs.md | 291 ++++++++++++ docs/guide/command-line/output.md | 269 +++++++++++ docs/guide/command-line/pipelines.md | 192 ++++++++ docs/guide/command-line/secrets.md | 184 ++++++++ .../relix/cli/packaging/CommandPages.java | 418 ++++++++++++++++++ .../relix/cli/docs/ExampleRunner.java | 292 ++++++++++++ .../relix/cli/docs/GuideExamplesTest.java | 134 ++++++ .../relix/cli/docs/GuidePages.java | 219 +++++++++ .../relix/cli/docs/GuideStructureTest.java | 169 +++++++ .../relix/cli/docs/ShellCommand.java | 223 ++++++++++ .../relix/cli/docs/TerminalMain.java | 54 +++ .../home/.relix/catalog/00-regions.relix | 8 + .../docs-fixture/home/.relix/profiles.json | 3 + .../home/shop/.relix/catalog/00-sources.relix | 10 + .../home/shop/.relix/catalog/10-views.relix | 5 + .../shop/.relix/catalog/20-warehouse.relix | 11 + .../home/shop/.relix/profiles.json | 6 + .../docs-fixture/home/shop/data/customers.csv | 5 + .../docs-fixture/home/shop/data/orders.csv | 5 + .../home/shop/data/returns.ndjson | 2 + .../reports/.relix/catalog/10-sample.relix | 5 + .../home/shop/reports/sample-orders.csv | 3 + .../home/shop/scripts/broken.relix | 1 + .../home/shop/scripts/by-city.relix | 2 + .../home/shop/scripts/messy.relix | 3 + .../home/shop/scripts/top-customers.relix | 2 + 69 files changed, 5107 insertions(+), 1 deletion(-) create mode 100644 docs/command-examples/bundle.md create mode 100644 docs/command-examples/catalog.md create mode 100644 docs/command-examples/check.md create mode 100644 docs/command-examples/completion.md create mode 100644 docs/command-examples/connectors.md create mode 100644 docs/command-examples/doc.md create mode 100644 docs/command-examples/drivers.md create mode 100644 docs/command-examples/explain.md create mode 100644 docs/command-examples/fmt.md create mode 100644 docs/command-examples/help.md create mode 100644 docs/command-examples/ir.md create mode 100644 docs/command-examples/optimize.md create mode 100644 docs/command-examples/provenance.md create mode 100644 docs/command-examples/relix.md create mode 100644 docs/command-examples/run.md create mode 100644 docs/command-examples/trace.md create mode 100644 docs/command-examples/version.md create mode 100644 docs/guide/command-line/README.md create mode 100644 docs/guide/command-line/catalog.md create mode 100644 docs/guide/command-line/commands/bundle.md create mode 100644 docs/guide/command-line/commands/catalog.md create mode 100644 docs/guide/command-line/commands/check.md create mode 100644 docs/guide/command-line/commands/completion.md create mode 100644 docs/guide/command-line/commands/connectors.md create mode 100644 docs/guide/command-line/commands/doc.md create mode 100644 docs/guide/command-line/commands/drivers.md create mode 100644 docs/guide/command-line/commands/explain.md create mode 100644 docs/guide/command-line/commands/fmt.md create mode 100644 docs/guide/command-line/commands/help.md create mode 100644 docs/guide/command-line/commands/ir.md create mode 100644 docs/guide/command-line/commands/optimize.md create mode 100644 docs/guide/command-line/commands/provenance.md create mode 100644 docs/guide/command-line/commands/relix.md create mode 100644 docs/guide/command-line/commands/run.md create mode 100644 docs/guide/command-line/commands/trace.md create mode 100644 docs/guide/command-line/commands/version.md create mode 100644 docs/guide/command-line/getting-started.md create mode 100644 docs/guide/command-line/index.json create mode 100644 docs/guide/command-line/inputs.md create mode 100644 docs/guide/command-line/output.md create mode 100644 docs/guide/command-line/pipelines.md create mode 100644 docs/guide/command-line/secrets.md create mode 100644 src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java create mode 100644 src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java create mode 100644 src/test/java/com/darkcollective/relix/cli/docs/GuideExamplesTest.java create mode 100644 src/test/java/com/darkcollective/relix/cli/docs/GuidePages.java create mode 100644 src/test/java/com/darkcollective/relix/cli/docs/GuideStructureTest.java create mode 100644 src/test/java/com/darkcollective/relix/cli/docs/ShellCommand.java create mode 100644 src/test/java/com/darkcollective/relix/cli/docs/TerminalMain.java create mode 100644 src/test/resources/docs-fixture/home/.relix/catalog/00-regions.relix create mode 100644 src/test/resources/docs-fixture/home/.relix/profiles.json create mode 100644 src/test/resources/docs-fixture/home/shop/.relix/catalog/00-sources.relix create mode 100644 src/test/resources/docs-fixture/home/shop/.relix/catalog/10-views.relix create mode 100644 src/test/resources/docs-fixture/home/shop/.relix/catalog/20-warehouse.relix create mode 100644 src/test/resources/docs-fixture/home/shop/.relix/profiles.json create mode 100644 src/test/resources/docs-fixture/home/shop/data/customers.csv create mode 100644 src/test/resources/docs-fixture/home/shop/data/orders.csv create mode 100644 src/test/resources/docs-fixture/home/shop/data/returns.ndjson create mode 100644 src/test/resources/docs-fixture/home/shop/reports/.relix/catalog/10-sample.relix create mode 100644 src/test/resources/docs-fixture/home/shop/reports/sample-orders.csv create mode 100644 src/test/resources/docs-fixture/home/shop/scripts/broken.relix create mode 100644 src/test/resources/docs-fixture/home/shop/scripts/by-city.relix create mode 100644 src/test/resources/docs-fixture/home/shop/scripts/messy.relix create mode 100644 src/test/resources/docs-fixture/home/shop/scripts/top-customers.relix diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fdd3747..2f2a6ac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,6 @@ # Releases the command (design §7.3): a platform archive for each of macOS, Linux and -# Windows, attached with their checksums to this repository's GitHub Release, and the +# Windows, attached with their checksums to this repository's GitHub Release with the +# command-line guide's archive (relix--docs.zip, design §7.4), and the # Homebrew formula and Scoop manifest that install them, pushed to the tap # (DarkCollective/homebrew-relix) and the bucket (DarkCollective/scoop-relix). # @@ -89,6 +90,12 @@ jobs: - name: Build, package and test the archive shell: bash run: ./gradlew clean build releaseArchive packageTest -PcliVersion="$VERSION" $SKIP_PIN + # The command-line guide, the same on every platform, so it is built once. Its + # examples have just run in the build above. + - name: Build the command-line guide's archive + if: matrix.platform == 'linux-x86_64' + shell: bash + run: ./gradlew docsArchive -PcliVersion="$VERSION" - name: Check the archive is this platform's shell: bash run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 98b2ec5..76b7bb6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -189,6 +189,27 @@ because that is the interface a script or a pipeline depends on. A change to a command, an option or an exit status updates its help text, which is also what the manual page and the site's command reference are generated from. +The command-line section of the programming guide is in +[`docs/guide/command-line`](docs/guide/command-line/README.md); the site renders it, +and each release attaches it as `relix--docs.zip`. Three things keep it true: + +- **The command pages are generated.** `docs/guide/command-line/commands/*.md` and + `index.json` come from the command's picocli model, followed by the hand-written + examples in `docs/command-examples/.md`. After changing an option or one of + those files, run `./gradlew commandPages` and commit what it writes; + `GuideStructureTest` fails while they are stale. +- **Every example runs.** A ```` ```shell ```` block is an example, and the plain block on + the line after it is what it prints on standard output; `GuideExamplesTest` runs it in + a copy of the fixture under `src/test/resources/docs-fixture` (the `~/shop` the pages + describe) and compares. An example exits 0 unless it ends `; echo "exit $?"`, which + prints its status for the page to show. Write commands a reader would type in a + ```` ```shell ```` block, never `bash` or `console`, and keep anything else — an install + command, a syntax summary — in a ```` ```text ```` block. +- **Every link resolves**, to a page or a heading, here or in the engine's manuals: + the section sits at `docs/guide/command-line` in the engine's tree, so + `../secrets.md` is the guide's page and `../../reference/README.md` the language + reference. A new page is listed in the section's `README.md`. + ## Questions Ask in [Discussions](https://github.com/DarkCollective/relix-cli/discussions). diff --git a/README.md b/README.md index 92adf9a..bb05783 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,10 @@ It is a front end to the Relix engine, [relix-core](https://github.com/DarkCollective/relix-core), and reaches it only through the engine's published API. +Its documentation is the command-line section of the Relix programming guide, in +[`docs/guide/command-line`](docs/guide/command-line/README.md), published on the +[Relix site](https://relix.darkcollective.com/guide/index.html). + ## Status Under development. There is no release yet, and the command line described above is diff --git a/build.gradle b/build.gradle index f0b356a..a3e44a3 100644 --- a/build.gradle +++ b/build.gradle @@ -57,6 +57,11 @@ sourceSets { } // The tests of the packaged command: they run its launcher, not its classes. packageTest + // CommandPagesTest regenerates the command pages to compare them with the checked-in ones. + test { + compileClasspath += sourceSets.packaging.output + runtimeClasspath += sourceSets.packaging.output + } } configurations { @@ -175,6 +180,12 @@ test { dependsOn unpackEngineDocs inputs.dir(engineDocsDir).withPropertyName('engineDocs') systemProperty 'relix.engine.docs', engineDocsDir.get().asFile.absolutePath + // The command-line guide's pages, which the docs tests run and check: without these + // inputs a docs-only change would leave the tests UP-TO-DATE, passing without running. + inputs.dir('docs/guide/command-line').withPropertyName('guide') + inputs.dir('docs/command-examples').withPropertyName('commandExamples') + systemProperty 'relix.cli.guide', file('docs/guide/command-line').absolutePath + systemProperty 'relix.cli.commandExamples', file('docs/command-examples').absolutePath finalizedBy tasks.named('jacocoTestReport') } @@ -383,6 +394,32 @@ def releaseFiles = tasks.register('releaseFiles', JavaExec) { } as CommandLineArgumentProvider) } +// ── The command-line guide (design §7.4) ────────────────────────────────────────────── +// +// ./gradlew commandPages rewrites docs/guide/command-line/commands/*.md and +// index.json from the command's picocli model +// ./gradlew docsArchive build/distributions/relix--docs.zip, the +// section the site renders in its programming guide +tasks.register('commandPages', JavaExec) { + group = 'documentation' + description = 'Rewrites the command pages of docs/guide/command-line from the command model.' + classpath = sourceSets.packaging.runtimeClasspath + mainClass = 'com.darkcollective.relix.cli.packaging.CommandPages' + args = [projectDir.absolutePath] +} + +tasks.register('docsArchive', Zip) { + group = 'documentation' + description = 'Builds the command-line guide archive the site renders from.' + from('docs/guide/command-line') { + into 'command-line' + } + archiveFileName = "relix-${version}-docs.zip" + destinationDirectory = layout.buildDirectory.dir('distributions') + preserveFileTimestamps = false + reproducibleFileOrder = true +} + def stageRelease = tasks.register('stageRelease', Sync) { description = 'Lays out the release archive\'s contents in build/release.' dependsOn tasks.named('jlink') diff --git a/docs/command-examples/bundle.md b/docs/command-examples/bundle.md new file mode 100644 index 0000000..8abaf2e --- /dev/null +++ b/docs/command-examples/bundle.md @@ -0,0 +1,10 @@ +## Examples + +A bundle is one JSON document per script, for a tool such as a query playground: + +```shell +relix bundle -e 'SELECT amount > 100 (Orders)' | jq -c 'keys' +``` +``` +["diagnostics","namespace","queries"] +``` diff --git a/docs/command-examples/catalog.md b/docs/command-examples/catalog.md new file mode 100644 index 0000000..17b0281 --- /dev/null +++ b/docs/command-examples/catalog.md @@ -0,0 +1,44 @@ +## Examples + +```shell +relix catalog ls -o tsv +``` +``` +name kind arity file +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/.relix/catalog/00-sources.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +```shell +relix catalog schema Customers -o tsv +``` +``` +column type +customer_id N +name S +city S +``` + +```shell +relix catalog where Orders -o tsv +``` +``` +file line kind state +~/shop/.relix/catalog/00-sources.relix 7 source wins +``` + +```shell +relix catalog trust --list -o tsv +``` +``` +directory state +~/shop trusted +~/shop/reports trusted +``` + +[The catalog](../catalog.md) covers discovery, the overlay and trust. diff --git a/docs/command-examples/check.md b/docs/command-examples/check.md new file mode 100644 index 0000000..a38dafa --- /dev/null +++ b/docs/command-examples/check.md @@ -0,0 +1,30 @@ +## Examples + +A script that analyses prints nothing, and exits 0: + +```shell +relix check scripts/top-customers.relix +``` + +A problem is printed on standard error, and the status is 3: + +```shell +relix check scripts/broken.relix 2>&1; echo "exit $?" +``` +``` +scripts/broken.relix:1:30: error: Undefined relation: 'Ordrs' +exit 3 +``` + +`--json` writes the diagnostics to standard output instead, as one document for a tool to +read: + +```shell +relix check --json scripts/broken.relix; echo "exit $?" +``` +``` +{"diagnostics":[{"file":"scripts/broken.relix","line":1,"column":30,"severity":"error","message":"Undefined relation: 'Ordrs'"}]} +exit 3 +``` + +[Pipelines](../pipelines.md#lint-every-script) checks every script in a tree. diff --git a/docs/command-examples/completion.md b/docs/command-examples/completion.md new file mode 100644 index 0000000..8317bd8 --- /dev/null +++ b/docs/command-examples/completion.md @@ -0,0 +1,14 @@ +## Examples + +The release archives carry each shell's script in `completions/`, and Homebrew installs +them. To load one by hand, add the line for your shell to its startup file: + +```text +bash source <(relix completion bash) +zsh source <(relix completion zsh) +fish relix completion fish > ~/.config/fish/completions/relix.fish +PowerShell relix completion powershell | Out-String | Invoke-Expression +``` + +The zsh script is also a completion function, so it can be saved as `_relix` in a +directory on `$fpath` instead. diff --git a/docs/command-examples/connectors.md b/docs/command-examples/connectors.md new file mode 100644 index 0000000..8e433cb --- /dev/null +++ b/docs/command-examples/connectors.md @@ -0,0 +1,22 @@ +## Examples + +The connectors that ship with `relix`, and any plugins installed in `~/.relix/connectors`: + +```shell +relix connectors ls +``` +``` +── connectors ────────────────────────────────────────────────────────────────── + type installed + ────── ───────── + clf true + csv true + gedcom true + log true +(4 rows) +``` + +`relix connectors ls --available` adds the connectors that can be installed, which it +fetches from the hosted catalog, and `relix connectors install TYPE` installs one. A +source whose type no installed connector reads — `mongodb`, say — fails with exit status +5, unless `--allow-download` lets the run fetch the connector. diff --git a/docs/command-examples/doc.md b/docs/command-examples/doc.md new file mode 100644 index 0000000..adc6a9b --- /dev/null +++ b/docs/command-examples/doc.md @@ -0,0 +1,38 @@ +## Examples + +A page is found by its glyph, its keyword or its name: + +```shell +relix doc --markdown σ | head -6 +``` +``` +# Name: Selection (σ / SELECT) + +# Syntax: +σ (Relation) +SELECT (Relation) +``` + +With no topic, `relix doc` lists the pages, as rows: + +```shell +relix doc -o tsv | head -4 +``` +``` +topic category title summary +getting-started guide Getting started Your first script — declare data, name a result, run it +glossary guide Glossary The vocabulary — relation, tuple, bag vs set, materialisation, pushdown +σ operator Selection Filter rows by a condition +``` + +`--function` lists, or looks up, the functions only: + +```shell +relix doc --function -o tsv | grep -i '^round' +``` +``` +Round function Round Round to nearest +``` + +On a terminal a page is shown through `$PAGER` (`less` when it is not set); `--no-pager` +writes it straight to standard output. diff --git a/docs/command-examples/drivers.md b/docs/command-examples/drivers.md new file mode 100644 index 0000000..4b202cf --- /dev/null +++ b/docs/command-examples/drivers.md @@ -0,0 +1,20 @@ +## Examples + +```shell +relix drivers ls +``` +``` +── drivers ───────────────────────────────────────────────────────────────────── + name scheme installed + ────────── ──────────────── ───────── + postgresql jdbc:postgresql: false + mysql jdbc:mysql: false +(2 rows) +``` + +`relix drivers install NAME` downloads one of those drivers, checks its checksum, and +installs it in `~/.relix/drivers`, or in `$RELIX_DRIVERS` when that is set. A run that +needs a driver that is not installed fails with exit status 5, unless `--allow-download` +lets it fetch the driver itself. The H2 driver is part of +`relix`, and needs no installing. A driver `relix drivers ls` does not list is installed +by copying its jar into the driver directory. diff --git a/docs/command-examples/explain.md b/docs/command-examples/explain.md new file mode 100644 index 0000000..36bd51d --- /dev/null +++ b/docs/command-examples/explain.md @@ -0,0 +1,20 @@ +## Examples + +```shell +relix explain -e 'GROUP city, COUNT(*) -> n (Customers JOIN Orders)' +``` +``` +Aggregate +└─ Join NATURAL/HASH build=RIGHT + ├─ Scan Customers + └─ Scan Orders +``` + +`--json` writes each query's plan as a JSON object instead: + +```shell +relix explain --json -e 'SELECT amount > 100 (Orders)' | jq -c 'keys' +``` +``` +["plan","query","script"] +``` diff --git a/docs/command-examples/fmt.md b/docs/command-examples/fmt.md new file mode 100644 index 0000000..582132f --- /dev/null +++ b/docs/command-examples/fmt.md @@ -0,0 +1,61 @@ +## Examples + +```shell +cat scripts/messy.relix +``` +``` +-- Orders worth a look +Large:={select amount>100(Orders)}; +query {project order_id,amount (Large)}; +``` + +`relix fmt` prints it canonically — one statement per line, spaced, each operator in one +spelling — keeping its comments: + +```shell +relix fmt scripts/messy.relix +``` +``` +-- Orders worth a look +Large := { σ amount > 100 (Orders) }; +query { π order_id, amount (Large) }; +``` + +`--keywords` writes the operators as ASCII keywords instead of glyphs: + +```shell +relix fmt --keywords scripts/messy.relix +``` +``` +-- Orders worth a look +Large := { SELECT amount > 100 (Orders) }; +query { PROJECT order_id, amount (Large) }; +``` + +`--check` changes nothing and names each file that would change, exiting 1 if there is +one; `-w` rewrites each such file in place: + +```shell +relix fmt --check --keywords scripts/messy.relix scripts/by-city.relix; echo "exit $?" +``` +``` +scripts/messy.relix +exit 1 +``` + +```shell +relix fmt -w --keywords scripts/messy.relix +relix fmt --check --keywords scripts/messy.relix; echo "exit $?" +``` +``` +exit 0 +``` + +With no file, `relix fmt` formats standard input to standard output: + +```shell +echo 'query {select amount>100(Orders)};' | relix fmt +``` +``` +query { σ amount > 100 (Orders) }; +``` diff --git a/docs/command-examples/help.md b/docs/command-examples/help.md new file mode 100644 index 0000000..f32a983 --- /dev/null +++ b/docs/command-examples/help.md @@ -0,0 +1,12 @@ +## Examples + +`relix help COMMAND` prints what `relix COMMAND --help` prints: + +```shell +relix help version | head -3 +``` +``` +Usage: relix version [-hNqVv] [--allow-download] [--remote] [-C=DIR] + [--max-fixpoint-rounds=N] [--max-materialized-rows=N] + [--max-processed-rows=N] [--now=INSTANT] [-P=NAME] +``` diff --git a/docs/command-examples/ir.md b/docs/command-examples/ir.md new file mode 100644 index 0000000..8e2cc5e --- /dev/null +++ b/docs/command-examples/ir.md @@ -0,0 +1,67 @@ +## Examples + +```shell +relix ir -e 'Big := { SELECT amount > 100 (Orders) };' -e 'query Big;' +``` +``` +════════════════════════════════════════════════════════════════════════════════ +RELIX IR namespace=default +════════════════════════════════════════════════════════════════════════════════ + kind: SRC=source INL=inline SYS=system QR=view DB=database + FN=function TVF=table-fn LIT=truth-literal + type: N=number S=string B=boolean ?=any [x]=array {…}=struct + D=date T=time TS=timestamp DUR=duration + mat: [bag]=bag [set]=dedup-set [sort]=sorted (stream=no label) + +── SYMBOLS ───────────────────────────────────────────────────────────────────── + Big QR order_id:N customer_id:N amount:N status:S placed:D + BigSpenders QR name:S total:N + catalog SYS name:S kind:S arity:N schema_code:S + columns SYS relation:S column:S type:S ordinal:N distinct_count:N … + Completed QR order_id:N customer_id:N amount:N status:S placed:D + connections SYS name:S dialect:S + Customers SRC customer_id:N name:S city:S + cycles QR name:S + dependencies QR dependent:S depends_on:S + deps TVF (r:S)→relation + events SYS seq:N stage:S code:S description:S target:S rows:N el… + find TVF (col:S)→relation + funcs TVF (cat:S)→relation + functions SYS name:S category:S arity:N max_arity:N return_type:S pu… + impact TVF (r:S)→relation + keys SYS relation:S key:N ordinal:N column:S + Orders SRC order_id:N customer_id:N amount:N status:S placed:D + plan SYS query:S node_id:N parent_id:N ordinal:N op:S label:S … + Regions INL city:S region:S + relations SYS name:S kind:S namespace:S materialization:S row_count:N… + rules QR seq:N code:S description:S target:S + schema TVF (rel:S)→relation + Spend QR customer_id:N total:N + Stock SRC sku:S on_hand:N + unused QR name:S + version SYS component:S kind:S version:S + +── EXPRESSION TREES ──────────────────────────────────────────────────────────── +Big [QR] order_id:N customer_id:N amount:N status:S placed:D + σ amount > 100 + └─ Orders [SRC] + +BigSpenders [QR] name:S total:N + π name, total + └─ ⋈ + ├─ Customers [SRC] + └─ σ total ≥ 200 + └─ Spend [QR] + +Completed [QR] order_id:N customer_id:N amount:N status:S placed:D + σ status = "completed" + └─ Orders [SRC] + +Spend [QR] customer_id:N total:N + γ [customer_id] SUM(amount)→total [bag] + └─ Completed [QR] + +── ROOT QUERIES ──────────────────────────────────────────────────────────────── + query Big [QR] +════════════════════════════════════════════════════════════════════════════════ +``` diff --git a/docs/command-examples/optimize.md b/docs/command-examples/optimize.md new file mode 100644 index 0000000..130dc2d --- /dev/null +++ b/docs/command-examples/optimize.md @@ -0,0 +1,39 @@ +## Examples + +The rewritten script is printed in the same form the engine runs it in: + +```shell +relix optimize -e "query { SELECT city = 'London' (Customers JOIN Orders) };" +``` +``` +query { (σ city = "London" (Customers)) ⋈ (Orders) }; +``` + +`--report` prints the rules that fired, and each query before and after: + +```shell +relix optimize --report -e "query { SELECT city = 'London' (Customers JOIN Orders) };" +``` +``` +════════════════════════════════════════════════════════════════════════════════ +RELIX OPTIMIZER namespace=default transformations=1 +════════════════════════════════════════════════════════════════════════════════ + +── SUMMARY ───────────────────────────────────────────────────────────────────── + SEL-005 Selection pushed into join input ...................................×1 + +── TREES ─────────────────────────────────────────────────────────────────────── + +── + before: σ city = "London" ((Customers) ⋈ (Orders)) + after: (σ city = "London" (Customers)) ⋈ (Orders) + +── DETAIL ────────────────────────────────────────────────────────────────────── + +── (1 transformation) + [SEL-005] selection pushed into left input of natural join @ :1:9 +════════════════════════════════════════════════════════════════════════════════ +``` + +[Pipelines](../pipelines.md#optimise-a-script-and-review-the-change) compares a script +with its rewrite. diff --git a/docs/command-examples/provenance.md b/docs/command-examples/provenance.md new file mode 100644 index 0000000..8046b59 --- /dev/null +++ b/docs/command-examples/provenance.md @@ -0,0 +1,54 @@ +## Examples + +The default semiring, `counting`, says in how many ways each row is derived. Two London +customers each derive the row `London`: + +```shell +relix provenance -e 'PROJECT city (Customers)' +``` +``` + +------ ------ +city [prov] +------ ------ +London 2 +Leeds 1 +York 1 +------ ------ +(3 tuples; provenance semiring: counting) +``` + +The `lineage` semiring names the source rows each result row came from: + +```shell +relix provenance --semiring=lineage -e 'PROJECT city (SELECT amount > 100 (Customers JOIN Orders))' +``` +``` + +------ ------------------------------------------- +city [prov] +------ ------------------------------------------- +London Customers#1·Orders#1 + Customers#3·Orders#4 +------ ------------------------------------------- +(1 tuple; provenance semiring: lineage) +``` + +In a pipe, or with `-o json`, the rows and their annotations are one JSON document: + +```shell +relix provenance -e 'PROJECT city (Customers)' -o json +``` +``` +{ + "query": "", + "semiring": "counting", + "schema": ["city"], + "tuples": [ + {"row": {"city": "London"}, "provenance": "2"}, + {"row": {"city": "Leeds"}, "provenance": "1"}, + {"row": {"city": "York"}, "provenance": "1"} + ] +} +``` + +The engine's [provenance page](../../provenance.md) explains each semiring. diff --git a/docs/command-examples/relix.md b/docs/command-examples/relix.md new file mode 100644 index 0000000..1dd61b9 --- /dev/null +++ b/docs/command-examples/relix.md @@ -0,0 +1,57 @@ +## Examples + +With no command, `relix` runs scripts, as [`relix run`](run.md) does: + +```shell +relix -e 'PROJECT name, city (Customers)' -o tsv +``` +``` +name city +Alice London +Bob Leeds +Carol London +Dan York +``` + +Global options go before or after the command's name: + +```shell +relix -C reports catalog ls -o tsv --no-header +``` +``` +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/reports/.relix/catalog/10-sample.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +## Environment + +| Variable | Effect | +|---|---| +| `RELIX_OUTPUT` | the output format when `-o` is not given | +| `RELIX_PROFILE` | the profile when `-P` is not given | +| `RELIX_NOW` | the instant `NOW()` returns when `--now` is not given | +| `RELIX_CATALOG_PATH` | catalog files to load, separated as in `PATH`, before the `--catalog` files | +| `RELIX_TRUST_ALL` | `1` or `true` trusts every project's `.relix/` directory | +| `RELIX_DRIVERS` | the directory JDBC drivers are installed in and loaded from, instead of `~/.relix/drivers` | +| `NO_COLOR` | when set, a table on a terminal is not coloured unless `--color=always` | +| `PAGER` | the program [`relix doc`](doc.md) shows a page through on a terminal; `cat` or empty shows it directly | + +A `${NAME}` placeholder in a declaration may also be resolved from the environment; see +[Secrets and profiles](../secrets.md). + +## Files + +| Path | What it holds | +|---|---| +| `~/.relix/catalog/*.relix` | your own catalog, which every directory under your home sees | +| `~/.relix/profiles.json` | your own profiles | +| `~/.relix/trusted` | the projects you have trusted, with a hash of each one's files | +| `~/.relix/drivers/` | installed JDBC drivers | +| `~/.relix/connectors/` | installed connector plugins | +| `~/.relix/cache/files/` | files fetched over `https` by a file connection | +| `.relix/catalog/*.relix`, `.relix/relixrc`, `.relix/profiles.json`, `.relix/root` | a project's catalog, defaults, profiles, and the marker that ends the walk up; see [The catalog](../catalog.md) | diff --git a/docs/command-examples/run.md b/docs/command-examples/run.md new file mode 100644 index 0000000..5da0dd2 --- /dev/null +++ b/docs/command-examples/run.md @@ -0,0 +1,40 @@ +## Examples + +`relix run` is what `relix` does with no command, so these two are the same: + +```shell +relix run -e 'GROUP status, SUM(amount) -> total (Orders)' +``` +``` +── ────────────────────────────────────────────────────────────── + status total + ───────── ───── + completed 400 + pending 50 +(2 rows) +``` + +```shell +relix -e 'GROUP status, SUM(amount) -> total (Orders)' -o csv +``` +``` +status,total +completed,400 +pending,50 +``` + +Each script file is a run of its own, with its own declarations: + +```shell +relix run scripts/top-customers.relix -a city=Leeds scripts/by-city.relix -o tsv +``` +``` +name total +Alice 200 +Carol 200 +name +Bob +``` + +[Binding inputs](../inputs.md), [Output formats](../output.md) and +[Pipelines](../pipelines.md) cover the options in use. diff --git a/docs/command-examples/trace.md b/docs/command-examples/trace.md new file mode 100644 index 0000000..f453d2c --- /dev/null +++ b/docs/command-examples/trace.md @@ -0,0 +1,24 @@ +## Examples + +Each event ends with how long it took, which varies from run to run, so here `cut` +leaves the timings off: + +```shell +relix trace -e 'SELECT amount > 100 (Orders)' | cut -d'(' -f1 +``` +``` +[EXECUTE ] orders SCAN scanned 4 rows +[EXECUTE ] ROWS query delivered 2 rows +``` + +`relix run --trace` writes the same feed to standard error while the rows go to standard +output, and `--trace=FILE` writes it to a file: + +```shell +relix --trace=trace.txt -e 'SELECT amount > 100 (Orders)' -o tsv +``` +``` +order_id customer_id amount status placed +1 1 120 completed 2026-09-02 +4 3 200 completed 2026-10-01 +``` diff --git a/docs/command-examples/version.md b/docs/command-examples/version.md new file mode 100644 index 0000000..e181891 --- /dev/null +++ b/docs/command-examples/version.md @@ -0,0 +1,11 @@ +## Examples + +`relix version`, like `relix --version`, prints one line: the version of the command, +then the version of the Relix engine it runs, as + +```text +relix 1.0.0 (engine 1.0.0) +``` + +The two are numbered independently: a release of the command names the engine release +it was built on. diff --git a/docs/design.md b/docs/design.md index 6ae03dd..1e84099 100644 --- a/docs/design.md +++ b/docs/design.md @@ -407,6 +407,13 @@ version of it. `.relix/` tree and fixture files, comparing stdout and the exit status — so a page cannot be published with an example the command does not honour. +The pages live in `docs/guide/command-line`, laid out as the engine's own manuals are +(a `README.md` index of chapters and pages), with `index.json` as the machine-readable +index. The site renders the section inside the engine's programming guide, at +`guide/command-line/`, so a link out of it is written as if it sat in the engine's +`docs/guide`. Until releases attach the archive, the site pins a commit of this +repository and reads the same tree from it. + ### 7.5 Tests Each command end-to-end through `Main` with captured streams; catalog discovery and diff --git a/docs/guide/command-line/README.md b/docs/guide/command-line/README.md new file mode 100644 index 0000000..5f45f38 --- /dev/null +++ b/docs/guide/command-line/README.md @@ -0,0 +1,65 @@ +# The relix command line + +Using Relix from a shell: the `relix` command, which runs scripts as one stage of a Unix +pipeline, and the commands beside it that check, explain, optimise and format them. + +`relix` writes rows to standard output and everything else to standard error. A script +comes from `-e`, a file or standard input; its data from files, standard input, or the +tables a project declares in its `.relix/` directory; and its rows leave as a table on a +terminal, or tab-separated values, CSV, NDJSON, JSON or Markdown for the next program. + +Every example in this section is run on each build of the command, against a small +project, and the block under it is what it printed. The command reference is generated +from the same model as `relix --help` and the man pages. + +## Using the command + +Installing it, and the shape every command shares; then data in, rows out. + +| Page | What it covers | +|---|---| +| [The relix command](getting-started.md) | Installing; a first query; where the script comes from; the commands; diagnostics and exit status | +| [Binding inputs](inputs.md) | `-i` and its formats; standard input; inferred and given headings; inputs that never end; shadowing the catalog; query parameters with `-a` | +| [Output formats](output.md) | `table`, `tsv`, `csv`, `ndjson`, `json` and `markdown`; NULLs and headers; scripts with several queries; `--fail-empty` and `--fail-rows`; messages; closed pipes and interrupts | + +## A project's context + +What a directory gives the commands run in it. + +| Page | What it covers | +|---|---| +| [The catalog](catalog.md) | `.relix/` directories and how they are found; trusting a project; a nearer declaration wins; inspecting the catalog; `--catalog`; `relixrc` | +| [Secrets and profiles](secrets.md) | `${NAME}` placeholders; `profiles.json` and `-P`; the environment and `-D`; private profile files; what is never printed | + +## Recipes + +| Page | What it covers | +|---|---| +| [Pipelines](pipelines.md) | With `jq`, `awk`, `sort` and `head`; with `find` and `xargs`; scripts in here-documents; a data check in CI; optimising a script and reviewing the change | + +## Command reference + +One page per command: what it does, its usage, arguments and options, its exit status, +and examples. + +| Page | What it covers | +|---|---| +| [`relix`](commands/relix.md) | The global options every command takes; the exit statuses; the environment variables and files | +| [`relix run`](commands/run.md) | Runs scripts and writes their queries' rows | +| [`relix check`](commands/check.md) | Analyses scripts without running them, and reports their problems | +| [`relix explain`](commands/explain.md) | Prints each query's physical plan | +| [`relix optimize`](commands/optimize.md) | Prints the script with each query rewritten by the optimiser | +| [`relix trace`](commands/trace.md) | Runs each query and prints what the engine rewrote, planned and ran | +| [`relix bundle`](commands/bundle.md) | Prints each script's JSON bundle of diagnostics, plans and events | +| [`relix ir`](commands/ir.md) | Prints the symbols, views and queries the analyser made of a script | +| [`relix provenance`](commands/provenance.md) | Runs each query and annotates each row with where it came from | +| [`relix fmt`](commands/fmt.md) | Prints scripts canonically, keeping their comments | +| [`relix catalog`](commands/catalog.md) | Lists the catalog's files, relations, columns and declarations; trusts a project | +| [`relix drivers`](commands/drivers.md) | Lists and installs JDBC drivers | +| [`relix connectors`](commands/connectors.md) | Lists and installs connector plugins | +| [`relix doc`](commands/doc.md) | Shows a page of the language reference | +| [`relix help`](commands/help.md) | Prints a command's usage | +| [`relix version`](commands/version.md) | Prints the versions of the command and of the engine | +| [`relix completion`](commands/completion.md) | Prints a completion script for bash, zsh, fish or PowerShell | + +A machine-readable index of these pages is `index.json`, beside this one. diff --git a/docs/guide/command-line/catalog.md b/docs/guide/command-line/catalog.md new file mode 100644 index 0000000..1078175 --- /dev/null +++ b/docs/guide/command-line/catalog.md @@ -0,0 +1,328 @@ +# The catalog + +A project's tables, connections and views are declared once, in its `.relix/` directory, +and every command run inside the project can name them. `relix` finds those directories by +walking up from where it starts, the way `git` finds `.git`, so the same command works +from any subdirectory. This page covers how the directories are laid out and found, how +a nearer declaration replaces a farther one, how to see what a run will load, and the +trust a project's directory needs before anything in it is loaded. + +## The layout + +A `.relix/` directory holds: + +| Path | What it holds | +|---|---| +| `catalog/*.relix` | the declarations: `source`, `connection`, views (`Name := { … };`), `def`, `import` | +| `relixrc` | defaults for the commands run below it: the output format and the profile | +| `profiles.json` | named sets of `${VAR}` values; see [Secrets and profiles](secrets.md) | +| `root` | a marker that stops the walk up at this directory | + +The shop the examples use has one: + +```shell +find .relix -type f | sort +``` +``` +.relix/catalog/00-sources.relix +.relix/catalog/10-views.relix +.relix/catalog/20-warehouse.relix +.relix/profiles.json +``` + +```shell +cat .relix/catalog/00-sources.relix +``` +``` +-- The shop's data. A relative path resolves against this file's directory. +source Customers from csv("../../data/customers.csv") { + header: true, + schema: { customer_id: NUMBER, name: STRING, city: STRING } +}; + +source Orders from csv("../../data/orders.csv") { + header: true, + schema: { order_id: NUMBER, customer_id: NUMBER, amount: NUMBER, status: STRING, placed: DATE } +}; +``` + +```shell +cat .relix/catalog/10-views.relix +``` +``` +Completed := { SELECT status = 'completed' (Orders) }; + +Spend := { GROUP customer_id, SUM(amount) -> total (Completed) }; + +BigSpenders := { PROJECT name, total (Customers JOIN (SELECT total >= 200 (Spend))) }; +``` + +A catalog file holds declarations only; a `query` statement in one is an error. A +relative path in a catalog file resolves against the file's own directory, not the +directory the command runs in, so `../../data/orders.csv` above means `~/shop/data` +from wherever in the project `relix` is started. + +Declaring costs nothing at run time: no file is read and no connection opened until a +query reaches the name. A large catalog costs a parse and an analysis, not a connection. + +## Finding the directories + +A run's catalog comes from these directories, loaded **outermost first**: + +1. `~/.relix`, your own, which every directory under your home sees. +2. Each `.relix/` from the outermost directory down to the one the command starts in, + walking up from it and stopping at your home directory. A `.relix/` that holds a file + named `root` ends the walk there, so the projects of a monorepo do not inherit each + other's declarations. Started outside your home directory, `relix` reads only that + directory's own `.relix/`. +3. The files named by `$RELIX_CATALOG_PATH`, separated as in `PATH`, then those named + with `--catalog FILE`, in order. + +Within a directory, `catalog/*.relix` load in the lexical order of their names, so +`00-sources.relix` comes before `10-views.relix`, as in `/etc/*.d`. + +`-N` (`--no-catalog`) leaves out every `.relix/` directory's catalog and `relixrc`; +`--catalog` files still load. `-C DIR` starts the walk from `DIR`. + +## A project must be trusted + +Loading code because of the directory you are standing in is a risk: a cloned +repository's `.relix/` could declare a source that sends a `${TOKEN}` from your +environment to a server of its author's choosing. So a project's `.relix/` is loaded only +once you have trusted it, as `direnv` does with an `.envrc`. Until then, it is skipped, +with a warning saying how to trust it, and a run that needs one of its names fails with +exit status 5: + +```shell +relix -e 'Customers' 2>&1; echo "exit $?" +``` +``` +relix: warning: skipping ~/shop/.relix, which is not trusted; to load it: relix catalog trust ~/shop +-e:1:1: error: Undefined relation: 'Customers' +relix: Customers is declared in ~/shop/.relix/catalog/00-sources.relix, which is not trusted; to load it: relix catalog trust ~/shop +exit 5 +``` + +`relix catalog trust` trusts the working directory's project, or the one it names. It +records a hash of every file under the `.relix/` directory in `~/.relix/trusted`: + +```shell +relix catalog trust +relix -e 'PROJECT name (Customers)' -o tsv +``` +``` +name +Alice +Bob +Carol +Dan +``` + +Changing, adding or removing any file under a trusted `.relix/` — a catalog file, the +`relixrc`, the `profiles.json` — makes it untrusted again until it is trusted again, so a +`git pull` cannot change what runs without you seeing it. `--list` shows each trusted +project and whether it has changed since: + +```shell +echo "Pending := { SELECT status = 'pending' (Orders) };" > .relix/catalog/30-pending.relix +relix catalog trust --list -o tsv +``` +``` +directory state +~/shop changed +``` + +```shell +relix catalog trust +relix catalog trust --list -o tsv +``` +``` +directory state +~/shop trusted +``` + +`--revoke DIR` removes a project from the record. + +Some things are trusted without asking, because you chose them: `~/.relix`, a script +named on the command line or given with `-e` or on standard input, and `--catalog` and +`$RELIX_CATALOG_PATH` files. `RELIX_TRUST_ALL=1` trusts every directory, for a CI +container whose checkout is the only thing in it. + +## A nearer declaration wins + +When two directories declare the same name, the **nearer** one's declaration replaces the +farther one's. That is an overlay, not an error, and it is what lets a sub-project change +one table while keeping everything else above it. `~/shop/reports` declares its own +`Orders`, a fixed sample to write reports against: + +```shell +cat reports/.relix/catalog/10-sample.relix +``` +``` +-- Reports are written against a fixed sample rather than the live orders. +source Orders from csv("../../sample-orders.csv") { + header: true, + schema: { order_id: NUMBER, customer_id: NUMBER, amount: NUMBER, status: STRING, placed: DATE } +}; +``` + +Every view the shop declares reads whichever `Orders` is nearest, so `BigSpenders` from +inside `reports` is computed over the sample: + +```shell +relix catalog trust reports +``` + +```shell +relix -e 'BigSpenders' +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 200 + Carol 200 +(2 rows) +``` + +```shell +relix -C reports -e 'BigSpenders' +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 500 + Dan 250 +(2 rows) +``` + +Each file is parsed first, the nearest declaration of every name kept, and only then are +the survivors defined together, so the order of the files decides which declaration +wins, never which can see which. A name is replaced only by a declaration of the same +kind: a relation by a relation, a function by a function. + +A file in an untrusted directory neither declares nor replaces anything. + +## Inspecting the catalog + +`relix catalog` answers what a run would load, without running anything: + +| Command | What it lists | +|---|---| +| `relix catalog files` | every catalog file, in load order, whether it is trusted, and the names it declares | +| `relix catalog ls` | every relation: its name, kind, number of columns, and the file whose declaration won | +| `relix catalog schema NAME` | the columns of a relation and their types | +| `relix catalog where NAME` | every declaration of a name, nearest first: the one that won, then those it replaced | + +Each writes rows like any query, as a table on a terminal and in any `-o` format: + +```shell +relix catalog files -o tsv +``` +``` +file trusted declares +~/.relix/catalog/00-regions.relix true Regions +~/shop/.relix/catalog/00-sources.relix true Customers Orders +~/shop/.relix/catalog/10-views.relix true Completed Spend BigSpenders +~/shop/.relix/catalog/20-warehouse.relix true warehouse Stock +~/shop/.relix/catalog/30-pending.relix true Pending +``` + +```shell +relix catalog ls -o tsv +``` +``` +name kind arity file +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/.relix/catalog/00-sources.relix +Pending QR 5 ~/shop/.relix/catalog/30-pending.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +The `kind` is the engine's: `SRC` a source, `INL` an inline table, `QR` a view, `DB` a +table reached through a connection. + +```shell +relix catalog schema Orders +``` +``` +── Orders ────────────────────────────────────────────────────────────────────── + column type + ─────────── ──── + order_id N + customer_id N + amount N + status S + placed D +(5 rows) +``` + +The type codes are the engine's: `N` a number, `S` a string, `B` a boolean, `D` a date, +`T` a time, `TS` a timestamp, `DUR` a duration, `?` any type. + +```shell +relix -C reports catalog where Orders -o tsv +``` +``` +file line kind state +~/shop/reports/.relix/catalog/10-sample.relix 2 source wins +~/shop/.relix/catalog/00-sources.relix 7 source shadowed +``` + +## Catalog files from elsewhere + +`--catalog FILE` adds a file to the catalog for one run, nearer than any `.relix/` +directory's. `$RELIX_CATALOG_PATH` names files the same way for every run. Both are +trusted, since you named them: + +```shell +cat > ~/rates.relix <<'EOF' +Rates := [ +| status | fee | +|-----------|-----| +| completed | 2 | +| pending | 0 | +]; +EOF +relix --catalog ~/rates.relix -e 'PROJECT order_id, fee (Orders JOIN Rates)' -o tsv +``` +``` +order_id fee +1 2 +2 2 +3 0 +4 2 +``` + +## Defaults: relixrc + +A `relixrc` file sets defaults for the commands run below it, as `key = value` lines; a +line starting `#` is a comment. The keys are: + +| Key | Default it sets | Beaten by | +|---|---|---| +| `output` | the output format | `-o`, `$RELIX_OUTPUT` | +| `profile` | the profile `${VAR}` placeholders resolve from | `-P`, `$RELIX_PROFILE` | + +The nearest file's value for a key wins. A key the command does not know, or a line that +is not `key = value`, is reported as a warning and ignored, so a file written for a later +version still works. A `relixrc` is part of its `.relix/` directory, so it is read only +once the directory is trusted: + +```shell +printf 'output = csv\n' > .relix/relixrc +relix catalog trust +relix -e 'PROJECT name (Customers)' +``` +``` +name +Alice +Bob +Carol +Dan +``` diff --git a/docs/guide/command-line/commands/bundle.md b/docs/guide/command-line/commands/bundle.md new file mode 100644 index 0000000..81ef74d --- /dev/null +++ b/docs/guide/command-line/commands/bundle.md @@ -0,0 +1,38 @@ +# relix bundle + +Prints each script's JSON bundle: diagnostics and, per query, the logical plan, the rewrites, the physical plan and the planner's events. A script that does not analyse still has one, holding its diagnostics. + +## Usage + +```text +relix bundle [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +A bundle is one JSON document per script, for a tool such as a query playground: + +```shell +relix bundle -e 'SELECT amount > 100 (Orders)' | jq -c 'keys' +``` +``` +["diagnostics","namespace","queries"] +``` diff --git a/docs/guide/command-line/commands/catalog.md b/docs/guide/command-line/commands/catalog.md new file mode 100644 index 0000000..e8ba414 --- /dev/null +++ b/docs/guide/command-line/commands/catalog.md @@ -0,0 +1,195 @@ +# relix catalog + +Inspects the catalog: the declarations a run takes from the .relix/ directories from ~/.relix down to the working directory, and from --catalog files. A project's .relix/ is loaded only once it is trusted (relix catalog trust). + +## Usage + +```text +relix catalog [OPTIONS] [COMMAND] +``` + +It also takes the [global options](relix.md#global-options). + +## relix catalog files + +Lists every catalog file, in the order they load, with the names each declares and whether it is trusted. An untrusted file is listed but not loaded. + +### Usage + +```text +relix catalog files [OPTIONS] +``` + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## relix catalog ls + +Lists the relations the catalog declares: name, kind, arity and the file whose declaration won. kind is the engine's: SRC a source, INL an inline table, QR a view, DB a connection's table. + +### Usage + +```text +relix catalog ls [OPTIONS] +``` + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## relix catalog schema + +Lists the columns of the relation NAME and their types, in order. + +### Usage + +```text +relix catalog schema [OPTIONS] NAME +``` + +### Arguments + +| Argument | Description | +|---|---| +| `NAME` | A relation the catalog declares. | + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +### Exit status + +| Status | Meaning | +|---|---| +| 0 | success | +| 3 | the catalog declares no relation NAME, or does not analyse | + +## relix catalog where + +Lists every declaration of NAME, nearest first: the one that won, then the ones it shadowed. A declaration in an untrusted directory is listed as untrusted. + +### Usage + +```text +relix catalog where [OPTIONS] NAME +``` + +### Arguments + +| Argument | Description | +|---|---| +| `NAME` | A name the catalog declares. | + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +### Exit status + +| Status | Meaning | +|---|---| +| 0 | success | +| 3 | the catalog declares no NAME | + +## relix catalog trust + +Trusts the .relix/ directory of DIR (default: the working directory), as its files are now: a project's catalog, relixrc and profiles.json are loaded only once it is trusted, and changing, adding or removing a file there means trusting it again. ~/.relix, --catalog files and the script itself are always trusted; RELIX_TRUST_ALL=1 trusts everything, for a CI container. + +### Usage + +```text +relix catalog trust [OPTIONS] [DIR] +``` + +### Arguments + +| Argument | Description | +|---|---| +| `DIR` | The project to trust, or its .relix/ directory. | + +### Options + +| Option | Description | +|---|---| +| `--list` | List the trusted directories and whether each has changed since. | +| `--revoke=DIR` | Stop trusting DIR. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +```shell +relix catalog ls -o tsv +``` +``` +name kind arity file +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/.relix/catalog/00-sources.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +```shell +relix catalog schema Customers -o tsv +``` +``` +column type +customer_id N +name S +city S +``` + +```shell +relix catalog where Orders -o tsv +``` +``` +file line kind state +~/shop/.relix/catalog/00-sources.relix 7 source wins +``` + +```shell +relix catalog trust --list -o tsv +``` +``` +directory state +~/shop trusted +~/shop/reports trusted +``` + +[The catalog](../catalog.md) covers discovery, the overlay and trust. diff --git a/docs/guide/command-line/commands/check.md b/docs/guide/command-line/commands/check.md new file mode 100644 index 0000000..253c6ca --- /dev/null +++ b/docs/guide/command-line/commands/check.md @@ -0,0 +1,70 @@ +# relix check + +Analyses scripts without running them, and reports what is wrong with them on standard error as FILE:LINE:COL: severity: message. + +```text +find . -name '*.relix' -print0 | xargs -0 relix check +``` + +## Usage + +```text +relix check [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--json` | Write the diagnostics to standard output as one JSON document instead. | + +It also takes the [global options](relix.md#global-options). + +## Exit status + +| Status | Meaning | +|---|---| +| 0 | every script analyses | +| 3 | a script does not parse or analyse | + +## Examples + +A script that analyses prints nothing, and exits 0: + +```shell +relix check scripts/top-customers.relix +``` + +A problem is printed on standard error, and the status is 3: + +```shell +relix check scripts/broken.relix 2>&1; echo "exit $?" +``` +``` +scripts/broken.relix:1:30: error: Undefined relation: 'Ordrs' +exit 3 +``` + +`--json` writes the diagnostics to standard output instead, as one document for a tool to +read: + +```shell +relix check --json scripts/broken.relix; echo "exit $?" +``` +``` +{"diagnostics":[{"file":"scripts/broken.relix","line":1,"column":30,"severity":"error","message":"Undefined relation: 'Ordrs'"}]} +exit 3 +``` + +[Pipelines](../pipelines.md#lint-every-script) checks every script in a tree. diff --git a/docs/guide/command-line/commands/completion.md b/docs/guide/command-line/commands/completion.md new file mode 100644 index 0000000..023a3f9 --- /dev/null +++ b/docs/guide/command-line/commands/completion.md @@ -0,0 +1,39 @@ +# relix completion + +Prints a completion script for SHELL: bash, zsh, fish or powershell. + +```text +bash: source <(relix completion bash) +zsh: source <(relix completion zsh) +fish: relix completion fish > ~/.config/fish/completions/relix.fish +powershell: relix completion powershell | Out-String | Invoke-Expression +``` + +## Usage + +```text +relix completion [OPTIONS] SHELL +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SHELL` | bash, zsh, fish or powershell. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +The release archives carry each shell's script in `completions/`, and Homebrew installs +them. To load one by hand, add the line for your shell to its startup file: + +```text +bash source <(relix completion bash) +zsh source <(relix completion zsh) +fish relix completion fish > ~/.config/fish/completions/relix.fish +PowerShell relix completion powershell | Out-String | Invoke-Expression +``` + +The zsh script is also a completion function, so it can be saved as `_relix` in a +directory on `$fpath` instead. diff --git a/docs/guide/command-line/commands/connectors.md b/docs/guide/command-line/commands/connectors.md new file mode 100644 index 0000000..a6b7fc7 --- /dev/null +++ b/docs/guide/command-line/commands/connectors.md @@ -0,0 +1,82 @@ +# relix connectors + +Lists and installs connector plugins, such as mongodb. A source of a type no installed connector reads needs one, installed here ahead of a run, or fetched during one with --allow-download. + +## Usage + +```text +relix connectors [OPTIONS] [COMMAND] +``` + +It also takes the [global options](relix.md#global-options). + +## relix connectors ls + +Lists the connectors installed: those shipped with relix and the plugins installed in ~/.relix/connectors. --available adds the ones the hosted catalog offers, which it fetches. + +### Usage + +```text +relix connectors ls [OPTIONS] +``` + +### Options + +| Option | Description | +|---|---| +| `--available` | Also list the connectors that can be installed, from the hosted catalog. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## relix connectors install + +Downloads the connector plugin TYPE, checks its checksums, and installs it. + +### Usage + +```text +relix connectors install [OPTIONS] TYPE +``` + +### Arguments + +| Argument | Description | +|---|---| +| `TYPE` | A connector type, such as mongodb. | + +It also takes the [global options](relix.md#global-options). + +### Exit status + +| Status | Meaning | +|---|---| +| 0 | installed, or already installed | +| 2 | the catalog has no connector of that type | +| 5 | the download failed | + +## Examples + +The connectors that ship with `relix`, and any plugins installed in `~/.relix/connectors`: + +```shell +relix connectors ls +``` +``` +── connectors ────────────────────────────────────────────────────────────────── + type installed + ────── ───────── + clf true + csv true + gedcom true + log true +(4 rows) +``` + +`relix connectors ls --available` adds the connectors that can be installed, which it +fetches from the hosted catalog, and `relix connectors install TYPE` installs one. A +source whose type no installed connector reads — `mongodb`, say — fails with exit status +5, unless `--allow-download` lets the run fetch the connector. diff --git a/docs/guide/command-line/commands/doc.md b/docs/guide/command-line/commands/doc.md new file mode 100644 index 0000000..5108066 --- /dev/null +++ b/docs/guide/command-line/commands/doc.md @@ -0,0 +1,75 @@ +# relix doc + +Shows the reference page for TOPIC, found by glyph, keyword or name (σ, select, selection), through $PAGER on a terminal. With no TOPIC, lists the pages. + +## Usage + +```text +relix doc [OPTIONS] [TOPIC] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `TOPIC` | A glyph, keyword or name, such as σ, select, join or Len. | + +## Options + +| Option | Description | +|---|---| +| `--function` | Look TOPIC up among the functions only, or list only them. | +| `--markdown` | Print the page's markdown as it is, not rendered. | +| `--no-pager` | Write the page to standard output even on a terminal. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## Exit status + +| Status | Meaning | +|---|---| +| 0 | success | +| 2 | no page has that topic | + +## Examples + +A page is found by its glyph, its keyword or its name: + +```shell +relix doc --markdown σ | head -6 +``` +``` +# Name: Selection (σ / SELECT) + +# Syntax: +σ (Relation) +SELECT (Relation) +``` + +With no topic, `relix doc` lists the pages, as rows: + +```shell +relix doc -o tsv | head -4 +``` +``` +topic category title summary +getting-started guide Getting started Your first script — declare data, name a result, run it +glossary guide Glossary The vocabulary — relation, tuple, bag vs set, materialisation, pushdown +σ operator Selection Filter rows by a condition +``` + +`--function` lists, or looks up, the functions only: + +```shell +relix doc --function -o tsv | grep -i '^round' +``` +``` +Round function Round Round to nearest +``` + +On a terminal a page is shown through `$PAGER` (`less` when it is not set); `--no-pager` +writes it straight to standard output. diff --git a/docs/guide/command-line/commands/drivers.md b/docs/guide/command-line/commands/drivers.md new file mode 100644 index 0000000..de2182d --- /dev/null +++ b/docs/guide/command-line/commands/drivers.md @@ -0,0 +1,79 @@ +# relix drivers + +Lists and installs JDBC drivers. A connection to a database needs its driver, installed here ahead of a run, or fetched during one with --allow-download. + +## Usage + +```text +relix drivers [OPTIONS] [COMMAND] +``` + +It also takes the [global options](relix.md#global-options). + +## relix drivers ls + +Lists the drivers relix can install: name, the JDBC scheme each serves, and whether one is installed. + +### Usage + +```text +relix drivers ls [OPTIONS] +``` + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## relix drivers install + +Downloads the driver NAME, checks its checksum, and installs it. + +### Usage + +```text +relix drivers install [OPTIONS] NAME +``` + +### Arguments + +| Argument | Description | +|---|---| +| `NAME` | A driver relix drivers ls lists, such as postgresql. | + +It also takes the [global options](relix.md#global-options). + +### Exit status + +| Status | Meaning | +|---|---| +| 0 | installed, or already installed | +| 2 | no driver has that name | +| 5 | the download failed | + +## Examples + +```shell +relix drivers ls +``` +``` +── drivers ───────────────────────────────────────────────────────────────────── + name scheme installed + ────────── ──────────────── ───────── + postgresql jdbc:postgresql: false + mysql jdbc:mysql: false +(2 rows) +``` + +`relix drivers install NAME` downloads one of those drivers, checks its checksum, and +installs it in `~/.relix/drivers`, or in `$RELIX_DRIVERS` when that is set. A run that +needs a driver that is not installed fails with exit status 5, unless `--allow-download` +lets it fetch the driver itself. The H2 driver is part of +`relix`, and needs no installing. A driver `relix drivers ls` does not list is installed +by copying its jar into the driver directory. diff --git a/docs/guide/command-line/commands/explain.md b/docs/guide/command-line/commands/explain.md new file mode 100644 index 0000000..4520817 --- /dev/null +++ b/docs/guide/command-line/commands/explain.md @@ -0,0 +1,50 @@ +# relix explain + +Prints the physical plan of each query: how the engine would run it. + +## Usage + +```text +relix explain [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--json` | Write one JSON object per query, {"script", "query", "plan"}, one per line. | +| `--query=NAME` | Explain only the query NAME. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +```shell +relix explain -e 'GROUP city, COUNT(*) -> n (Customers JOIN Orders)' +``` +``` +Aggregate +└─ Join NATURAL/HASH build=RIGHT + ├─ Scan Customers + └─ Scan Orders +``` + +`--json` writes each query's plan as a JSON object instead: + +```shell +relix explain --json -e 'SELECT amount > 100 (Orders)' | jq -c 'keys' +``` +``` +["plan","query","script"] +``` diff --git a/docs/guide/command-line/commands/fmt.md b/docs/guide/command-line/commands/fmt.md new file mode 100644 index 0000000..fdfed97 --- /dev/null +++ b/docs/guide/command-line/commands/fmt.md @@ -0,0 +1,96 @@ +# relix fmt + +Prints scripts canonically, keeping their comments. With no SCRIPT, formats standard input. + +## Usage + +```text +relix fmt [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files; - is standard input, which is also read when none is named. | + +## Options + +| Option | Description | +|---|---| +| `-w` | Rewrite each file that changes in place, instead of printing it. | +| `--check` | Change nothing; print the name of each file that would change, and exit 1 if any would. | +| `--glyphs` | Write operators as glyphs: σ, ⋈, ∧ (the default). | +| `--keywords` | Write operators as ASCII keywords: SELECT, JOIN, AND. | + +It also takes the [global options](relix.md#global-options). + +## Exit status + +| Status | Meaning | +|---|---| +| 0 | success; with --check, no file would change | +| 1 | with --check, a file would change | +| 3 | a script does not parse | + +## Examples + +```shell +cat scripts/messy.relix +``` +``` +-- Orders worth a look +Large:={select amount>100(Orders)}; +query {project order_id,amount (Large)}; +``` + +`relix fmt` prints it canonically — one statement per line, spaced, each operator in one +spelling — keeping its comments: + +```shell +relix fmt scripts/messy.relix +``` +``` +-- Orders worth a look +Large := { σ amount > 100 (Orders) }; +query { π order_id, amount (Large) }; +``` + +`--keywords` writes the operators as ASCII keywords instead of glyphs: + +```shell +relix fmt --keywords scripts/messy.relix +``` +``` +-- Orders worth a look +Large := { SELECT amount > 100 (Orders) }; +query { PROJECT order_id, amount (Large) }; +``` + +`--check` changes nothing and names each file that would change, exiting 1 if there is +one; `-w` rewrites each such file in place: + +```shell +relix fmt --check --keywords scripts/messy.relix scripts/by-city.relix; echo "exit $?" +``` +``` +scripts/messy.relix +exit 1 +``` + +```shell +relix fmt -w --keywords scripts/messy.relix +relix fmt --check --keywords scripts/messy.relix; echo "exit $?" +``` +``` +exit 0 +``` + +With no file, `relix fmt` formats standard input to standard output: + +```shell +echo 'query {select amount>100(Orders)};' | relix fmt +``` +``` +query { σ amount > 100 (Orders) }; +``` diff --git a/docs/guide/command-line/commands/help.md b/docs/guide/command-line/commands/help.md new file mode 100644 index 0000000..9f4359d --- /dev/null +++ b/docs/guide/command-line/commands/help.md @@ -0,0 +1,30 @@ +# relix help + +When no COMMAND is given, the usage help for the main command is displayed. If a COMMAND is specified, the help for that command is shown. + +## Usage + +```text +relix help [OPTIONS] [COMMAND] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `COMMAND` | The COMMAND to display the usage help message for. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +`relix help COMMAND` prints what `relix COMMAND --help` prints: + +```shell +relix help version | head -3 +``` +``` +Usage: relix version [-hNqVv] [--allow-download] [--remote] [-C=DIR] + [--max-fixpoint-rounds=N] [--max-materialized-rows=N] + [--max-processed-rows=N] [--now=INSTANT] [-P=NAME] +``` diff --git a/docs/guide/command-line/commands/ir.md b/docs/guide/command-line/commands/ir.md new file mode 100644 index 0000000..968419d --- /dev/null +++ b/docs/guide/command-line/commands/ir.md @@ -0,0 +1,95 @@ +# relix ir + +Prints the IR report: the symbols, views and queries the analyser made of each script. + +## Usage + +```text +relix ir [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +```shell +relix ir -e 'Big := { SELECT amount > 100 (Orders) };' -e 'query Big;' +``` +``` +════════════════════════════════════════════════════════════════════════════════ +RELIX IR namespace=default +════════════════════════════════════════════════════════════════════════════════ + kind: SRC=source INL=inline SYS=system QR=view DB=database + FN=function TVF=table-fn LIT=truth-literal + type: N=number S=string B=boolean ?=any [x]=array {…}=struct + D=date T=time TS=timestamp DUR=duration + mat: [bag]=bag [set]=dedup-set [sort]=sorted (stream=no label) + +── SYMBOLS ───────────────────────────────────────────────────────────────────── + Big QR order_id:N customer_id:N amount:N status:S placed:D + BigSpenders QR name:S total:N + catalog SYS name:S kind:S arity:N schema_code:S + columns SYS relation:S column:S type:S ordinal:N distinct_count:N … + Completed QR order_id:N customer_id:N amount:N status:S placed:D + connections SYS name:S dialect:S + Customers SRC customer_id:N name:S city:S + cycles QR name:S + dependencies QR dependent:S depends_on:S + deps TVF (r:S)→relation + events SYS seq:N stage:S code:S description:S target:S rows:N el… + find TVF (col:S)→relation + funcs TVF (cat:S)→relation + functions SYS name:S category:S arity:N max_arity:N return_type:S pu… + impact TVF (r:S)→relation + keys SYS relation:S key:N ordinal:N column:S + Orders SRC order_id:N customer_id:N amount:N status:S placed:D + plan SYS query:S node_id:N parent_id:N ordinal:N op:S label:S … + Regions INL city:S region:S + relations SYS name:S kind:S namespace:S materialization:S row_count:N… + rules QR seq:N code:S description:S target:S + schema TVF (rel:S)→relation + Spend QR customer_id:N total:N + Stock SRC sku:S on_hand:N + unused QR name:S + version SYS component:S kind:S version:S + +── EXPRESSION TREES ──────────────────────────────────────────────────────────── +Big [QR] order_id:N customer_id:N amount:N status:S placed:D + σ amount > 100 + └─ Orders [SRC] + +BigSpenders [QR] name:S total:N + π name, total + └─ ⋈ + ├─ Customers [SRC] + └─ σ total ≥ 200 + └─ Spend [QR] + +Completed [QR] order_id:N customer_id:N amount:N status:S placed:D + σ status = "completed" + └─ Orders [SRC] + +Spend [QR] customer_id:N total:N + γ [customer_id] SUM(amount)→total [bag] + └─ Completed [QR] + +── ROOT QUERIES ──────────────────────────────────────────────────────────────── + query Big [QR] +════════════════════════════════════════════════════════════════════════════════ +``` diff --git a/docs/guide/command-line/commands/optimize.md b/docs/guide/command-line/commands/optimize.md new file mode 100644 index 0000000..8fbd381 --- /dev/null +++ b/docs/guide/command-line/commands/optimize.md @@ -0,0 +1,68 @@ +# relix optimize + +Prints the script with each query rewritten by the optimiser, so that it works as a filter: relix optimize slow.relix > fast.relix + +## Usage + +```text +relix optimize [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--report` | Print the rules that fired and each query before and after, instead. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +The rewritten script is printed in the same form the engine runs it in: + +```shell +relix optimize -e "query { SELECT city = 'London' (Customers JOIN Orders) };" +``` +``` +query { (σ city = "London" (Customers)) ⋈ (Orders) }; +``` + +`--report` prints the rules that fired, and each query before and after: + +```shell +relix optimize --report -e "query { SELECT city = 'London' (Customers JOIN Orders) };" +``` +``` +════════════════════════════════════════════════════════════════════════════════ +RELIX OPTIMIZER namespace=default transformations=1 +════════════════════════════════════════════════════════════════════════════════ + +── SUMMARY ───────────────────────────────────────────────────────────────────── + SEL-005 Selection pushed into join input ...................................×1 + +── TREES ─────────────────────────────────────────────────────────────────────── + +── + before: σ city = "London" ((Customers) ⋈ (Orders)) + after: (σ city = "London" (Customers)) ⋈ (Orders) + +── DETAIL ────────────────────────────────────────────────────────────────────── + +── (1 transformation) + [SEL-005] selection pushed into left input of natural join @ :1:9 +════════════════════════════════════════════════════════════════════════════════ +``` + +[Pipelines](../pipelines.md#optimise-a-script-and-review-the-change) compares a script +with its rewrite. diff --git a/docs/guide/command-line/commands/provenance.md b/docs/guide/command-line/commands/provenance.md new file mode 100644 index 0000000..2c4346d --- /dev/null +++ b/docs/guide/command-line/commands/provenance.md @@ -0,0 +1,86 @@ +# relix provenance + +Runs each query and prints its rows, each annotated with where it came from in a provenance semiring. + +## Usage + +```text +relix provenance [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--semiring=NAME` | The semiring: boolean, counting, tropical, security, lineage, cheapest-route, or one an installed library offers (default: counting). | +| `--weight=COLUMN` | The column a weighted closure takes each edge's weight from, such as --semiring tropical --weight cost for shortest paths. | +| `-o`, `--output=FORMAT` | table or json (default: table on a terminal, json in a pipe). | +| `--query=NAME` | Annotate only the query NAME. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +The default semiring, `counting`, says in how many ways each row is derived. Two London +customers each derive the row `London`: + +```shell +relix provenance -e 'PROJECT city (Customers)' +``` +``` + +------ ------ +city [prov] +------ ------ +London 2 +Leeds 1 +York 1 +------ ------ +(3 tuples; provenance semiring: counting) +``` + +The `lineage` semiring names the source rows each result row came from: + +```shell +relix provenance --semiring=lineage -e 'PROJECT city (SELECT amount > 100 (Customers JOIN Orders))' +``` +``` + +------ ------------------------------------------- +city [prov] +------ ------------------------------------------- +London Customers#1·Orders#1 + Customers#3·Orders#4 +------ ------------------------------------------- +(1 tuple; provenance semiring: lineage) +``` + +In a pipe, or with `-o json`, the rows and their annotations are one JSON document: + +```shell +relix provenance -e 'PROJECT city (Customers)' -o json +``` +``` +{ + "query": "", + "semiring": "counting", + "schema": ["city"], + "tuples": [ + {"row": {"city": "London"}, "provenance": "2"}, + {"row": {"city": "Leeds"}, "provenance": "1"}, + {"row": {"city": "York"}, "provenance": "1"} + ] +} +``` + +The engine's [provenance page](../../provenance.md) explains each semiring. diff --git a/docs/guide/command-line/commands/relix.md b/docs/guide/command-line/commands/relix.md new file mode 100644 index 0000000..948ac64 --- /dev/null +++ b/docs/guide/command-line/commands/relix.md @@ -0,0 +1,155 @@ +# relix + +Runs Relix relational-algebra scripts as a stage of a Unix pipeline. Rows go to standard output; everything else goes to standard error. + +## Usage + +```text +relix [OPTIONS] [SCRIPT...] [COMMAND] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Run options + +With no command, `relix` is `relix run`, and takes its options: + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | +| `--line-buffered` | Flush each row as it is written, for a reader that wants it at once. | +| `--query=NAME` | Print only the query NAME. A machine format (tsv, csv, ndjson, json) prints one query: the script's last, unless this names another. | +| `--all` | With ndjson, print every query, each row tagged "_query": NAME. | +| `--fail-empty` | Exit 1 when the queries printed return no rows. | +| `--fail-rows` | Exit 1 when the queries printed return any row. | +| `--trace=FILE` | Write the event feed (what was rewritten, planned and run) to standard error, or to FILE with --trace=FILE, while the rows go to standard output. | + +## Global options + +Every command takes these, before or after its name. + +| Option | Description | +|---|---| +| `-C DIR` | Act as if started in DIR: relative paths and .relix/ discovery start there. | +| `-P`, `--profile=NAME` | The profile ${VAR} placeholders resolve from (default: $RELIX_PROFILE). | +| `-D NAME=VALUE` | Set one ${VAR} for this run; it beats the profile and the environment. Not for secrets: it lands in shell history and ps. | +| `-a`, `--arg=NAME=VALUE` | Bind the query parameter $NAME to VALUE. Repeatable. The value is bound, never pasted into the script, so it cannot change the query. | +| `-N`, `--no-catalog` | Ignore the .relix/ directories' catalogs and relixrc; --catalog files still load. | +| `--catalog=FILE` | Add a catalog file, nearer than the .relix/ directories'. Repeatable. $RELIX_CATALOG_PATH adds files too, separated as in PATH. | +| `-q`, `--quiet` | Print errors only, not warnings. | +| `-v`, `--verbose` | Also print notices (drivers loaded); -vv adds timings. | +| `--now=INSTANT` | Pin NOW() to an ISO-8601 instant, such as 2026-07-22T12:00:00Z (default: $RELIX_NOW). | +| `--max-fixpoint-rounds=N` | Fail a recursive query that needs more than N rounds. | +| `--max-materialized-rows=N` | Fail a query in which one blocking operator buffers more than N rows. | +| `--max-processed-rows=N` | Fail a query that processes more than N rows in all. | +| `--timeout=DURATION` | Fail a query that runs longer than DURATION: 500ms, 30s, 5m, 1h, or ISO-8601 (PT30S). | +| `--sandbox=FILE` | Run under the sandbox FILE describes. | +| `--remote` | Permit http(s) file locations. | +| `--allow-download` | Permit fetching a missing JDBC driver or connector during the run. | +| `-h`, `--help` | Show this help message and exit. | +| `-V`, `--version` | Print version information and exit. | + +## Commands + +| Command | What it does | +|---|---| +| [`relix run`](run.md) | Runs scripts and writes their queries' rows to standard output. | +| [`relix check`](check.md) | Analyses scripts without running them, and reports what is wrong with them on standard error as FILE:LINE:COL: severity: message. | +| [`relix explain`](explain.md) | Prints the physical plan of each query: how the engine would run it. | +| [`relix optimize`](optimize.md) | Prints the script with each query rewritten by the optimiser, so that it works as a filter: relix optimize slow.relix > fast.relix. | +| [`relix trace`](trace.md) | Runs each query and prints its event feed, what the engine rewrote, planned and ran, discarding the rows. | +| [`relix bundle`](bundle.md) | Prints each script's JSON bundle: diagnostics and, per query, the logical plan, the rewrites, the physical plan and the planner's events. | +| [`relix ir`](ir.md) | Prints the IR report: the symbols, views and queries the analyser made of each script. | +| [`relix provenance`](provenance.md) | Runs each query and prints its rows, each annotated with where it came from in a provenance semiring. | +| [`relix fmt`](fmt.md) | Prints scripts canonically, keeping their comments. | +| [`relix catalog`](catalog.md) | Inspects the catalog: the declarations a run takes from the .relix/ directories from ~/.relix down to the working directory, and from --catalog files. | +| [`relix drivers`](drivers.md) | Lists and installs JDBC drivers. | +| [`relix connectors`](connectors.md) | Lists and installs connector plugins, such as mongodb. | +| [`relix doc`](doc.md) | Shows the reference page for TOPIC, found by glyph, keyword or name (σ, select, selection), through $PAGER on a terminal. | +| [`relix help`](help.md) | When no COMMAND is given, the usage help for the main command is displayed. | +| [`relix version`](version.md) | Prints the versions of relix and of the Relix engine it runs. | +| [`relix completion`](completion.md) | Prints a completion script for SHELL: bash, zsh, fish or powershell. | + +## Exit status + +| Status | Meaning | +|---|---| +| 0 | success | +| 1 | an assertion failed (--fail-empty, --fail-rows) | +| 2 | usage error (bad option, nothing to run) | +| 3 | the script did not parse or analyse | +| 4 | execution failed (a data error, a limit hit) | +| 5 | environment (a missing driver, an untrusted catalog, a connection refused) | +| 70 | internal error in relix | +| 130 | interrupted (SIGINT) | +| 141 | standard output closed (SIGPIPE) | + +## Examples + +With no command, `relix` runs scripts, as [`relix run`](run.md) does: + +```shell +relix -e 'PROJECT name, city (Customers)' -o tsv +``` +``` +name city +Alice London +Bob Leeds +Carol London +Dan York +``` + +Global options go before or after the command's name: + +```shell +relix -C reports catalog ls -o tsv --no-header +``` +``` +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/reports/.relix/catalog/10-sample.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +## Environment + +| Variable | Effect | +|---|---| +| `RELIX_OUTPUT` | the output format when `-o` is not given | +| `RELIX_PROFILE` | the profile when `-P` is not given | +| `RELIX_NOW` | the instant `NOW()` returns when `--now` is not given | +| `RELIX_CATALOG_PATH` | catalog files to load, separated as in `PATH`, before the `--catalog` files | +| `RELIX_TRUST_ALL` | `1` or `true` trusts every project's `.relix/` directory | +| `RELIX_DRIVERS` | the directory JDBC drivers are installed in and loaded from, instead of `~/.relix/drivers` | +| `NO_COLOR` | when set, a table on a terminal is not coloured unless `--color=always` | +| `PAGER` | the program [`relix doc`](doc.md) shows a page through on a terminal; `cat` or empty shows it directly | + +A `${NAME}` placeholder in a declaration may also be resolved from the environment; see +[Secrets and profiles](../secrets.md). + +## Files + +| Path | What it holds | +|---|---| +| `~/.relix/catalog/*.relix` | your own catalog, which every directory under your home sees | +| `~/.relix/profiles.json` | your own profiles | +| `~/.relix/trusted` | the projects you have trusted, with a hash of each one's files | +| `~/.relix/drivers/` | installed JDBC drivers | +| `~/.relix/connectors/` | installed connector plugins | +| `~/.relix/cache/files/` | files fetched over `https` by a file connection | +| `.relix/catalog/*.relix`, `.relix/relixrc`, `.relix/profiles.json`, `.relix/root` | a project's catalog, defaults, profiles, and the marker that ends the walk up; see [The catalog](../catalog.md) | diff --git a/docs/guide/command-line/commands/run.md b/docs/guide/command-line/commands/run.md new file mode 100644 index 0000000..5a62bc1 --- /dev/null +++ b/docs/guide/command-line/commands/run.md @@ -0,0 +1,78 @@ +# relix run + +Runs scripts and writes their queries' rows to standard output. + +## Usage + +```text +relix run [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | +| `--line-buffered` | Flush each row as it is written, for a reader that wants it at once. | +| `--query=NAME` | Print only the query NAME. A machine format (tsv, csv, ndjson, json) prints one query: the script's last, unless this names another. | +| `--all` | With ndjson, print every query, each row tagged "_query": NAME. | +| `--fail-empty` | Exit 1 when the queries printed return no rows. | +| `--fail-rows` | Exit 1 when the queries printed return any row. | +| `--trace=FILE` | Write the event feed (what was rewritten, planned and run) to standard error, or to FILE with --trace=FILE, while the rows go to standard output. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +`relix run` is what `relix` does with no command, so these two are the same: + +```shell +relix run -e 'GROUP status, SUM(amount) -> total (Orders)' +``` +``` +── ────────────────────────────────────────────────────────────── + status total + ───────── ───── + completed 400 + pending 50 +(2 rows) +``` + +```shell +relix -e 'GROUP status, SUM(amount) -> total (Orders)' -o csv +``` +``` +status,total +completed,400 +pending,50 +``` + +Each script file is a run of its own, with its own declarations: + +```shell +relix run scripts/top-customers.relix -a city=Leeds scripts/by-city.relix -o tsv +``` +``` +name total +Alice 200 +Carol 200 +name +Bob +``` + +[Binding inputs](../inputs.md), [Output formats](../output.md) and +[Pipelines](../pipelines.md) cover the options in use. diff --git a/docs/guide/command-line/commands/trace.md b/docs/guide/command-line/commands/trace.md new file mode 100644 index 0000000..ab01e12 --- /dev/null +++ b/docs/guide/command-line/commands/trace.md @@ -0,0 +1,53 @@ +# relix trace + +Runs each query and prints its event feed, what the engine rewrote, planned and ran, discarding the rows. relix run --trace keeps the rows too. + +## Usage + +```text +relix trace [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--query=NAME` | Trace only the query NAME. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +Each event ends with how long it took, which varies from run to run, so here `cut` +leaves the timings off: + +```shell +relix trace -e 'SELECT amount > 100 (Orders)' | cut -d'(' -f1 +``` +``` +[EXECUTE ] orders SCAN scanned 4 rows +[EXECUTE ] ROWS query delivered 2 rows +``` + +`relix run --trace` writes the same feed to standard error while the rows go to standard +output, and `--trace=FILE` writes it to a file: + +```shell +relix --trace=trace.txt -e 'SELECT amount > 100 (Orders)' -o tsv +``` +``` +order_id customer_id amount status placed +1 1 120 completed 2026-09-02 +4 3 200 completed 2026-10-01 +``` diff --git a/docs/guide/command-line/commands/version.md b/docs/guide/command-line/commands/version.md new file mode 100644 index 0000000..0419b2e --- /dev/null +++ b/docs/guide/command-line/commands/version.md @@ -0,0 +1,23 @@ +# relix version + +Prints the versions of relix and of the Relix engine it runs. + +## Usage + +```text +relix version [OPTIONS] +``` + +It also takes the [global options](relix.md#global-options). + +## Examples + +`relix version`, like `relix --version`, prints one line: the version of the command, +then the version of the Relix engine it runs, as + +```text +relix 1.0.0 (engine 1.0.0) +``` + +The two are numbered independently: a release of the command names the engine release +it was built on. diff --git a/docs/guide/command-line/getting-started.md b/docs/guide/command-line/getting-started.md new file mode 100644 index 0000000..f5b036d --- /dev/null +++ b/docs/guide/command-line/getting-started.md @@ -0,0 +1,218 @@ +# The relix command + +`relix` runs Relix scripts from a shell. It is built to be one stage of a Unix pipeline: +the script comes from the command line, a file or standard input; data comes from files, +standard input or the tables a project declares; and the rows go to standard output in a +format the next program reads. Everything else — errors, warnings, progress — goes to +standard error, so a pipeline never mistakes a message for a row. + +This page installs the command, runs a first query, and shows the shape every command +shares. The pages after it take one subject each, and the +[command reference](commands/relix.md) lists every command and option. + +## Installing + +`relix` ships as a self-contained archive for macOS, Linux and Windows: a launcher, its +own Java runtime, the man pages and completion scripts for bash, zsh, fish and +PowerShell. Nothing else needs to be installed, and no Java is needed on the machine. + +On macOS and Linux, with Homebrew: + +```text +brew install DarkCollective/relix/relix +``` + +On Windows, with Scoop: + +```text +scoop bucket add relix https://github.com/DarkCollective/scoop-relix +scoop install relix +``` + +Or download the archive for your platform from the +[releases page](https://github.com/DarkCollective/relix-cli/releases), unpack it, and put +its `bin` directory on your `PATH`. Each release lists the archives' SHA-256 checksums +in `checksums.txt`. + +`relix version` prints the version of the command and the version of the Relix engine it +runs, as `relix VERSION (engine VERSION)`. The two are numbered independently. + +## The examples on these pages + +Every example in this section is run on each build of the command, and the block under +it is what it printed. The examples run in `~/shop`, a small project with a customer +file, an order file and a stock database: + +```shell +cat data/customers.csv data/orders.csv +``` +``` +customer_id,name,city +1,Alice,London +2,Bob,Leeds +3,Carol,London +4,Dan,York +order_id,customer_id,amount,status,placed +1,1,120,completed,2026-09-02 +2,1,80,completed,2026-09-14 +3,2,50,pending,2026-09-20 +4,3,200,completed,2026-10-01 +``` + +The project declares its tables once, in `~/shop/.relix/catalog/`, so the examples can +name `Customers` and `Orders` without declaring them. [The catalog](catalog.md) explains +how. A path printed under your home directory is shown as `~`. + +## A first query + +`-e` takes Relix text. A bare expression is run as a query: + +```shell +relix -e 'SELECT amount > 100 (Orders)' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id customer_id amount status placed + ──────── ─────────── ────── ───────── ────────── + 1 1 120 completed 2026-09-02 + 4 3 200 completed 2026-10-01 +(2 rows) +``` + +On a terminal the rows are a table. In a pipe they are tab-separated values, one row per +line under a header, which is what `cut`, `sort`, `awk` and `column` expect: + +```shell +relix -e 'SELECT amount > 100 (Orders)' | cut -f1,3 +``` +``` +order_id amount +1 120 +4 200 +``` + +The language itself — every operator, predicate and function — is the subject of the +[language reference](../../reference/README.md). Operators have a Unicode glyph and an ASCII +keyword, and the command reads both: `σ amount > 100 (Orders)` is the same query. + +## Where the script comes from + +A command reads its script from the first of these that is given: + +1. **`-e TEXT`**, repeatable. The texts are joined, in order, into one script, so one + `-e` can declare a view and the next use it. A text that is a bare relational + expression rather than statements is run as `query { … };`. +2. **Script files**, named as arguments. Each runs in a session of its own, so two files on + one command line never see each other's declarations. `-` names standard input. +3. **Standard input**, when nothing is named and standard input is not a terminal. + +With none of them, the command prints its usage and exits 2. + +```shell +relix -e 'Big := { SELECT amount > 100 (Orders) };' -e 'query { PROJECT order_id (Big) };' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id + ──────── + 1 + 4 +(2 rows) +``` + +A script file holds statements, as a script always does: + +```shell +cat scripts/top-customers.relix +``` +``` +-- The customers who have spent the most, best first. +query { SORT total DESC (PROJECT name, total (Customers JOIN Spend)) }; +``` + +```shell +relix scripts/top-customers.relix +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 200 + Carol 200 +(2 rows) +``` + +and a script can arrive on standard input: + +```shell +echo 'query { PROJECT name (Customers) };' | relix -o csv +``` +``` +name +Alice +Bob +Carol +Dan +``` + +A relative path in a script — a `csv("…")` source, an `import` — resolves against the +script file's own directory, and in `-e` text or standard input against the working +directory. `-C DIR` makes the command act as if it were started in `DIR`. + +## Commands + +`relix` with no command name runs scripts; it is `relix run`. Every other job is a command +of its own, and each writes one thing to standard output: + +| Command | Standard output | +|---|---| +| [`relix run`](commands/run.md) | the rows of the queries (the default) | +| [`relix check`](commands/check.md) | nothing; problems go to standard error, or `--json` to standard output | +| [`relix explain`](commands/explain.md) | the physical plan of each query | +| [`relix optimize`](commands/optimize.md) | the script, rewritten by the optimiser | +| [`relix trace`](commands/trace.md) | what the engine rewrote, planned and ran | +| [`relix bundle`](commands/bundle.md) | one JSON document per script: diagnostics, plans and events | +| [`relix ir`](commands/ir.md) | the symbols, views and queries the analyser made of the script | +| [`relix provenance`](commands/provenance.md) | the rows, each with where it came from | +| [`relix fmt`](commands/fmt.md) | the script, printed canonically | +| [`relix catalog`](commands/catalog.md) | the catalog's files, relations, columns and trust | +| [`relix drivers`](commands/drivers.md), [`relix connectors`](commands/connectors.md) | the JDBC drivers and connector plugins, and installing them | +| [`relix doc`](commands/doc.md) | a page of the language reference | +| [`relix completion`](commands/completion.md) | a completion script for a shell | +| [`relix help`](commands/help.md), [`relix version`](commands/version.md) | usage, and the versions | + +Options follow the usual conventions: short options cluster (`-qN`), a long option takes +its value as `--output=csv` or `--output csv`, `--` ends the options, and `-` is standard +input. The [global options](commands/relix.md#global-options) — the profile, the limits, +`-C` and the rest — are accepted by every command, before or after its name. + +## When something goes wrong + +A problem in a script is reported on standard error as `FILE:LINE:COLUMN: severity: +message`, the shape compilers use, so an editor or a CI annotator can jump to it: + +```shell +relix check scripts/broken.relix 2>&1; echo "exit $?" +``` +``` +scripts/broken.relix:1:30: error: Undefined relation: 'Ordrs' +exit 3 +``` + +The exit status says what kind of failure it was, so a script can tell a typo from a +database that is down: + +| Status | Meaning | +|---|---| +| 0 | success | +| 1 | an assertion asked for failed: `--fail-empty` or `--fail-rows` | +| 2 | a usage error: a bad option, nothing to run | +| 3 | the script did not parse or analyse | +| 4 | the query failed while it ran: a data error, a limit hit | +| 5 | the environment: a missing driver, an untrusted catalog, a connection refused | +| 70 | an internal error in `relix`; the stack trace goes to standard error | +| 130 | interrupted with Ctrl-C (`SIGINT`) | +| 141 | standard output was closed by the reader (`SIGPIPE`) | + +`-q` keeps only the errors, `-v` adds notices such as the catalog files and drivers +loaded, and `-vv` adds timings. diff --git a/docs/guide/command-line/index.json b/docs/guide/command-line/index.json new file mode 100644 index 0000000..e2921a8 --- /dev/null +++ b/docs/guide/command-line/index.json @@ -0,0 +1,28 @@ +{ + "title": "The relix command line", + "pages": [ + {"path": "getting-started.md", "chapter": "Using the command", "title": "The relix command", "summary": "Installing; a first query; where the script comes from; the commands; diagnostics and exit status"}, + {"path": "inputs.md", "chapter": "Using the command", "title": "Binding inputs", "summary": "`-i` and its formats; standard input; inferred and given headings; inputs that never end; shadowing the catalog; query parameters with `-a`"}, + {"path": "output.md", "chapter": "Using the command", "title": "Output formats", "summary": "`table`, `tsv`, `csv`, `ndjson`, `json` and `markdown`; NULLs and headers; scripts with several queries; `--fail-empty` and `--fail-rows`; messages; closed pipes and interrupts"}, + {"path": "catalog.md", "chapter": "A project's context", "title": "The catalog", "summary": "`.relix/` directories and how they are found; trusting a project; a nearer declaration wins; inspecting the catalog; `--catalog`; `relixrc`"}, + {"path": "secrets.md", "chapter": "A project's context", "title": "Secrets and profiles", "summary": "`${NAME}` placeholders; `profiles.json` and `-P`; the environment and `-D`; private profile files; what is never printed"}, + {"path": "pipelines.md", "chapter": "Recipes", "title": "Pipelines", "summary": "With `jq`, `awk`, `sort` and `head`; with `find` and `xargs`; scripts in here-documents; a data check in CI; optimising a script and reviewing the change"}, + {"path": "commands/relix.md", "chapter": "Command reference", "title": "relix", "summary": "The global options every command takes; the exit statuses; the environment variables and files", "command": "relix"}, + {"path": "commands/run.md", "chapter": "Command reference", "title": "relix run", "summary": "Runs scripts and writes their queries' rows", "command": "relix run"}, + {"path": "commands/check.md", "chapter": "Command reference", "title": "relix check", "summary": "Analyses scripts without running them, and reports their problems", "command": "relix check"}, + {"path": "commands/explain.md", "chapter": "Command reference", "title": "relix explain", "summary": "Prints each query's physical plan", "command": "relix explain"}, + {"path": "commands/optimize.md", "chapter": "Command reference", "title": "relix optimize", "summary": "Prints the script with each query rewritten by the optimiser", "command": "relix optimize"}, + {"path": "commands/trace.md", "chapter": "Command reference", "title": "relix trace", "summary": "Runs each query and prints what the engine rewrote, planned and ran", "command": "relix trace"}, + {"path": "commands/bundle.md", "chapter": "Command reference", "title": "relix bundle", "summary": "Prints each script's JSON bundle of diagnostics, plans and events", "command": "relix bundle"}, + {"path": "commands/ir.md", "chapter": "Command reference", "title": "relix ir", "summary": "Prints the symbols, views and queries the analyser made of a script", "command": "relix ir"}, + {"path": "commands/provenance.md", "chapter": "Command reference", "title": "relix provenance", "summary": "Runs each query and annotates each row with where it came from", "command": "relix provenance"}, + {"path": "commands/fmt.md", "chapter": "Command reference", "title": "relix fmt", "summary": "Prints scripts canonically, keeping their comments", "command": "relix fmt"}, + {"path": "commands/catalog.md", "chapter": "Command reference", "title": "relix catalog", "summary": "Lists the catalog's files, relations, columns and declarations; trusts a project", "command": "relix catalog", "subcommands": [{"command": "relix catalog files", "anchor": "relix-catalog-files"}, {"command": "relix catalog ls", "anchor": "relix-catalog-ls"}, {"command": "relix catalog schema", "anchor": "relix-catalog-schema"}, {"command": "relix catalog where", "anchor": "relix-catalog-where"}, {"command": "relix catalog trust", "anchor": "relix-catalog-trust"}]}, + {"path": "commands/drivers.md", "chapter": "Command reference", "title": "relix drivers", "summary": "Lists and installs JDBC drivers", "command": "relix drivers", "subcommands": [{"command": "relix drivers ls", "anchor": "relix-drivers-ls"}, {"command": "relix drivers install", "anchor": "relix-drivers-install"}]}, + {"path": "commands/connectors.md", "chapter": "Command reference", "title": "relix connectors", "summary": "Lists and installs connector plugins", "command": "relix connectors", "subcommands": [{"command": "relix connectors ls", "anchor": "relix-connectors-ls"}, {"command": "relix connectors install", "anchor": "relix-connectors-install"}]}, + {"path": "commands/doc.md", "chapter": "Command reference", "title": "relix doc", "summary": "Shows a page of the language reference", "command": "relix doc"}, + {"path": "commands/help.md", "chapter": "Command reference", "title": "relix help", "summary": "Prints a command's usage", "command": "relix help"}, + {"path": "commands/version.md", "chapter": "Command reference", "title": "relix version", "summary": "Prints the versions of the command and of the engine", "command": "relix version"}, + {"path": "commands/completion.md", "chapter": "Command reference", "title": "relix completion", "summary": "Prints a completion script for bash, zsh, fish or PowerShell", "command": "relix completion"} + ] +} diff --git a/docs/guide/command-line/inputs.md b/docs/guide/command-line/inputs.md new file mode 100644 index 0000000..6f0dc12 --- /dev/null +++ b/docs/guide/command-line/inputs.md @@ -0,0 +1,291 @@ +# Binding inputs + +A script names the relations it reads. Most of the time those are declared somewhere — in +the script, or in the project's [catalog](catalog.md) — but a pipeline often has data that +exists only for this run: a file another program wrote, the output of `jq`, a log being +followed. `-i` binds a relation name to such data for one run, without a declaration. +`-a` does the same for a single value, a query parameter. + +## `-i NAME=[FORMAT:]LOCATION` + +`-i` binds the relation `NAME` to the data at `LOCATION`, which is a file, `-` for +standard input, or an `http(s)` URL. It is repeatable, one relation per `-i`: + +```shell +cat data/returns.ndjson +``` +``` +{"order_id": 2, "reason": "damaged", "refund": {"amount": 80, "currency": "GBP"}} +{"order_id": 4, "reason": "late", "refund": {"amount": 20, "currency": "GBP"}} +``` + +```shell +relix -i Returns=data/returns.ndjson -e 'PROJECT order_id, reason (Returns)' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id reason + ──────── ─────── + 2 damaged + 4 late +(2 rows) +``` + +A bound relation is used like any other, and joins with the catalog's tables: + +```shell +relix -i Returns=data/returns.ndjson -e 'PROJECT order_id, amount, reason (Orders JOIN Returns)' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id amount reason + ──────── ────── ─────── + 2 80 damaged + 4 200 late +(2 rows) +``` + +A relative path resolves against the working directory, or against `-C DIR`. The same name +may not be bound twice. + +### Formats + +`FORMAT` is one of: + +| Format | What it reads | +|---|---| +| `csv` | comma-separated values, RFC 4180, with a header row | +| `tsv` | tab-separated values, with a header row | +| `json` | one JSON array of objects | +| `ndjson` | one JSON object per line, the format `jq -c` writes | + +Without a format, the file's extension decides: `.csv`, `.tsv` or `.tab`, `.json`, and +`.ndjson` or `.jsonl`. A file with any other extension needs its format named: + +```shell +cp data/orders.csv orders.txt +relix -i O=orders.txt -e 'O' 2>&1; echo "exit $?" +``` +``` +relix: -i O=orders.txt: cannot tell the format from the extension .txt; name it, as NAME=csv:LOCATION (csv, tsv, json or ndjson) +exit 2 +``` + +```shell +relix -i O=csv:orders.txt -e 'GROUP status, COUNT(*) -> n (O)' +``` +``` +── ────────────────────────────────────────────────────────────── + status n + ───────── ─ + completed 3 + pending 1 +(2 rows) +``` + +A nested JSON object becomes a nested value, which the language's dotted paths and +`UNNEST` take apart; [Nested data](../nested.md) covers them. + +### Standard input + +`-` binds standard input. Standard input has no extension, so it always names its format: + +```shell +jq -c 'select(.refund.amount > 50)' data/returns.ndjson \ + | relix -i Returns=ndjson:- -e 'PROJECT name, order_id, reason (Customers JOIN Orders JOIN Returns)' +``` +``` +── ────────────────────────────────────────────────────────────── + name order_id reason + ───── ──────── ─────── + Alice 2 damaged +(1 row) +``` + +Only one input can be standard input, and when one is, the script cannot be read from it +too: it comes from `-e` or a file. + +```shell +relix -i A=csv:- -i B=csv:- -e 'A' 2>&1 < data/orders.csv; echo "exit $?" +``` +``` +relix: standard input (-) can be bound to one input only +exit 2 +``` + +### The heading of an input + +The columns of a bound input and their types are inferred. For `csv` and `tsv` the names +are the header row's; for `json` and `ndjson` they are the keys of the objects. The types +are inferred from the values in the first 1000 records, or in as many as `--infer-rows=N` +says, so `order_id` above is a number and `placed` a date. + +`--schema NAME='{ … }'` gives the heading instead, in the syntax a `source` declaration's +`schema` uses. Here the `amount` column is read as a string rather than a number: + +```shell +relix -i O=data/orders.csv --schema O='{ order_id: NUMBER, amount: STRING }' -e 'PROJECT order_id, amount (O)' -o json +``` +``` +[ + {"order_id":1,"amount":"120"}, + {"order_id":2,"amount":"80"}, + {"order_id":3,"amount":"50"}, + {"order_id":4,"amount":"200"} +] +``` + +`--schema` and `--unbounded` name an input bound with `-i`; naming any other is a usage +error. + +### Inputs that never end + +An input such as `tail -f`'s never ends. `--unbounded=NAME` declares that, and two things +follow. The rows stream: each is filtered, joined against bounded tables and written as +it arrives, rather than after the input ends. And a query that would have to read all of +the input before writing anything — a `GROUP`, a `SORT`, a `DISTINCT` — is refused before +it starts, rather than waiting forever: + +```shell +printf '{"level":"error","msg":"disk"}\n{"level":"info","msg":"ok"}\n' | relix -i Log=ndjson:- --unbounded=Log -e "SELECT level = 'error' (Log)" -o ndjson +``` +``` +{"level":"error","msg":"disk"} +``` + +```shell +printf '{"level":"error"}\n' | relix -i Log=ndjson:- --unbounded=Log -e 'GROUP level, COUNT(*) -> n (Log)' -o ndjson 2>&1; echo "exit $?" +``` +``` +relix: -e: cannot materialise unbounded relation for blocking operator γ (GROUP); add a bound (e.g. λ n) below it +exit 3 +``` + +A `table` needs every row before it can align the columns, so an unbounded input is +written in a format that streams — `tsv`, `csv`, `ndjson` or `json` — and on a terminal +that means naming one with `-o`. A `json` input is one array, read whole, so it cannot be +unbounded. Write one object per +line and bind it as `ndjson`. With `--line-buffered` each row is written out as soon as it +is produced, for a reader that wants it at once; see [Output formats](output.md). + +### Files on the web + +A location that is an `http://` or `https://` URL is fetched, and only with `--remote`, +which a run must give for anything to be fetched. Without it, binding a URL is a usage +error. The format comes from the URL's path, or is named. + +### Shadowing the catalog + +A bound input replaces a relation of the same name that the catalog declares, for this +run. Every view that reads the name reads the input instead, so a view written against the +real `Orders` can be run over a sample file without changing it: + +```shell +relix -e 'BigSpenders' +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 200 + Carol 200 +(2 rows) +``` + +```shell +relix -i Orders=reports/sample-orders.csv -e 'BigSpenders' +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 500 + Dan 250 +(2 rows) +``` + +## `-a NAME=VALUE`: query parameters + +A script can leave a value to be supplied when it runs, as a parameter: `$city` in + +```shell +cat scripts/by-city.relix +``` +``` +-- The customers of one city, named when the script runs: relix -a city=London +query { PROJECT name (SELECT city = $city (Customers)) }; +``` + +`-a city=London` binds it (the long form is `--arg`): + +```shell +relix -a city=London scripts/by-city.relix +``` +``` +── ────────────────────────────────────────────────────────────── + name + ───── + Alice + Carol +(2 rows) +``` + +```shell +relix --arg city=Leeds scripts/by-city.relix -o tsv +``` +``` +name +Bob +``` + +The value is bound to the parameter, never pasted into the script's text, so whatever it +holds — quotes, a semicolon, a fragment of Relix — it is only ever a value, and cannot +change what the query does: + +```shell +relix -a "city=London' OR 1 = 1 --" scripts/by-city.relix -o tsv +``` +``` +name +``` + +A parameter takes the type of what it is compared with, and the text `-a` gives is read +as that type: a number, a date, `true` or `false`. Text that is not of the type is an +error naming the parameter: + +```shell +relix -a since=2026-09-15 -e 'PROJECT order_id, placed (SELECT placed >= $since (Orders))' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id placed + ──────── ────────── + 3 2026-09-20 + 4 2026-10-01 +(2 rows) +``` + +```shell +relix -a since=yesterday -e 'SELECT placed >= $since (Orders)' 2>&1; echo "exit $?" +``` +``` +relix: -e: parameter $since is compared with a DATE, so it cannot be bound to the text 'yesterday' +exit 3 +``` + +A query cannot run until every parameter it reaches has a value: + +```shell +relix scripts/by-city.relix 2>&1; echo "exit $?" +``` +``` +relix: scripts/by-city.relix: parameter $city is not bound; give it a value with -a city=VALUE +exit 3 +``` + +The [parameter reference page](../../reference/language/parameter.md) has the rules in +full. + +`-a` binds a value a query compares with. A value a *declaration* needs — a database's +address, a password, a token — is a placeholder, `${NAME}`, given by a profile, the +environment or `-D`, and the subject of [Secrets and profiles](secrets.md). diff --git a/docs/guide/command-line/output.md b/docs/guide/command-line/output.md new file mode 100644 index 0000000..580ac4e --- /dev/null +++ b/docs/guide/command-line/output.md @@ -0,0 +1,269 @@ +# Output formats + +Rows go to standard output and nothing else does. Errors, warnings, notices and +progress go to standard error, so the rows can be piped, redirected or captured without +anything to strip out of them. This page covers the formats rows are written in, which +queries of a script are written, how a run can assert something about its rows, and how +a run ends when its reader stops reading. + +## Choosing a format + +`-o FORMAT` (`--output=FORMAT`) chooses one of: + +| Format | Shape | Streams | +|---|---|---| +| `table` | aligned columns under a rule, with a row count | no | +| `tsv` | tab-separated values under a header row | yes | +| `csv` | comma-separated values under a header row, RFC 4180 | yes | +| `ndjson` | one JSON object per row, one per line | yes | +| `json` | one JSON array of objects | yes | +| `markdown` | a Markdown table | no | + +Without `-o`, the format is `$RELIX_OUTPUT`, else the `output` a project's +[`relixrc`](catalog.md#defaults-relixrc) sets, else `table` when standard output is a +terminal and `tsv` when it is not. So the same command prints a table to a person and +tab-separated values to a pipe. + +```shell +relix -e 'Customers' -o csv +``` +``` +customer_id,name,city +1,Alice,London +2,Bob,Leeds +3,Carol,London +4,Dan,York +``` + +```shell +relix -e 'Customers' -o ndjson +``` +``` +{"customer_id":1,"name":"Alice","city":"London"} +{"customer_id":2,"name":"Bob","city":"Leeds"} +{"customer_id":3,"name":"Carol","city":"London"} +{"customer_id":4,"name":"Dan","city":"York"} +``` + +```shell +relix -e 'PROJECT name, city (Customers)' -o json +``` +``` +[ + {"name":"Alice","city":"London"}, + {"name":"Bob","city":"Leeds"}, + {"name":"Carol","city":"London"}, + {"name":"Dan","city":"York"} +] +``` + +```shell +relix -e 'PROJECT name, city (Customers)' -o markdown +``` +``` +### + +| name | city | +| --- | --- | +| Alice | London | +| Bob | Leeds | +| Carol | London | +| Dan | York | +``` + +```shell +RELIX_OUTPUT=tsv relix -e 'PROJECT name (Customers)' +``` +``` +name +Alice +Bob +Carol +Dan +``` + +`tsv` writes a tab, a newline, a carriage return or a backslash inside a value as `\t`, +`\n`, `\r` or `\\`, so that every line is one row and every tab separates two columns; +this is the form `cut`, `awk` and `sort -t` read. `csv` quotes a value holding a comma, a +quote or a line break instead. `json` and `ndjson` write numbers as JSON numbers, booleans +as `true` and `false`, NULL as `null`, and dates, times and durations as ISO-8601 +strings. + +A format that streams writes each row as it is produced, so a result larger than memory +goes straight through and the first rows arrive before the last are computed. `table` and +`markdown` align their columns, which needs every row first. In a table a value longer +than 30 characters is cut short, ending `…`; the machine formats write every value whole. + +### Headers, NULLs and colour + +| Option | Effect | +|---|---| +| `--no-header` | leave out the header row of `csv` and `tsv` | +| `--null=STRING` | write NULL as `STRING`; by default it is empty in `csv`, `tsv` and `markdown`, and `NULL` in a table | +| `--color=WHEN` | colour a table `auto`, `always` or `never`; `auto` colours on a terminal unless `$NO_COLOR` is set | + +```shell +relix -e 'PROJECT name, order_id (Customers LJOIN Customers.customer_id = Orders.customer_id Orders)' -o csv --null=- +``` +``` +name,order_id +Alice,1 +Alice,2 +Bob,3 +Carol,4 +Dan,- +``` + +```shell +relix -e 'PROJECT name (Customers)' -o tsv --no-header +``` +``` +Alice +Bob +Carol +Dan +``` + +## Scripts with several queries + +A script may hold several `query` statements. A table prints each of them, under its +name: the view's name for `query Name;`, and ``, its place among the +script's queries, for `query { … };`. + +```shell +relix -e 'query Completed;' -e 'query BigSpenders;' +``` +``` +── Completed ─────────────────────────────────────────────────────────────────── + order_id customer_id amount status placed + ──────── ─────────── ────── ───────── ────────── + 1 1 120 completed 2026-09-02 + 2 1 80 completed 2026-09-14 + 4 3 200 completed 2026-10-01 +(3 rows) +── BigSpenders ───────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 200 + Carol 200 +(2 rows) +``` + +A machine format holds one result, since a CSV file with two headers is not a CSV file. It +prints the script's **last** query, as a shell function returns its last command's status, +or the one `--query=NAME` names: + +```shell +relix -e 'query Completed;' -e 'query BigSpenders;' -o tsv +``` +``` +name total +Alice 200 +Carol 200 +``` + +```shell +relix -e 'query Completed;' -e 'query BigSpenders;' -o tsv --query=Completed +``` +``` +order_id customer_id amount status placed +1 1 120 completed 2026-09-02 +2 1 80 completed 2026-09-14 +4 3 200 completed 2026-10-01 +``` + +`--all` prints every query in `ndjson`, each row tagged with the query it belongs to in a +`"_query"` field, so one stream can carry several results and `jq` can take them apart: + +```shell +relix -e 'query Spend;' -e 'query BigSpenders;' -o ndjson --all +``` +``` +{"_query":"Spend","customer_id":1,"total":200} +{"_query":"Spend","customer_id":3,"total":200} +{"_query":"BigSpenders","name":"Alice","total":200} +{"_query":"BigSpenders","name":"Carol","total":200} +``` + +With any other machine format `--all` is a usage error. + +Several script files on one command line are independent runs, and their output is +written one after another in the order the files were named. + +## Asserting something about the rows + +`--fail-empty` makes a run exit 1 when the queries it printed returned no rows, and +`--fail-rows` when they returned any. The rows are printed either way. This is the +`grep -q` idiom, for a data check in CI or in a shell `if`: + +```shell +relix --fail-rows -e 'SELECT amount < 0 (Orders)' -o tsv; echo "exit $?" +``` +``` +order_id customer_id amount status placed +exit 0 +``` + +```shell +relix --fail-rows -e "SELECT status = 'pending' (Orders)" -o tsv > /dev/null || echo "orders are still pending" +``` +``` +orders are still pending +``` + +[Pipelines](pipelines.md#a-data-check-in-ci) shows one in a CI job. + +## Where messages go + +A problem in a script is reported on standard error as `FILE:LINE:COLUMN: severity: +message`. `FILE` is the script's path, `-e` (or `-e#2` for the second of several `-e` +texts), or ``; `LINE` and `COLUMN` count within that text, so a message about an +`-e` points into what was typed. A problem with the run itself — an option, a file that +is not there, a database that refuses the connection — is one line beginning `relix:`. + +```shell +relix -e 'SELECT amount > 100 (Ordrs)' 2>&1; echo "exit $?" +``` +``` +-e:1:22: error: Undefined relation: 'Ordrs' +exit 3 +``` + +Warnings are printed too, unless `-q` is given. `-v` adds notices — each catalog file +read, each JDBC driver loaded — and `-vv` adds how long each stage took. `relix check +--json` writes the diagnostics to standard output as one JSON document instead, for a +tool to read; see [`relix check`](commands/check.md). + +The [exit status](getting-started.md#when-something-goes-wrong) says which kind of +failure it was. + +## When the reader stops reading + +A program at the other end of a pipe can stop reading at any time: `head` does after its +last line. When it does, `relix` stops: the query in flight is cancelled — a plan stops +pulling rows, a database cursor is closed — and the command exits 141, the status a shell +gives a program ended by `SIGPIPE`, without printing anything. So a query over a large or +endless relation can be cut short with `head`: + +```shell +relix -e 'Orders' -o tsv | head -3 +``` +``` +order_id customer_id amount status placed +1 1 120 completed 2026-09-02 +2 1 80 completed 2026-09-14 +``` + +Ctrl-C (`SIGINT`) likewise cancels the query and closes the session, and the command exits +130. + +Standard output is written in blocks and flushed at the end of each result. With +`--line-buffered` each row is flushed as it is written, for a reader that wants each row +at once — an input following a log, read by something that reacts to each line. + +## Following what the engine did + +`--trace` writes the run's event feed — what the optimiser rewrote, the plan it chose, +how many rows each step produced — to standard error while the rows go to standard +output; `--trace=FILE` writes it to `FILE` instead. [`relix trace`](commands/trace.md) +prints the same feed on standard output, without the rows. diff --git a/docs/guide/command-line/pipelines.md b/docs/guide/command-line/pipelines.md new file mode 100644 index 0000000..8f4fe90 --- /dev/null +++ b/docs/guide/command-line/pipelines.md @@ -0,0 +1,192 @@ +# Pipelines + +`relix` reads text and writes text, so it works with the programs a shell already has. +This page is a set of recipes: `relix` with `jq` for JSON, with `find` and `xargs` for +many files, with `awk`, `sort` and `head` for its rows, and as a check in CI. + +The rule that makes all of them work: **standard output carries rows and nothing else.** +Messages go to standard error, the format in a pipe is tab-separated values unless `-o` +says otherwise, and the [exit status](getting-started.md#when-something-goes-wrong) says +what happened. + +## With jq + +`jq` is the tool for shaping JSON, and `relix` the one for joining and aggregating it. A +pipeline passes JSON between them as NDJSON, one object per line, which `jq -c` writes, +`-i NAME=ndjson:-` reads and `-o ndjson` writes. + +Here `jq` flattens the returns file, `relix` joins it to the customers and orders, and +`jq` formats the result: + +```shell +jq -c '{order_id, reason, refund: .refund.amount}' data/returns.ndjson \ + | relix -i Returns=ndjson:- -e 'PROJECT name, order_id, refund (Customers JOIN Orders JOIN Returns)' -o ndjson \ + | jq -r '"\(.name) was refunded \(.refund) on order \(.order_id)"' +``` +``` +Alice was refunded 80 on order 2 +Carol was refunded 20 on order 4 +``` + +`relix` can also hand `jq` a single JSON document, an array, with `-o json`: + +```shell +relix -e 'GROUP city, COUNT(*) -> customers (Customers)' -o json | jq -c 'map({(.city): .customers}) | add' +``` +``` +{"London":2,"Leeds":1,"York":1} +``` + +## With awk, sort and cut + +Tab-separated values are what the classic text tools split on. The header is the first +line; `--no-header` leaves it out. + +```shell +relix -e 'Orders' --no-header | awk -F'\t' '{ total[$4] += $3 } END { for (s in total) print s, total[s] }' | sort +``` +``` +completed 400 +pending 50 +``` + +## Stopping early + +When the reader of a pipe stops, `relix` stops too: it cancels the query rather than +computing rows nobody will read. `head` can cut a large result short: + +```shell +relix -e 'SORT amount DESC (Orders)' | head -3 +``` +``` +order_id customer_id amount status placed +4 3 200 completed 2026-10-01 +1 1 120 completed 2026-09-02 +``` + +## With find and xargs + +### Lint every script + +`relix check` analyses scripts without running them, reporting each problem as +`FILE:LINE:COLUMN: severity: message` and exiting 3 if any script has one. Over every +script in a tree: + +```shell +find scripts -name '*.relix' | sort | xargs relix check 2>&1 || echo "a script has a problem" +``` +``` +scripts/broken.relix:1:30: error: Undefined relation: 'Ordrs' +a script has a problem +``` + +`xargs` fails when any run of the command it starts fails, so the pipeline's status says +whether every script passed. Use `find … -print0 | xargs -0` when a file name may hold a +space. + +### Run a query over each of many files + +`-i` binds one file per run, so `xargs -I{}` runs a query over each file in turn. Here +each CSV file's row count, with its name: + +```shell +find data reports -name '*.csv' | sort | xargs -I{} sh -c 'printf "%s\t" {}; relix -i T={} -e "GROUP COUNT(*) -> n (T)" -o tsv --no-header' +``` +``` +data/customers.csv 4 +data/orders.csv 4 +reports/sample-orders.csv 2 +``` + +### Check every script's formatting + +`relix fmt --check` prints the name of each file that is not formatted canonically, and +exits 1 if there is one; `relix fmt -w` rewrites them in place. The shop's scripts spell +operators as ASCII keywords, so they are checked against `--keywords`: + +```shell +find scripts -name '*.relix' | sort | xargs relix fmt --check --keywords || echo "a script needs formatting" +``` +``` +scripts/messy.relix +scripts/top-customers.relix +a script needs formatting +``` + +## Scripts written in the shell + +A script can be written inline with a here-document, which keeps a longer query readable +and still needs no file: + +```shell +relix -o csv <<'EOF' +Late := { SELECT placed >= DATE '2026-09-15' (Orders) }; +query { PROJECT name, order_id (Customers JOIN Late) }; +EOF +``` +``` +name,order_id +Bob,3 +Carol,4 +``` + +A value the shell knows goes in as a [parameter](inputs.md#a-name-value-query-parameters), +never pasted into the text: + +```shell +city=London +relix -a city="$city" scripts/by-city.relix -o tsv +``` +``` +name +Alice +Carol +``` + +## A data check in CI + +`--fail-rows` exits 1 when the query returns any row, and `--fail-empty` when it returns +none, so a data assertion is one line of a CI job: + +```shell +relix --fail-rows -e 'SELECT amount <= 0 (Orders)' -o tsv > /dev/null && echo "every order has a positive amount" +``` +``` +every order has a positive amount +``` + +```shell +relix --fail-rows -e "SELECT status = 'pending' (Orders)" -o tsv || echo "pending orders found" +``` +``` +order_id customer_id amount status placed +3 2 50 pending 2026-09-20 +pending orders found +``` + +A CI job runs in a fresh container where the project's `.relix/` cannot have been +trusted by hand; `RELIX_TRUST_ALL=1` trusts it, and a profile or the job's secret +environment variables supply the [placeholders](secrets.md). + +## Optimise a script, and review the change + +`relix optimize` prints a script with each query rewritten by the optimiser, so it works +as a filter, and `diff` shows what it changed: + +```shell +cat > slow.relix <<'EOF' +query { SELECT city = 'London' (Customers JOIN Orders) }; +EOF +relix optimize slow.relix | diff slow.relix -; echo "exit $?" +``` +``` +1c1 +< query { SELECT city = 'London' (Customers JOIN Orders) }; +--- +> query { (σ city = "London" (Customers)) ⋈ (Orders) }; +exit 1 +``` + +`diff` exits 1 when the files differ. To keep the rewritten script, redirect it: +`relix optimize slow.relix > fast.relix`. [`relix optimize --report`](commands/optimize.md) +lists the rules that fired instead. diff --git a/docs/guide/command-line/secrets.md b/docs/guide/command-line/secrets.md new file mode 100644 index 0000000..2e2d57c --- /dev/null +++ b/docs/guide/command-line/secrets.md @@ -0,0 +1,184 @@ +# Secrets and profiles + +A connection to a database needs an address and a password; a source that calls an API +needs a token. None of them belongs in a script or a catalog file that is committed and +shared. They are written as **placeholders**, `${NAME}`, and the command supplies the +values when a query runs: from `-D`, from a named **profile**, or from the environment. + +## Placeholders + +The shop's stock lives in a database, declared in the catalog with placeholders where +its address and credentials go: + +```shell +cat .relix/catalog/20-warehouse.relix +``` +``` +-- The stock database. Its address and credentials come from placeholders. +connection warehouse from database { + url: "${WAREHOUSE_URL}", + user: "${WAREHOUSE_USER}", + password: "${WAREHOUSE_PASSWORD}" +}; + +source Stock from warehouse { + table: "STOCK", + schema: { sku: STRING, on_hand: NUMBER } +}; +``` + +A placeholder may stand in any string value of a declaration: a URL, a file path, a +table name, a header, a credential. The engine's +[Secrets in declarations](../secrets.md) page has the rules. + +A placeholder is resolved each time a query runs, for the declarations that query +reaches, and only then. A query that never reaches the warehouse needs none of its +values, and one that does fails before anything is sent when a value is missing, naming +the placeholder; [below](#the-environment-and-d) is an example. + +Each `${NAME}` is looked up in three places, and the first that has it wins: + +1. **`-D NAME=VALUE`** on the command line. +2. **The selected profile**, from the `profiles.json` files. +3. **The environment** of the process. + +## Profiles + +A profile is a named set of values, kept in a `profiles.json` file in a `.relix/` +directory: one JSON object whose members are profiles, each an object of string values. +The shop's project keeps the parts of its `dev` profile that are not secret: + +```shell +cat .relix/profiles.json +``` +``` +{ + "dev": { + "WAREHOUSE_URL": "jdbc:h2:~/shop/data/warehouse", + "WAREHOUSE_USER": "shop" + } +} +``` + +and the password is in your own `~/.relix/profiles.json`, outside the project and its +repository: + +```shell +cat ~/.relix/profiles.json +``` +``` +{ + "dev": { "WAREHOUSE_PASSWORD": "s3cret" } +} +``` + +`-P NAME` (`--profile=NAME`) selects a profile. Its values are merged across every +`profiles.json` the run reads, outermost first, so a nearer file's value for a name +replaces a farther one's, and a profile may be split between your own file and the +project's, as `dev` is here: + +```shell +relix -P dev -e 'Stock' +``` +``` +── ────────────────────────────────────────────────────────────── + sku on_hand + ──── ─────── + mug 12 + tee 0 + tote 31 +(3 rows) +``` + +Without `-P`, the profile is `$RELIX_PROFILE`, else the `profile` a +[`relixrc`](catalog.md#defaults-relixrc) names. With none of them, no profile is +selected and placeholders come from `-D` and the environment alone. + +```shell +RELIX_PROFILE=dev relix -e 'PROJECT sku (SELECT on_hand = 0 (Stock))' -o tsv +``` +``` +sku +tee +``` + +Naming a profile no file defines is a usage error, which lists the ones that are defined: + +```shell +relix -P production -e 'Stock' 2>&1; echo "exit $?" +``` +``` +relix: no profile 'production' (defined: dev) +exit 2 +``` + +A project's `profiles.json` is part of its `.relix/` directory, so it is read only once +the project is [trusted](catalog.md#a-project-must-be-trusted); `~/.relix/profiles.json` +is always read. + +### A profile file must be private + +A profile may hold a password, so a `profiles.json` that its group or others can read is +refused, as `ssh` refuses a private key anyone can read. The error names the `chmod` that +fixes it: + +```shell +chmod 644 ~/.relix/profiles.json +relix -P dev -e 'Stock' 2>&1; echo "exit $?" +chmod 600 ~/.relix/profiles.json +``` +``` +relix: ~/.relix/profiles.json is readable by group or others, and a profile may hold credentials; make it private with: chmod 600 ~/.relix/profiles.json +exit 5 +``` + +Where the file system has no POSIX permissions, as on Windows, the check is not made. + +## The environment, and `-D` + +A value not in the selected profile comes from the environment, which is what a CI job's +secrets and a twelve-factor deployment use. Here no profile is selected: `-D` gives the +address and the user, and the environment the password: + +```shell +WAREHOUSE_PASSWORD=s3cret relix -D WAREHOUSE_URL=jdbc:h2:$HOME/shop/data/warehouse -D WAREHOUSE_USER=shop -e 'Stock' -o tsv +``` +``` +sku on_hand +mug 12 +tee 0 +tote 31 +``` + +Without the password, the query fails before the connection is opened, and the error +names what is missing. The status is 3, as for any script that cannot be analysed: + +```shell +relix -D WAREHOUSE_URL=jdbc:h2:$HOME/shop/data/warehouse -D WAREHOUSE_USER=shop -e 'Stock' 2>&1; echo "exit $?" +``` +``` +relix: -e: placeholder ${WAREHOUSE_PASSWORD} in connection 'warehouse' has no value: the session's placeholder resolver returned none +exit 3 +``` + +`-D NAME=VALUE` sets one value for one run, and beats the profile and the environment. +It is for values that are not secret: a command line is kept in your shell's history and +shown by `ps` to everyone on the machine. A secret belongs in a private `profiles.json`, +or in the environment. + +## What is never printed + +The placeholder stays in the declaration, and the value goes only to the connector that +needs it, for the run that needs it. Commands that print a script or a plan — +`relix fmt`, `relix ir`, `relix bundle`, `relix explain`, `relix catalog` — print +`${WAREHOUSE_PASSWORD}`, never the password, and so does an error message: + +```shell +relix fmt .relix/catalog/20-warehouse.relix +``` +``` +-- The stock database. Its address and credentials come from placeholders. +connection warehouse from jdbc { password: "${WAREHOUSE_PASSWORD}", url: "${WAREHOUSE_URL}", user: "${WAREHOUSE_USER}" }; + +source Stock from warehouse { table: "STOCK", schema: { sku: string, on_hand: number } }; +``` diff --git a/src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java b/src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java new file mode 100644 index 0000000..f5a2458 --- /dev/null +++ b/src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java @@ -0,0 +1,418 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.packaging; + +import com.darkcollective.relix.cli.command.RelixCommand; +import com.darkcollective.relix.cli.io.Host; +import com.darkcollective.relix.cli.io.Interruption; +import picocli.CommandLine; +import picocli.CommandLine.Help; +import picocli.CommandLine.Model.ArgSpec; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Model.OptionSpec; +import picocli.CommandLine.Model.PositionalParamSpec; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Writes the command reference of the programming guide's command-line section: a + * Markdown page per command, from the same picocli model {@code --help} and the man pages + * are printed from, so the three cannot drift (design §7.4), and {@code index.json}, the + * section's machine-readable index. + * + *

A page is the command's description, usage, arguments, options, subcommands and exit + * status, then the hand-written part from {@code docs/command-examples/.md}: its + * examples, which the examples test runs like every other page's. The root command's page, + * {@code relix.md}, holds the global options every command takes; the others link to it + * rather than repeat them. + * + *

The pages are checked in, so that a change to the command's options shows up in its + * documentation in the same diff; {@code CommandPagesTest} fails when they are stale, and + * {@code ./gradlew commandPages} rewrites them. + */ +public final class CommandPages { + + /** The section's pages, relative to the repository. */ + public static final String GUIDE = "docs/guide/command-line"; + + /** The hand-written part of each command page, relative to the repository. */ + public static final String EXAMPLES = "docs/command-examples"; + + /** Where the command pages go, relative to the section. */ + public static final String COMMANDS = "commands"; + + /** The section's machine-readable index, relative to the section. */ + public static final String INDEX = "index.json"; + + /** The width the usage synopsis is wrapped to. */ + private static final int WIDTH = 80; + + private static final Pattern INDEX_CHAPTER = Pattern.compile("^##\\s+(.+?)\\s*$"); + private static final Pattern INDEX_TITLE = Pattern.compile("^#\\s+(.+?)\\s*$"); + private static final Pattern LINK = Pattern.compile("\\[([^\\]]+)]\\(([^)#\\s]+\\.md)\\)"); + + private CommandPages() { + } + + /** + * Writes the pages and the index. + * + * @param args the repository's directory + * @throws IOException if one cannot be written + */ + public static void main(String[] args) throws IOException { + if (args.length != 1) { + throw new IllegalArgumentException("usage: CommandPages REPOSITORY"); + } + Path repository = Path.of(args[0]); + Path guide = repository.resolve(GUIDE); + Map files = generate(repository.resolve(EXAMPLES), guide); + Path commands = guide.resolve(COMMANDS); + Files.createDirectories(commands); + try (var stale = Files.list(commands)) { + for (Path file : stale.toList()) { + if (!files.containsKey(COMMANDS + "/" + file.getFileName())) { + Files.delete(file); + } + } + } + for (Map.Entry file : files.entrySet()) { + Files.writeString(guide.resolve(file.getKey()), file.getValue(), StandardCharsets.UTF_8); + } + } + + /** + * Every file this writes, by its path relative to the section. + * + * @param examples the directory of the command pages' hand-written parts + * @param guide the section, whose {@code README.md} the index is read from + * @return the command pages and {@code index.json} + */ + public static Map generate(Path examples, Path guide) { + CommandSpec root = new CommandLine(new RelixCommand(Host.system(), new Interruption())).getCommandSpec(); + Map files = new LinkedHashMap<>(); + List commands = new ArrayList<>(); + commands.add(root); + commands.addAll(visible(root)); + for (CommandSpec command : commands) { + String name = fileName(command); + files.put(COMMANDS + "/" + name, page(command, root, examples.resolve(name))); + } + files.put(INDEX, index(guide.resolve("README.md"), commands)); + return files; + } + + // ── a command's page ───────────────────────────────────────────────────────────── + + private static String page(CommandSpec command, CommandSpec root, Path examples) { + boolean isRoot = command == root; + StringBuilder md = new StringBuilder(); + md.append("# ").append(command.qualifiedName()).append("\n\n"); + description(md, command); + section(md, command, "##", isRoot); + if (isRoot) { + md.append("## Global options\n\n"); + md.append("Every command takes these, before or after its name.\n\n"); + options(md, global(root)); + md.append("## Commands\n\n"); + md.append("| Command | What it does |\n|---|---|\n"); + for (CommandSpec sub : visible(root)) { + md.append("| [`").append(sub.qualifiedName()).append("`](").append(fileName(sub)).append(") | ") + .append(cell(firstSentence(sub))).append(" |\n"); + } + md.append('\n'); + exitStatus(md, root, "##"); + } + for (CommandSpec sub : visible(command)) { + if (isRoot) { + break; + } + md.append("## ").append(sub.qualifiedName()).append("\n\n"); + description(md, sub); + section(md, sub, "###", false); + } + String handWritten = read(examples); + if (handWritten != null) { + md.append(handWritten.strip()).append('\n'); + } + return md.toString().replaceAll("\n{3,}", "\n\n").strip() + "\n"; + } + + /** Usage, arguments, options and exit status, under headings of {@code level}. */ + private static void section(StringBuilder md, CommandSpec command, String level, boolean isRoot) { + md.append(level).append(" Usage").append("\n\n```text\n") + .append(synopsis(command)).append("```\n\n"); + List positionals = command.positionalParameters().stream() + .filter(p -> !p.hidden()).toList(); + if (!positionals.isEmpty()) { + md.append(level).append(" Arguments\n\n| Argument | Description |\n|---|---|\n"); + for (PositionalParamSpec p : positionals) { + md.append("| `").append(p.paramLabel()).append(p.arity().max() > 1 ? "...`" : "`").append(" | ") + .append(cell(description(p))).append(" |\n"); + } + md.append('\n'); + } + List own = command.options().stream() + .filter(o -> !o.hidden() && !o.inherited() && !o.usageHelp() && !o.versionHelp()) + .filter(o -> !isRoot || !global(command).contains(o)) + .toList(); + if (!own.isEmpty()) { + md.append(level).append(isRoot ? " Run options" : " Options").append("\n\n"); + if (isRoot) { + md.append("With no command, `relix` is `relix run`, and takes its options:\n\n"); + } + options(md, own); + } + if (!isRoot) { + md.append("It also takes the [global options](relix.md#global-options).\n\n"); + } + if (!isRoot) { + exitStatus(md, command, level); + } + } + + private static void exitStatus(StringBuilder md, CommandSpec command, String level) { + Map exits = command.usageMessage().exitCodeList(); + if (!exits.isEmpty()) { + md.append(level).append(" Exit status\n\n| Status | Meaning |\n|---|---|\n"); + exits.forEach((code, meaning) -> md.append("| ").append(code.strip()).append(" | ") + .append(cell(meaning)).append(" |\n")); + md.append('\n'); + } + } + + private static void options(StringBuilder md, List options) { + md.append("| Option | Description |\n|---|---|\n"); + for (OptionSpec o : options) { + md.append("| ").append(optionNames(o)).append(" | ").append(cell(description(o))).append(" |\n"); + } + md.append('\n'); + } + + private static String optionNames(OptionSpec option) { + String label = option.typeInfo().isBoolean() || option.arity().max() == 0 ? "" : option.paramLabel(); + List names = new ArrayList<>(); + for (String name : option.names()) { + boolean last = name.equals(option.longestName()); + if (label.isEmpty() || !last) { + names.add("`" + name + "`"); + } else { + String separator = name.startsWith("--") ? "=" : " "; + names.add("`" + name + separator + label + "`"); + } + } + return String.join(", ", names); + } + + private static List global(CommandSpec root) { + CommandSpec mixin = root.mixins().get("options"); + Set names = mixin.options().stream().map(OptionSpec::longestName).collect(Collectors.toSet()); + List global = new ArrayList<>(); + for (OptionSpec o : root.options()) { + if (names.contains(o.longestName()) || o.usageHelp() || o.versionHelp()) { + global.add(o); + } + } + return global; + } + + private static void description(StringBuilder md, CommandSpec command) { + List code = new ArrayList<>(); + List prose = new ArrayList<>(); + for (String line : command.usageMessage().description()) { + for (String part : line.split("%n", -1)) { + if (part.startsWith(" ") && !part.isBlank()) { + flush(md, prose); + code.add(part.strip()); + continue; + } + if (!code.isEmpty()) { + md.append("```text\n").append(String.join("\n", code)).append("\n```\n\n"); + code.clear(); + } + if (part.isBlank()) { + flush(md, prose); + } else { + prose.add(part.strip()); + } + } + } + flush(md, prose); + if (!code.isEmpty()) { + md.append("```text\n").append(String.join("\n", code)).append("\n```\n\n"); + } + } + + private static void flush(StringBuilder md, List prose) { + if (!prose.isEmpty()) { + md.append(String.join(" ", prose)).append("\n\n"); + prose.clear(); + } + } + + private static String synopsis(CommandSpec command) { + command.usageMessage().autoWidth(false).width(WIDTH).abbreviateSynopsis(true); + Help help = new Help(command, Help.defaultColorScheme(Help.Ansi.OFF)); + return help.synopsis(0); + } + + private static String description(ArgSpec arg) { + return String.join(" ", arg.description()).replaceAll("\\s+", " ").strip(); + } + + private static String firstSentence(CommandSpec command) { + String text = String.join(" ", command.usageMessage().description()) + .replace("%n", " ").replaceAll("\\s+", " ").strip(); + int stop = text.indexOf(". "); + String sentence = stop < 0 ? text : text.substring(0, stop + 1); + return sentence.endsWith(".") ? sentence : sentence + "."; + } + + private static String cell(String text) { + return text.replace("|", "\\|"); + } + + private static List visible(CommandSpec command) { + return command.subcommands().entrySet().stream() + .filter(e -> e.getKey().equals(e.getValue().getCommandSpec().name())) + .map(e -> e.getValue().getCommandSpec()) + .filter(c -> !c.usageMessage().hidden()) + .toList(); + } + + /** {@code relix.md} for the root, {@code run.md} for {@code relix run}. */ + private static String fileName(CommandSpec command) { + return command.name() + ".md"; + } + + // ── index.json ─────────────────────────────────────────────────────────────────── + + /** + * The section's index as JSON: each page in reading order, with its chapter, its title + * and what the index says it covers, and for a command page the command and its + * subcommands, each with the anchor of its section. + */ + private static String index(Path readme, List commands) { + StringBuilder json = new StringBuilder(); + json.append("{\n \"title\": ").append(string(title(readme))).append(",\n \"pages\": [\n"); + List entries = new ArrayList<>(); + Map byFile = new LinkedHashMap<>(); + for (CommandSpec command : commands) { + byFile.put(COMMANDS + "/" + fileName(command), command); + } + String chapter = null; + for (String line : lines(readme)) { + Matcher heading = INDEX_CHAPTER.matcher(line); + if (heading.matches()) { + chapter = heading.group(1); + continue; + } + Matcher link = LINK.matcher(line); + if (chapter == null || !line.startsWith("|") || !link.find()) { + continue; + } + String path = link.group(2); + String title = link.group(1).replace("`", ""); + String[] cells = line.strip().split("(? 2 ? cells[2].strip().replace("\\|", "|") : ""; + StringBuilder entry = new StringBuilder(); + entry.append(" {\"path\": ").append(string(path)) + .append(", \"chapter\": ").append(string(chapter)) + .append(", \"title\": ").append(string(title)) + .append(", \"summary\": ").append(string(summary)); + CommandSpec command = byFile.get(path); + if (command != null) { + entry.append(", \"command\": ").append(string(command.qualifiedName())); + List subs = visible(command).stream() + .filter(c -> command.parent() != null) + .map(c -> "{\"command\": " + string(c.qualifiedName()) + ", \"anchor\": " + + string(slug(c.qualifiedName())) + "}") + .toList(); + if (!subs.isEmpty()) { + entry.append(", \"subcommands\": [").append(String.join(", ", subs)).append(']'); + } + } + entry.append('}'); + entries.add(entry.toString()); + } + json.append(String.join(",\n", entries)).append("\n ]\n}\n"); + return json.toString(); + } + + private static String title(Path readme) { + for (String line : lines(readme)) { + Matcher title = INDEX_TITLE.matcher(line); + if (title.matches()) { + return title.group(1); + } + } + throw new IllegalStateException(readme + " has no title"); + } + + private static String slug(String text) { + return text.toLowerCase(java.util.Locale.ROOT).replaceAll("[^a-z0-9]+", "-").replaceAll("(^-|-$)", ""); + } + + private static String string(String text) { + StringBuilder out = new StringBuilder("\""); + for (char c : text.toCharArray()) { + switch (c) { + case '"' -> out.append("\\\""); + case '\\' -> out.append("\\\\"); + case '\n' -> out.append("\\n"); + default -> { + if (c < 0x20) { + out.append(String.format("\\u%04x", (int) c)); + } else { + out.append(c); + } + } + } + } + return out.append('"').toString(); + } + + private static List lines(Path file) { + try { + return Files.readAllLines(file, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException("cannot read " + file, e); + } + } + + private static String read(Path file) { + if (!Files.isRegularFile(file)) { + return null; + } + try { + return Files.readString(file, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException("cannot read " + file, e); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java b/src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java new file mode 100644 index 0000000..32c0d82 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java @@ -0,0 +1,292 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import com.darkcollective.relix.cli.Main; +import com.darkcollective.relix.cli.io.Host; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.PosixFilePermissions; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +/** + * Runs the guide's examples where a reader would: in {@code ~/shop}, a copy of the fixture + * tree under {@code src/test/resources/docs-fixture/home}, which is the home directory. + * + *

The fixture is the shop the pages describe: a user-level {@code ~/.relix} with a + * table and a profile, the project {@code ~/shop} with its catalog, data files, scripts and + * profiles, and {@code ~/shop/reports}, a sub-project that overlays {@code Orders}. Setting + * it up does what a fresh copy cannot carry: {@code @HOME@} in a profile becomes the home + * directory's path, every {@code profiles.json} is made private, and the H2 database the + * {@code warehouse} connection opens is written, with user {@code shop} and the password + * {@code ~/.relix/profiles.json} holds. + * + *

An example that is a plain {@code relix} command ({@link ShellCommand}) runs in this + * JVM, with standard output a terminal and standard input a terminal unless the example + * redirects it, as a reader typing it would have them. Anything else runs through + * {@code bash}, with a {@code relix} on its {@code PATH} that starts the command in a JVM + * of its own; there standard output is a pipe, as it is for a reader's pipeline. Both get + * the same environment: the fixture's {@code HOME}, {@code NO_COLOR=1}, {@code TZ=UTC}, + * and nothing of the user running the build. + */ +final class ExampleRunner implements AutoCloseable { + + private static final String FIXTURE = "/docs-fixture/home"; + + /** The H2 database's credentials, as the fixture's profiles give them. */ + private static final String WAREHOUSE_USER = "shop"; + private static final String WAREHOUSE_PASSWORD = "s3cret"; + + private final Path home; + private final Path shop; + private final Path bin; + private final Map environment; + private final Path stdin; + private final Path stdout; + + /** + * A fresh copy of the fixture. + * + * @param root an empty directory to put it in + * @param trusted whether {@code ~/shop} and {@code ~/shop/reports} start trusted + */ + ExampleRunner(Path root, boolean trusted) { + try { + this.home = Files.createDirectories(root.resolve("home")).toRealPath(); + copy(fixture(), home); + this.shop = home.resolve("shop"); + for (Path profiles : List.of(home.resolve(".relix/profiles.json"), shop.resolve(".relix/profiles.json"))) { + Files.writeString(profiles, Files.readString(profiles).replace("@HOME@", home.toString())); + Files.setPosixFilePermissions(profiles, PosixFilePermissions.fromString("rw-------")); + } + warehouse(shop.resolve("data/warehouse")); + this.bin = Files.createDirectories(root.resolve("bin")); + this.environment = environment(home, bin); + this.stdin = root.resolve("stdin.txt"); + this.stdout = root.resolve("stdout.txt"); + shim(bin.resolve("relix"), home); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + if (trusted) { + for (Path project : List.of(shop, shop.resolve("reports"))) { + Output trust = runInProcess(new ShellCommand(Map.of(), List.of("catalog", "trust", project.toString()), + null, null, false, false)); + if (trust.status() != 0) { + throw new IllegalStateException("cannot trust " + project + ": " + trust); + } + } + } + } + + /** {@return the fixture's home directory} */ + Path home() { + return home; + } + + /** + * What an example did. + * + * @param status its exit status + * @param out what it wrote to standard output, with the home directory written {@code ~} + * @param err what it wrote to standard error + * @param shell whether it ran through {@code bash} + */ + record Output(int status, String out, String err, boolean shell) { + @Override + public String toString() { + return (shell ? "bash" : "in-process") + ", exit " + status + "\n── stdout ──\n" + out + + "── stderr ──\n" + err; + } + } + + /** + * Runs an example. + * + * @param command the example's text + * @return what it did + */ + Output run(String command) { + return ShellCommand.parse(command).map(this::runInProcess).orElseGet(() -> runInBash(command)); + } + + private Output runInProcess(ShellCommand command) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayOutputStream err = new ByteArrayOutputStream(); + OutputStream errTarget = command.mergeStderr() ? out : err; + InputStream in; + try { + in = command.heredoc() != null + ? new ByteArrayInputStream(command.heredoc().getBytes(StandardCharsets.UTF_8)) + : command.stdinFile() != null ? Files.newInputStream(shop.resolve(command.stdinFile())) + : new ByteArrayInputStream(new byte[0]); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + Map env = new HashMap<>(environment); + env.putAll(command.environment()); + boolean stdinIsTerminal = command.heredoc() == null && command.stdinFile() == null; + Host host = new Host(in, out, new PrintStream(errTarget, true, StandardCharsets.UTF_8), env, shop, home, + stdinIsTerminal, true); + // The engine finds its driver directory and file cache under the JVM's user.home, + // not $HOME, so the fixture is made that too for the run, as the bash shim does. + String userHome = System.getProperty("user.home"); + int status; + try { + System.setProperty("user.home", home.toString()); + status = Main.run(host, command.arguments().toArray(String[]::new)); + } finally { + System.setProperty("user.home", userHome); + } + String stdout = out.toString(StandardCharsets.UTF_8); + if (command.echoStatus()) { + stdout += "exit " + status + "\n"; + status = 0; + } + return new Output(status, tilde(stdout), err.toString(StandardCharsets.UTF_8), false); + } + + private Output runInBash(String command) { + // Descriptors 8 and 9 are the example's own standard input and output, which the + // shim compares a stage's streams with to tell where a reader's terminal would be. + ProcessBuilder builder = new ProcessBuilder("bash", "-c", "exec 8<&0 9>&1\n" + command) + .directory(shop.toFile()); + builder.environment().clear(); + builder.environment().putAll(environment); + try { + Files.write(stdin, new byte[0]); + Files.write(stdout, new byte[0]); + Path err = Files.createTempFile(bin.getParent(), "stderr", ".txt"); + Process process = builder.redirectInput(stdin.toFile()).redirectOutput(stdout.toFile()) + .redirectError(err.toFile()).start(); + if (!process.waitFor(2, TimeUnit.MINUTES)) { + process.destroyForcibly(); + throw new IllegalStateException("timed out: " + command); + } + return new Output(process.exitValue(), tilde(Files.readString(stdout)), Files.readString(err), true); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + + /** The home directory's path, which differs on every run, written as a reader's shell abbreviates it. */ + private String tilde(String text) { + return text.replace(home.toString(), "~"); + } + + @Override + public void close() { + // Nothing stays open: each example's JVM or session ends with it. + } + + private static Map environment(Path home, Path bin) { + Map env = new HashMap<>(); + // The relix under test first; then where jq, find and xargs are on a build machine. + env.put("PATH", bin + ":/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin"); + env.put("HOME", home.toString()); + env.put("NO_COLOR", "1"); + env.put("TZ", "UTC"); + env.put("LC_ALL", "C"); + env.put("PAGER", "cat"); + return env; + } + + /** + * A {@code relix} for {@code bash}: this JVM's class path and {@link TerminalMain}, the + * fixture as the home directory a JDBC driver's {@code ~} means too, and standard input + * and output taken for terminals when they are the example's own — descriptors 8 and + * 9 — as they would be a reader's. + */ + private static void shim(Path file, Path home) throws IOException { + String java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + String script = "#!/bin/sh\n" + + "in=false; [ /dev/fd/0 -ef /dev/fd/8 ] && in=true\n" + + "out=false; [ /dev/fd/1 -ef /dev/fd/9 ] && out=true\n" + + "exec " + quote(java) + " -XX:TieredStopAtLevel=1 -Xshare:auto" + // The launcher's streams are UTF-8 whatever the locale, so a glyph in a + // message reads as it does on a reader's terminal. + + " -Dstdout.encoding=UTF-8 -Dstderr.encoding=UTF-8" + + " -Duser.home=" + quote(home.toString()) + + " -Drelix.docs.stdinIsTerminal=$in -Drelix.docs.stdoutIsTerminal=$out" + + " -cp " + quote(System.getProperty("java.class.path")) + + " " + TerminalMain.class.getName() + " \"$@\"\n"; + Files.writeString(file, script); + Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rwxr-xr-x")); + } + + private static String quote(String word) { + return "'" + word.replace("'", "'\\''") + "'"; + } + + private static Path fixture() { + URL url = ExampleRunner.class.getResource(FIXTURE); + if (url == null) { + throw new IllegalStateException("no " + FIXTURE + " on the test class path"); + } + try { + return Path.of(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalStateException(e); + } + } + + private static void copy(Path from, Path to) throws IOException { + try (Stream walk = Files.walk(from)) { + for (Path source : walk.toList()) { + Path target = to.resolve(from.relativize(source).toString()); + if (Files.isDirectory(source)) { + Files.createDirectories(target); + } else { + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + } + } + } + } + + /** The stock database the fixture's {@code warehouse} connection opens. */ + private static void warehouse(Path file) { + try (Connection connection = DriverManager.getConnection("jdbc:h2:" + file, WAREHOUSE_USER, WAREHOUSE_PASSWORD); + Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE STOCK (SKU VARCHAR(16) PRIMARY KEY, ON_HAND INT)"); + statement.execute("INSERT INTO STOCK VALUES ('mug', 12), ('tee', 0), ('tote', 31)"); + } catch (SQLException e) { + throw new IllegalStateException("cannot write the warehouse database: " + e.getMessage(), e); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/GuideExamplesTest.java b/src/test/java/com/darkcollective/relix/cli/docs/GuideExamplesTest.java new file mode 100644 index 0000000..0b43af0 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/GuideExamplesTest.java @@ -0,0 +1,134 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Runs every example in the command-line section of the programming guide and compares + * what it writes to standard output, and its exit status, with what the page shows + * (design §7.4), so that no page can be published with an example the command does not + * honour. + * + *

An example is a {@code shell} block; the plain block on the line after it is its + * standard output, and an example with none must print nothing. Its exit status must be + * 0, unless it ends {@code ; echo "exit $?"}, which prints the status for the page to + * show. Standard error is not compared, except where an example sends it to standard + * output with {@code 2>&1}. + * + *

Each page runs in a fresh copy of the fixture ({@link ExampleRunner}), its examples + * in order, so that one may build on what an earlier one did. The project starts trusted, + * except on the pages listed in {@link #UNTRUSTED}, which show trusting it. + * + *

Comparison forgives trailing whitespace on a line, which a table pads with and an + * editor strips, and the path of the fixture's home directory, written {@code ~}. Nothing + * else. + * + *

The examples run where {@code bash} is a POSIX shell, so not on Windows; the pages + * are written for one. Every other platform the build runs on runs them all. + */ +@DisabledOnOs(OS.WINDOWS) +@DisplayName("The command-line guide's examples print what the pages show") +class GuideExamplesTest { + + /** Pages whose project starts untrusted, because they show trusting it. */ + private static final Set UNTRUSTED = Set.of("catalog.md"); + + @TempDir + Path root; + + @TestFactory + List everyExample() { + Path directory = GuidePages.directory(); + List pages = new ArrayList<>(); + for (String page : GuidePages.files(directory)) { + List examples = GuidePages.examples(GuidePages.lines(directory.resolve(page))); + if (examples.isEmpty()) { + continue; + } + AtomicReference runner = new AtomicReference<>(); + List tests = new ArrayList<>(); + for (GuidePages.Example example : examples) { + tests.add(DynamicTest.dynamicTest(page + ":" + example.line(), () -> { + if (runner.get() == null) { + Path fixture = Files.createDirectories(root.resolve(page.replace('/', '_'))); + runner.set(new ExampleRunner(fixture, !UNTRUSTED.contains(page))); + } + check(page, example, runner.get().run(example.command())); + })); + } + pages.add(DynamicContainer.dynamicContainer(page, tests)); + } + assertThat(pages).as("pages with examples").isNotEmpty(); + return pages; + } + + private static void check(String page, GuidePages.Example example, ExampleRunner.Output output) { + String where = page + ":" + example.line() + "\n" + example.command() + "\n"; + assertThat(output.status()) + .as(where + "exits 0 (end it with ; echo \"exit $?\" to show another status)\n" + output) + .isZero(); + assertThat(normalise(output.out())) + .as(where + output) + .isEqualTo(example.expected() == null ? "" : normalise(example.expected())); + } + + private static String normalise(String text) { + return text.lines().map(String::stripTrailing).reduce((a, b) -> a + "\n" + b).orElse("").stripTrailing(); + } + + @Test + @DisplayName("every command a page shows a reader is a shell example, which runs") + void noUnrunCommandBlocks() { + Path directory = GuidePages.directory(); + List found = new ArrayList<>(); + for (String page : GuidePages.files(directory)) { + for (GuidePages.Block block : GuidePages.blocks(GuidePages.lines(directory.resolve(page)))) { + if (Set.of("bash", "sh", "console", "zsh", "shell-session").contains(block.lang())) { + found.add(page + ":" + block.line() + " (" + block.lang() + ")"); + } + } + } + assertThat(found).as("blocks the examples test would not run; write them as ```shell").isEmpty(); + } + + static String read(Path file) { + try { + return Files.readString(file); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/GuidePages.java b/src/test/java/com/darkcollective/relix/cli/docs/GuidePages.java new file mode 100644 index 0000000..46cfcf6 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/GuidePages.java @@ -0,0 +1,219 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +/** + * The command-line section of the programming guide, {@code docs/guide/command-line}: its + * pages, and the fenced blocks and links in each. + * + *

The pages are Markdown in the dialect the site and the engine's manuals share. A + * fenced block is an example when its language is {@code shell}; the plain fence + * immediately after it, if there is one, is what the example writes to standard output. + */ +final class GuidePages { + + /** The section's directory, relative to the repository. */ + static final Path DIRECTORY = Path.of("docs/guide/command-line"); + + /** The language of a block the examples test runs. */ + static final String EXAMPLE = "shell"; + + private static final Pattern FENCE = Pattern.compile("^```\\s*([A-Za-z0-9_-]*)\\s*$"); + private static final Pattern LINK = Pattern.compile("\\[([^\\]]*)]\\(([^)\\s]+)\\)"); + private static final Pattern HEADING = Pattern.compile("^(#{1,6})\\s+(.+?)\\s*$"); + + private GuidePages() { + } + + /** + * A fenced block. + * + * @param lang its language, empty for a plain fence + * @param body its lines, joined + * @param line the line of its opening fence, from 1 + */ + record Block(String lang, String body, int line) { + } + + /** + * An example and what it prints. + * + * @param line the line of its opening fence + * @param command the block's text + * @param expected the plain block after it, or {@code null} when it prints nothing + */ + record Example(int line, String command, String expected) { + } + + /** + * A link written in a page. + * + * @param line the line it is on + * @param href its target, as written + */ + record Link(int line, String href) { + } + + /** {@return the section's directory: {@code relix.cli.guide}, or the repository's} */ + static Path directory() { + String property = System.getProperty("relix.cli.guide"); + return property != null ? Path.of(property) : DIRECTORY; + } + + /** + * Every Markdown file of the section, as paths relative to its directory. + * + * @param directory the section's directory + * @return the files, in path order + */ + static List files(Path directory) { + try (Stream walk = Files.walk(directory)) { + return walk.filter(p -> p.toString().endsWith(".md")) + .map(p -> directory.relativize(p).toString().replace('\\', '/')) + .sorted() + .toList(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + static List lines(Path file) { + try { + return Files.readAllLines(file, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * The fenced blocks of a page. + * + * @param lines the page + * @return its blocks, in order + */ + static List blocks(List lines) { + List blocks = new ArrayList<>(); + for (int i = 0; i < lines.size(); i++) { + Matcher open = FENCE.matcher(lines.get(i)); + if (!open.matches()) { + continue; + } + int start = i; + List body = new ArrayList<>(); + for (i++; i < lines.size() && !lines.get(i).strip().equals("```"); i++) { + body.add(lines.get(i)); + } + blocks.add(new Block(open.group(1), String.join("\n", body), start + 1)); + } + return blocks; + } + + /** + * The examples of a page: each {@code shell} block, paired with the plain block that + * follows on the very next line. + * + * @param lines the page + * @return its examples, in order + */ + static List examples(List lines) { + List blocks = blocks(lines); + List examples = new ArrayList<>(); + for (int i = 0; i < blocks.size(); i++) { + Block block = blocks.get(i); + if (!block.lang().equals(EXAMPLE)) { + continue; + } + String expected = null; + if (i + 1 < blocks.size()) { + Block next = blocks.get(i + 1); + int closing = block.line() + (int) block.body().lines().count() + 1; + if (next.lang().isEmpty() && next.line() == closing + 1) { + expected = next.body(); + i++; + } + } + examples.add(new Example(block.line(), block.body(), expected)); + } + return examples; + } + + /** + * The links of a page, outside its fenced blocks and code spans. + * + * @param lines the page + * @return its links, in order + */ + static List links(List lines) { + List links = new ArrayList<>(); + boolean fenced = false; + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i); + if (FENCE.matcher(line).matches()) { + fenced = !fenced; + continue; + } + if (fenced) { + continue; + } + Matcher link = LINK.matcher(line.replaceAll("`[^`]*`", "")); + while (link.find()) { + links.add(new Link(i + 1, link.group(2))); + } + } + return links; + } + + /** + * The anchors a page's headings get on the site: the heading's text, lower-cased, with + * each run of anything but a letter or digit made a hyphen. + * + * @param lines the page + * @return its anchors + */ + static List anchors(List lines) { + List anchors = new ArrayList<>(); + boolean fenced = false; + for (String line : lines) { + if (FENCE.matcher(line).matches()) { + fenced = !fenced; + continue; + } + Matcher heading = HEADING.matcher(line); + if (!fenced && heading.matches()) { + anchors.add(slug(heading.group(2).replace("`", ""))); + } + } + return anchors; + } + + static String slug(String text) { + String slug = text.toLowerCase(java.util.Locale.ROOT) + .replaceAll("[^a-z0-9]+", "-") + .replaceAll("(^-|-$)", ""); + return slug.isEmpty() ? "section" : slug; + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/GuideStructureTest.java b/src/test/java/com/darkcollective/relix/cli/docs/GuideStructureTest.java new file mode 100644 index 0000000..0e9a0d9 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/GuideStructureTest.java @@ -0,0 +1,169 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import com.darkcollective.relix.cli.packaging.CommandPages; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Holds the command-line section of the programming guide together: every command has a + * page and the pages are what the command model makes of it now, the index lists every + * page, and no link between pages, or into the engine's manuals, is broken. + * + *

The section is rendered by the site inside the engine's programming guide, at + * {@code guide/command-line/}, so a link out of it is written as if it sat in the + * engine's {@code docs/guide/command-line}: {@code ../secrets.md} is a guide page and + * {@code ../../reference/README.md} the language reference. Such a link is resolved here + * against the pinned engine's manuals, which the build unpacks. + */ +@DisplayName("The command-line guide") +class GuideStructureTest { + + private static final Pattern INDEX_ROW_LINK = Pattern.compile("^\\|\\s*\\[[^\\]]+]\\(([^)#]+\\.md)\\)"); + + private final Path guide = GuidePages.directory(); + + @Test + @DisplayName("has the command pages and index the command model makes now") + void commandPagesAreCurrent() { + Map generated = CommandPages.generate(examples(), guide); + List stale = new ArrayList<>(); + generated.forEach((path, text) -> { + Path file = guide.resolve(path); + if (!Files.isRegularFile(file) || !GuideExamplesTest.read(file).equals(text)) { + stale.add(path); + } + }); + for (String page : GuidePages.files(guide)) { + if (page.startsWith(CommandPages.COMMANDS + "/") && !generated.containsKey(page)) { + stale.add(page + " (no such command)"); + } + } + assertThat(stale).as("stale command pages; run ./gradlew commandPages").isEmpty(); + } + + @Test + @DisplayName("gives every hand-written part of a command page a command") + void everyExamplesFileHasACommand() { + Set commands = new LinkedHashSet<>(); + for (String path : CommandPages.generate(examples(), guide).keySet()) { + if (path.startsWith(CommandPages.COMMANDS + "/")) { + commands.add(path.substring(CommandPages.COMMANDS.length() + 1)); + } + } + List orphans = new ArrayList<>(); + for (String file : GuidePages.files(examples())) { + if (!commands.contains(file)) { + orphans.add(file); + } + } + assertThat(orphans).as("files in docs/command-examples naming no command").isEmpty(); + } + + @Test + @DisplayName("lists every page in its index, and links nothing that is not there") + void indexListsEveryPage() { + Set listed = new LinkedHashSet<>(); + for (String line : GuidePages.lines(guide.resolve("README.md"))) { + Matcher row = INDEX_ROW_LINK.matcher(line); + if (row.find()) { + listed.add(row.group(1)); + } + } + Set pages = new LinkedHashSet<>(GuidePages.files(guide)); + pages.remove("README.md"); + assertThat(listed).as("pages the index lists").containsExactlyInAnyOrderElementsOf(pages); + } + + @Test + @DisplayName("has no broken link, to a page or to a heading") + void noBrokenLinks() { + Path engine = engineDocs(); + List broken = new ArrayList<>(); + for (String page : GuidePages.files(guide)) { + List lines = GuidePages.lines(guide.resolve(page)); + for (GuidePages.Link link : GuidePages.links(lines)) { + String problem = problem(page, link.href(), engine); + if (problem != null) { + broken.add(page + ":" + link.line() + ": " + link.href() + ": " + problem); + } + } + } + assertThat(broken).isEmpty(); + } + + /** What is wrong with a link written on {@code page}, or {@code null} when nothing is. */ + private String problem(String page, String href, Path engine) { + if (href.startsWith("https://") || href.startsWith("http://") || href.startsWith("mailto:")) { + return null; + } + int hash = href.indexOf('#'); + String target = hash < 0 ? href : href.substring(0, hash); + String anchor = hash < 0 ? null : href.substring(hash + 1); + Path file; + if (target.isEmpty()) { + file = guide.resolve(page); + } else { + if (!target.endsWith(".md")) { + return "not a page"; + } + // The section as it sits in the engine's tree: docs/guide/command-line/. + Path virtual = Path.of("docs/guide/command-line").resolve(page).resolveSibling(target).normalize(); + String path = virtual.toString().replace('\\', '/'); + if (path.equals("docs/guide/command-line/README.md")) { + return "the index is not a page of its own on the site; link a page"; + } + if (path.startsWith("docs/guide/command-line/")) { + file = guide.resolve(path.substring("docs/guide/command-line/".length())); + } else if (path.startsWith("docs/")) { + file = engine.resolve(path); + } else { + return "outside the manuals"; + } + } + if (!Files.isRegularFile(file)) { + return "no such page"; + } + if (anchor != null && !GuidePages.anchors(GuidePages.lines(file)).contains(anchor)) { + return "no such heading"; + } + return null; + } + + private static Path examples() { + String property = System.getProperty("relix.cli.commandExamples"); + return Path.of(property != null ? property : CommandPages.EXAMPLES); + } + + private static Path engineDocs() { + String property = System.getProperty("relix.engine.docs"); + assertThat(property).as("relix.engine.docs, the unpacked engine manuals").isNotBlank(); + return Path.of(property); + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/ShellCommand.java b/src/test/java/com/darkcollective/relix/cli/docs/ShellCommand.java new file mode 100644 index 0000000..544ff67 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/ShellCommand.java @@ -0,0 +1,223 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * An example that is one {@code relix} command and nothing a shell would have to do for + * it, so that it can run inside the test's JVM rather than through {@code bash}. + * + *

The forms recognised are the ones the guide's pages write: + * + * 

+ * [NAME=VALUE ...] relix ARG ... [< FILE | <<'EOF' ... EOF] [2>&1 | 2>/dev/null] [; echo "exit $?"]
+ *
+ * + *

An argument may be quoted with single or double quotes or escaped with a backslash, + * and a line may continue with a trailing backslash. Anything else a shell would act on + * — a pipe, an expansion, a glob, a {@code ~}, a second command — makes the example not + * one of these, and it runs through {@code bash}. The test errs that way: an example + * this class wrongly took for a plain command would run differently from how a reader's + * shell runs it, while one it wrongly passes to {@code bash} is only slower. + * + * @param environment the {@code NAME=VALUE} words before {@code relix} + * @param arguments the words after it + * @param stdinFile the file standard input is redirected from, or {@code null} + * @param heredoc the here-document standard input holds, or {@code null} + * @param mergeStderr whether {@code 2>&1} sends standard error to standard output + * @param echoStatus whether the command is followed by {@code echo "exit $?"} + */ +record ShellCommand(Map environment, List arguments, String stdinFile, + String heredoc, boolean mergeStderr, boolean echoStatus) { + + /** What prints the exit status after an example, so the page can show it. */ + static final String ECHO_STATUS = "; echo \"exit $?\""; + + private static final Pattern ASSIGNMENT = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*=.*", Pattern.DOTALL); + private static final Pattern HEREDOC = Pattern.compile("<<-?'([A-Za-z_]+)'"); + + /** + * Reads an example as a plain {@code relix} command, if it is one. + * + * @param text the example, as the page's {@code shell} block holds it + * @return the command, or empty when a shell has to run it + */ + static Optional parse(String text) { + String body = text.strip(); + String heredoc = null; + int newline = body.indexOf('\n'); + Matcher here = HEREDOC.matcher(newline < 0 ? body : body.substring(0, newline)); + if (here.find()) { + // relix ... <<'EOF' on the first line, the document after it, EOF alone last. + String delimiter = here.group(1); + List lines = body.lines().toList(); + if (lines.size() < 2 || !lines.getLast().equals(delimiter)) { + return Optional.empty(); + } + heredoc = String.join("\n", lines.subList(1, lines.size() - 1)) + "\n"; + body = lines.getFirst().substring(0, here.start()) + lines.getFirst().substring(here.end()); + } else { + body = body.replace("\\\n", " "); + if (body.contains("\n")) { + return Optional.empty(); + } + } + + boolean echoStatus = false; + if (body.endsWith(ECHO_STATUS)) { + echoStatus = true; + body = body.substring(0, body.length() - ECHO_STATUS.length()); + } + + List words = words(body); + if (words == null) { + return Optional.empty(); + } + Map environment = new LinkedHashMap<>(); + int i = 0; + while (i < words.size() && ASSIGNMENT.matcher(words.get(i)).matches()) { + String word = words.get(i++); + int equals = word.indexOf('='); + environment.put(word.substring(0, equals), word.substring(equals + 1)); + } + if (i >= words.size() || !words.get(i).equals("relix")) { + return Optional.empty(); + } + i++; + + List arguments = new ArrayList<>(); + String stdinFile = null; + boolean mergeStderr = false; + for (; i < words.size(); i++) { + String word = words.get(i); + switch (word) { + case "\u0000<" -> { + if (i + 1 >= words.size() || stdinFile != null || heredoc != null) { + return Optional.empty(); + } + stdinFile = words.get(++i); + } + case "\u00002>&1" -> mergeStderr = true; + case "\u00002>/dev/null" -> { + // Standard error is not compared, so discarding it changes nothing. + } + default -> { + if (word.startsWith("\u0000")) { + return Optional.empty(); + } + arguments.add(word); + } + } + } + return Optional.of(new ShellCommand(environment, arguments, stdinFile, heredoc, mergeStderr, echoStatus)); + } + + /** + * Splits a line into words as a shell would, or returns {@code null} when the line + * holds anything a shell would do more with than quote. The redirections this class + * knows come back as words of their own, marked with a leading NUL that no word typed + * on a page can hold. + */ + private static List words(String line) { + List words = new ArrayList<>(); + StringBuilder word = null; + int i = 0; + while (i < line.length()) { + char c = line.charAt(i); + if (c == ' ' || c == '\t') { + if (word != null) { + words.add(word.toString()); + word = null; + } + i++; + continue; + } + if (word == null) { + if (line.startsWith("2>&1", i) || line.startsWith("2>/dev/null", i)) { + String redirect = line.startsWith("2>&1", i) ? "2>&1" : "2>/dev/null"; + int end = i + redirect.length(); + if (end == line.length() || line.charAt(end) == ' ') { + words.add("\u0000" + redirect); + i = end; + continue; + } + } + if (c == '<') { + words.add("\u0000<"); + i++; + continue; + } + if (c == '~' || c == '#') { + return null; + } + word = new StringBuilder(); + } + switch (c) { + case '\'' -> { + int close = line.indexOf('\'', i + 1); + if (close < 0) { + return null; + } + word.append(line, i + 1, close); + i = close + 1; + } + case '"' -> { + i++; + while (i < line.length() && line.charAt(i) != '"') { + char d = line.charAt(i); + if (d == '$' || d == '`' || d == '!') { + return null; + } + if (d == '\\' && i + 1 < line.length() && "\"\\".indexOf(line.charAt(i + 1)) >= 0) { + d = line.charAt(++i); + } + word.append(d); + i++; + } + if (i >= line.length()) { + return null; + } + i++; + } + case '\\' -> { + if (i + 1 >= line.length()) { + return null; + } + word.append(line.charAt(i + 1)); + i += 2; + } + default -> { + if ("|&;<>()$`*?[]{}!".indexOf(c) >= 0) { + return null; + } + word.append(c); + i++; + } + } + } + if (word != null) { + words.add(word.toString()); + } + return words; + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/TerminalMain.java b/src/test/java/com/darkcollective/relix/cli/docs/TerminalMain.java new file mode 100644 index 0000000..a722193 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/TerminalMain.java @@ -0,0 +1,54 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import com.darkcollective.relix.cli.Main; +import com.darkcollective.relix.cli.io.Host; +import com.darkcollective.relix.cli.io.Interruption; + +import java.io.FileDescriptor; +import java.io.FileOutputStream; +import java.nio.file.Path; + +/** + * The {@code relix} an example's {@code bash} runs: the command's own entry point, over a + * host whose standard streams are terminals where a reader's would be. + * + *

A reader typing {@code relix -e … | jq} has a terminal on {@code relix}'s standard + * input, and one on {@code jq}'s standard output; the example's run has files there, which + * {@link ExampleRunner} gives the shim. The shim compares each stream with them and says + * which is "the terminal" in two system properties, so that the command chooses the + * format, and whether to read a script from standard input, as it would for the reader. + */ +public final class TerminalMain { + + private TerminalMain() { + } + + /** + * Runs the command and exits with its status. + * + * @param args the command line + */ + public static void main(String[] args) { + Interruption interruption = new Interruption(); + Runtime.getRuntime().addShutdownHook(new Thread(interruption::interrupt, "relix-interrupt")); + Host host = new Host(System.in, new FileOutputStream(FileDescriptor.out), System.err, System.getenv(), + Path.of(System.getProperty("user.dir")), Path.of(System.getProperty("user.home")), + Boolean.getBoolean("relix.docs.stdinIsTerminal"), Boolean.getBoolean("relix.docs.stdoutIsTerminal")); + System.exit(Main.run(host, interruption, args)); + } +} diff --git a/src/test/resources/docs-fixture/home/.relix/catalog/00-regions.relix b/src/test/resources/docs-fixture/home/.relix/catalog/00-regions.relix new file mode 100644 index 0000000..b22f3d6 --- /dev/null +++ b/src/test/resources/docs-fixture/home/.relix/catalog/00-regions.relix @@ -0,0 +1,8 @@ +-- Your own tables: every directory under your home sees them. +Regions := [ +| city | region | +|--------|------------| +| London | South East | +| Leeds | North | +| York | North | +]; diff --git a/src/test/resources/docs-fixture/home/.relix/profiles.json b/src/test/resources/docs-fixture/home/.relix/profiles.json new file mode 100644 index 0000000..c289ff4 --- /dev/null +++ b/src/test/resources/docs-fixture/home/.relix/profiles.json @@ -0,0 +1,3 @@ +{ + "dev": { "WAREHOUSE_PASSWORD": "s3cret" } +} diff --git a/src/test/resources/docs-fixture/home/shop/.relix/catalog/00-sources.relix b/src/test/resources/docs-fixture/home/shop/.relix/catalog/00-sources.relix new file mode 100644 index 0000000..bc6ca4f --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/.relix/catalog/00-sources.relix @@ -0,0 +1,10 @@ +-- The shop's data. A relative path resolves against this file's directory. +source Customers from csv("../../data/customers.csv") { + header: true, + schema: { customer_id: NUMBER, name: STRING, city: STRING } +}; + +source Orders from csv("../../data/orders.csv") { + header: true, + schema: { order_id: NUMBER, customer_id: NUMBER, amount: NUMBER, status: STRING, placed: DATE } +}; diff --git a/src/test/resources/docs-fixture/home/shop/.relix/catalog/10-views.relix b/src/test/resources/docs-fixture/home/shop/.relix/catalog/10-views.relix new file mode 100644 index 0000000..8a7cd7f --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/.relix/catalog/10-views.relix @@ -0,0 +1,5 @@ +Completed := { SELECT status = 'completed' (Orders) }; + +Spend := { GROUP customer_id, SUM(amount) -> total (Completed) }; + +BigSpenders := { PROJECT name, total (Customers JOIN (SELECT total >= 200 (Spend))) }; diff --git a/src/test/resources/docs-fixture/home/shop/.relix/catalog/20-warehouse.relix b/src/test/resources/docs-fixture/home/shop/.relix/catalog/20-warehouse.relix new file mode 100644 index 0000000..cb4e96c --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/.relix/catalog/20-warehouse.relix @@ -0,0 +1,11 @@ +-- The stock database. Its address and credentials come from placeholders. +connection warehouse from database { + url: "${WAREHOUSE_URL}", + user: "${WAREHOUSE_USER}", + password: "${WAREHOUSE_PASSWORD}" +}; + +source Stock from warehouse { + table: "STOCK", + schema: { sku: STRING, on_hand: NUMBER } +}; diff --git a/src/test/resources/docs-fixture/home/shop/.relix/profiles.json b/src/test/resources/docs-fixture/home/shop/.relix/profiles.json new file mode 100644 index 0000000..3ee415f --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/.relix/profiles.json @@ -0,0 +1,6 @@ +{ + "dev": { + "WAREHOUSE_URL": "jdbc:h2:@HOME@/shop/data/warehouse", + "WAREHOUSE_USER": "shop" + } +} diff --git a/src/test/resources/docs-fixture/home/shop/data/customers.csv b/src/test/resources/docs-fixture/home/shop/data/customers.csv new file mode 100644 index 0000000..1124ade --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/data/customers.csv @@ -0,0 +1,5 @@ +customer_id,name,city +1,Alice,London +2,Bob,Leeds +3,Carol,London +4,Dan,York diff --git a/src/test/resources/docs-fixture/home/shop/data/orders.csv b/src/test/resources/docs-fixture/home/shop/data/orders.csv new file mode 100644 index 0000000..aa5f79c --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/data/orders.csv @@ -0,0 +1,5 @@ +order_id,customer_id,amount,status,placed +1,1,120,completed,2026-09-02 +2,1,80,completed,2026-09-14 +3,2,50,pending,2026-09-20 +4,3,200,completed,2026-10-01 diff --git a/src/test/resources/docs-fixture/home/shop/data/returns.ndjson b/src/test/resources/docs-fixture/home/shop/data/returns.ndjson new file mode 100644 index 0000000..1131b81 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/data/returns.ndjson @@ -0,0 +1,2 @@ +{"order_id": 2, "reason": "damaged", "refund": {"amount": 80, "currency": "GBP"}} +{"order_id": 4, "reason": "late", "refund": {"amount": 20, "currency": "GBP"}} diff --git a/src/test/resources/docs-fixture/home/shop/reports/.relix/catalog/10-sample.relix b/src/test/resources/docs-fixture/home/shop/reports/.relix/catalog/10-sample.relix new file mode 100644 index 0000000..aa0a3b4 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/reports/.relix/catalog/10-sample.relix @@ -0,0 +1,5 @@ +-- Reports are written against a fixed sample rather than the live orders. +source Orders from csv("../../sample-orders.csv") { + header: true, + schema: { order_id: NUMBER, customer_id: NUMBER, amount: NUMBER, status: STRING, placed: DATE } +}; diff --git a/src/test/resources/docs-fixture/home/shop/reports/sample-orders.csv b/src/test/resources/docs-fixture/home/shop/reports/sample-orders.csv new file mode 100644 index 0000000..5ce152b --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/reports/sample-orders.csv @@ -0,0 +1,3 @@ +order_id,customer_id,amount,status,placed +101,1,500,completed,2026-01-05 +102,4,250,completed,2026-01-06 diff --git a/src/test/resources/docs-fixture/home/shop/scripts/broken.relix b/src/test/resources/docs-fixture/home/shop/scripts/broken.relix new file mode 100644 index 0000000..995f15d --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/scripts/broken.relix @@ -0,0 +1 @@ +query { SELECT amount > 100 (Ordrs) }; diff --git a/src/test/resources/docs-fixture/home/shop/scripts/by-city.relix b/src/test/resources/docs-fixture/home/shop/scripts/by-city.relix new file mode 100644 index 0000000..bfb5427 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/scripts/by-city.relix @@ -0,0 +1,2 @@ +-- The customers of one city, named when the script runs: relix -a city=London +query { PROJECT name (SELECT city = $city (Customers)) }; diff --git a/src/test/resources/docs-fixture/home/shop/scripts/messy.relix b/src/test/resources/docs-fixture/home/shop/scripts/messy.relix new file mode 100644 index 0000000..a533d40 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/scripts/messy.relix @@ -0,0 +1,3 @@ +-- Orders worth a look +Large:={select amount>100(Orders)}; +query {project order_id,amount (Large)}; diff --git a/src/test/resources/docs-fixture/home/shop/scripts/top-customers.relix b/src/test/resources/docs-fixture/home/shop/scripts/top-customers.relix new file mode 100644 index 0000000..a2fff29 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/scripts/top-customers.relix @@ -0,0 +1,2 @@ +-- The customers who have spent the most, best first. +query { SORT total DESC (PROJECT name, total (Customers JOIN Spend)) }; From d4c28606592c9119f9b7a16febdf7594f06ede63 Mon Sep 17 00:00:00 2001 From: David Tooke Date: Fri, 9 Oct 2026 17:59:08 -0400 Subject: [PATCH 2/4] Name the guide's command-line chapters so they read among the guide's own The site renders these chapters after the engine's programming guide's, so each says it is about the command line. --- docs/guide/command-line/README.md | 6 +++--- docs/guide/command-line/index.json | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/guide/command-line/README.md b/docs/guide/command-line/README.md index 5f45f38..1aa3479 100644 --- a/docs/guide/command-line/README.md +++ b/docs/guide/command-line/README.md @@ -12,7 +12,7 @@ Every example in this section is run on each build of the command, against a sma project, and the block under it is what it printed. The command reference is generated from the same model as `relix --help` and the man pages. -## Using the command +## The command line Installing it, and the shape every command shares; then data in, rows out. @@ -22,7 +22,7 @@ Installing it, and the shape every command shares; then data in, rows out. | [Binding inputs](inputs.md) | `-i` and its formats; standard input; inferred and given headings; inputs that never end; shadowing the catalog; query parameters with `-a` | | [Output formats](output.md) | `table`, `tsv`, `csv`, `ndjson`, `json` and `markdown`; NULLs and headers; scripts with several queries; `--fail-empty` and `--fail-rows`; messages; closed pipes and interrupts | -## A project's context +## Projects on the command line What a directory gives the commands run in it. @@ -31,7 +31,7 @@ What a directory gives the commands run in it. | [The catalog](catalog.md) | `.relix/` directories and how they are found; trusting a project; a nearer declaration wins; inspecting the catalog; `--catalog`; `relixrc` | | [Secrets and profiles](secrets.md) | `${NAME}` placeholders; `profiles.json` and `-P`; the environment and `-D`; private profile files; what is never printed | -## Recipes +## Command-line recipes | Page | What it covers | |---|---| diff --git a/docs/guide/command-line/index.json b/docs/guide/command-line/index.json index e2921a8..7830f5e 100644 --- a/docs/guide/command-line/index.json +++ b/docs/guide/command-line/index.json @@ -1,12 +1,12 @@ { "title": "The relix command line", "pages": [ - {"path": "getting-started.md", "chapter": "Using the command", "title": "The relix command", "summary": "Installing; a first query; where the script comes from; the commands; diagnostics and exit status"}, - {"path": "inputs.md", "chapter": "Using the command", "title": "Binding inputs", "summary": "`-i` and its formats; standard input; inferred and given headings; inputs that never end; shadowing the catalog; query parameters with `-a`"}, - {"path": "output.md", "chapter": "Using the command", "title": "Output formats", "summary": "`table`, `tsv`, `csv`, `ndjson`, `json` and `markdown`; NULLs and headers; scripts with several queries; `--fail-empty` and `--fail-rows`; messages; closed pipes and interrupts"}, - {"path": "catalog.md", "chapter": "A project's context", "title": "The catalog", "summary": "`.relix/` directories and how they are found; trusting a project; a nearer declaration wins; inspecting the catalog; `--catalog`; `relixrc`"}, - {"path": "secrets.md", "chapter": "A project's context", "title": "Secrets and profiles", "summary": "`${NAME}` placeholders; `profiles.json` and `-P`; the environment and `-D`; private profile files; what is never printed"}, - {"path": "pipelines.md", "chapter": "Recipes", "title": "Pipelines", "summary": "With `jq`, `awk`, `sort` and `head`; with `find` and `xargs`; scripts in here-documents; a data check in CI; optimising a script and reviewing the change"}, + {"path": "getting-started.md", "chapter": "The command line", "title": "The relix command", "summary": "Installing; a first query; where the script comes from; the commands; diagnostics and exit status"}, + {"path": "inputs.md", "chapter": "The command line", "title": "Binding inputs", "summary": "`-i` and its formats; standard input; inferred and given headings; inputs that never end; shadowing the catalog; query parameters with `-a`"}, + {"path": "output.md", "chapter": "The command line", "title": "Output formats", "summary": "`table`, `tsv`, `csv`, `ndjson`, `json` and `markdown`; NULLs and headers; scripts with several queries; `--fail-empty` and `--fail-rows`; messages; closed pipes and interrupts"}, + {"path": "catalog.md", "chapter": "Projects on the command line", "title": "The catalog", "summary": "`.relix/` directories and how they are found; trusting a project; a nearer declaration wins; inspecting the catalog; `--catalog`; `relixrc`"}, + {"path": "secrets.md", "chapter": "Projects on the command line", "title": "Secrets and profiles", "summary": "`${NAME}` placeholders; `profiles.json` and `-P`; the environment and `-D`; private profile files; what is never printed"}, + {"path": "pipelines.md", "chapter": "Command-line recipes", "title": "Pipelines", "summary": "With `jq`, `awk`, `sort` and `head`; with `find` and `xargs`; scripts in here-documents; a data check in CI; optimising a script and reviewing the change"}, {"path": "commands/relix.md", "chapter": "Command reference", "title": "relix", "summary": "The global options every command takes; the exit statuses; the environment variables and files", "command": "relix"}, {"path": "commands/run.md", "chapter": "Command reference", "title": "relix run", "summary": "Runs scripts and writes their queries' rows", "command": "relix run"}, {"path": "commands/check.md", "chapter": "Command reference", "title": "relix check", "summary": "Analyses scripts without running them, and reports their problems", "command": "relix check"}, From abd0747474f18dbcee00e2086e648fc849af6833 Mon Sep 17 00:00:00 2001 From: David Tooke Date: Fri, 9 Oct 2026 18:00:36 -0400 Subject: [PATCH 3/4] Run the guide's examples in a UTF-8 locale, and keep its pages' LF endings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Linux the JVM decodes its arguments by the locale, so under LC_ALL=C `relix doc σ` looked up '??'. A reader's terminal is UTF-8. On Windows, text=auto checked the generated command pages out with CRLF endings, which then differed from the generator's output; they keep LF, as the .relix files the tests compare do. --- .gitattributes | 4 ++++ .../java/com/darkcollective/relix/cli/docs/ExampleRunner.java | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 958cc52..58365f1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,6 +9,10 @@ gradlew text eol=lf *.ps1 text eol=crlf # Relix scripts: the tests compare what fmt and optimize write with them byte for byte. *.relix text eol=lf +# The command-line guide: its command pages are generated, and compared with the +# generator's output byte for byte. +docs/guide/command-line/** text eol=lf +docs/command-examples/** text eol=lf # Binary files. *.jar binary diff --git a/src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java b/src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java index 32c0d82..09032fb 100644 --- a/src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java +++ b/src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java @@ -222,7 +222,9 @@ private static Map environment(Path home, Path bin) { env.put("HOME", home.toString()); env.put("NO_COLOR", "1"); env.put("TZ", "UTC"); - env.put("LC_ALL", "C"); + // A UTF-8 locale, as a reader's terminal has: on Linux the JVM decodes its + // arguments by the locale, and `relix doc σ` would otherwise not say σ. + env.put("LC_ALL", System.getProperty("os.name").startsWith("Mac") ? "en_US.UTF-8" : "C.UTF-8"); env.put("PAGER", "cat"); return env; } From b349618fa81e0d973ee24375a8c398677bd8a067 Mon Sep 17 00:00:00 2001 From: David Tooke Date: Fri, 9 Oct 2026 18:03:59 -0400 Subject: [PATCH 4/4] Write the command pages with LF endings on Windows too picocli wraps the usage synopsis with the platform's line separator, so on Windows the generated pages held CRLF and never matched the checked-in ones. --- .../com/darkcollective/relix/cli/packaging/CommandPages.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java b/src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java index f5a2458..cff1f97 100644 --- a/src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java +++ b/src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java @@ -160,7 +160,9 @@ private static String page(CommandSpec command, CommandSpec root, Path examples) if (handWritten != null) { md.append(handWritten.strip()).append('\n'); } - return md.toString().replaceAll("\n{3,}", "\n\n").strip() + "\n"; + // picocli wraps the synopsis with the platform's line separator; a page has LF + // endings everywhere, as it is checked in. + return md.toString().replace("\r\n", "\n").replaceAll("\n{3,}", "\n\n").strip() + "\n"; } /** Usage, arguments, options and exit status, under headings of {@code level}. */