diff --git a/.gitattributes b/.gitattributes index 958cc52..58365f1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,6 +9,10 @@ gradlew text eol=lf *.ps1 text eol=crlf # Relix scripts: the tests compare what fmt and optimize write with them byte for byte. *.relix text eol=lf +# The command-line guide: its command pages are generated, and compared with the +# generator's output byte for byte. +docs/guide/command-line/** text eol=lf +docs/command-examples/** text eol=lf # Binary files. *.jar binary diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fdd3747..2f2a6ac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,6 @@ # Releases the command (design §7.3): a platform archive for each of macOS, Linux and -# Windows, attached with their checksums to this repository's GitHub Release, and the +# Windows, attached with their checksums to this repository's GitHub Release with the +# command-line guide's archive (relix--docs.zip, design §7.4), and the # Homebrew formula and Scoop manifest that install them, pushed to the tap # (DarkCollective/homebrew-relix) and the bucket (DarkCollective/scoop-relix). # @@ -89,6 +90,12 @@ jobs: - name: Build, package and test the archive shell: bash run: ./gradlew clean build releaseArchive packageTest -PcliVersion="$VERSION" $SKIP_PIN + # The command-line guide, the same on every platform, so it is built once. Its + # examples have just run in the build above. + - name: Build the command-line guide's archive + if: matrix.platform == 'linux-x86_64' + shell: bash + run: ./gradlew docsArchive -PcliVersion="$VERSION" - name: Check the archive is this platform's shell: bash run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 98b2ec5..76b7bb6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -189,6 +189,27 @@ because that is the interface a script or a pipeline depends on. A change to a command, an option or an exit status updates its help text, which is also what the manual page and the site's command reference are generated from. +The command-line section of the programming guide is in +[`docs/guide/command-line`](docs/guide/command-line/README.md); the site renders it, +and each release attaches it as `relix--docs.zip`. Three things keep it true: + +- **The command pages are generated.** `docs/guide/command-line/commands/*.md` and + `index.json` come from the command's picocli model, followed by the hand-written + examples in `docs/command-examples/.md`. After changing an option or one of + those files, run `./gradlew commandPages` and commit what it writes; + `GuideStructureTest` fails while they are stale. +- **Every example runs.** A ```` ```shell ```` block is an example, and the plain block on + the line after it is what it prints on standard output; `GuideExamplesTest` runs it in + a copy of the fixture under `src/test/resources/docs-fixture` (the `~/shop` the pages + describe) and compares. An example exits 0 unless it ends `; echo "exit $?"`, which + prints its status for the page to show. Write commands a reader would type in a + ```` ```shell ```` block, never `bash` or `console`, and keep anything else — an install + command, a syntax summary — in a ```` ```text ```` block. +- **Every link resolves**, to a page or a heading, here or in the engine's manuals: + the section sits at `docs/guide/command-line` in the engine's tree, so + `../secrets.md` is the guide's page and `../../reference/README.md` the language + reference. A new page is listed in the section's `README.md`. + ## Questions Ask in [Discussions](https://github.com/DarkCollective/relix-cli/discussions). diff --git a/README.md b/README.md index 92adf9a..bb05783 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,10 @@ It is a front end to the Relix engine, [relix-core](https://github.com/DarkCollective/relix-core), and reaches it only through the engine's published API. +Its documentation is the command-line section of the Relix programming guide, in +[`docs/guide/command-line`](docs/guide/command-line/README.md), published on the +[Relix site](https://relix.darkcollective.com/guide/index.html). + ## Status Under development. There is no release yet, and the command line described above is diff --git a/build.gradle b/build.gradle index f0b356a..a3e44a3 100644 --- a/build.gradle +++ b/build.gradle @@ -57,6 +57,11 @@ sourceSets { } // The tests of the packaged command: they run its launcher, not its classes. packageTest + // CommandPagesTest regenerates the command pages to compare them with the checked-in ones. + test { + compileClasspath += sourceSets.packaging.output + runtimeClasspath += sourceSets.packaging.output + } } configurations { @@ -175,6 +180,12 @@ test { dependsOn unpackEngineDocs inputs.dir(engineDocsDir).withPropertyName('engineDocs') systemProperty 'relix.engine.docs', engineDocsDir.get().asFile.absolutePath + // The command-line guide's pages, which the docs tests run and check: without these + // inputs a docs-only change would leave the tests UP-TO-DATE, passing without running. + inputs.dir('docs/guide/command-line').withPropertyName('guide') + inputs.dir('docs/command-examples').withPropertyName('commandExamples') + systemProperty 'relix.cli.guide', file('docs/guide/command-line').absolutePath + systemProperty 'relix.cli.commandExamples', file('docs/command-examples').absolutePath finalizedBy tasks.named('jacocoTestReport') } @@ -383,6 +394,32 @@ def releaseFiles = tasks.register('releaseFiles', JavaExec) { } as CommandLineArgumentProvider) } +// ── The command-line guide (design §7.4) ────────────────────────────────────────────── +// +// ./gradlew commandPages rewrites docs/guide/command-line/commands/*.md and +// index.json from the command's picocli model +// ./gradlew docsArchive build/distributions/relix--docs.zip, the +// section the site renders in its programming guide +tasks.register('commandPages', JavaExec) { + group = 'documentation' + description = 'Rewrites the command pages of docs/guide/command-line from the command model.' + classpath = sourceSets.packaging.runtimeClasspath + mainClass = 'com.darkcollective.relix.cli.packaging.CommandPages' + args = [projectDir.absolutePath] +} + +tasks.register('docsArchive', Zip) { + group = 'documentation' + description = 'Builds the command-line guide archive the site renders from.' + from('docs/guide/command-line') { + into 'command-line' + } + archiveFileName = "relix-${version}-docs.zip" + destinationDirectory = layout.buildDirectory.dir('distributions') + preserveFileTimestamps = false + reproducibleFileOrder = true +} + def stageRelease = tasks.register('stageRelease', Sync) { description = 'Lays out the release archive\'s contents in build/release.' dependsOn tasks.named('jlink') diff --git a/docs/command-examples/bundle.md b/docs/command-examples/bundle.md new file mode 100644 index 0000000..8abaf2e --- /dev/null +++ b/docs/command-examples/bundle.md @@ -0,0 +1,10 @@ +## Examples + +A bundle is one JSON document per script, for a tool such as a query playground: + +```shell +relix bundle -e 'SELECT amount > 100 (Orders)' | jq -c 'keys' +``` +``` +["diagnostics","namespace","queries"] +``` diff --git a/docs/command-examples/catalog.md b/docs/command-examples/catalog.md new file mode 100644 index 0000000..17b0281 --- /dev/null +++ b/docs/command-examples/catalog.md @@ -0,0 +1,44 @@ +## Examples + +```shell +relix catalog ls -o tsv +``` +``` +name kind arity file +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/.relix/catalog/00-sources.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +```shell +relix catalog schema Customers -o tsv +``` +``` +column type +customer_id N +name S +city S +``` + +```shell +relix catalog where Orders -o tsv +``` +``` +file line kind state +~/shop/.relix/catalog/00-sources.relix 7 source wins +``` + +```shell +relix catalog trust --list -o tsv +``` +``` +directory state +~/shop trusted +~/shop/reports trusted +``` + +[The catalog](../catalog.md) covers discovery, the overlay and trust. diff --git a/docs/command-examples/check.md b/docs/command-examples/check.md new file mode 100644 index 0000000..a38dafa --- /dev/null +++ b/docs/command-examples/check.md @@ -0,0 +1,30 @@ +## Examples + +A script that analyses prints nothing, and exits 0: + +```shell +relix check scripts/top-customers.relix +``` + +A problem is printed on standard error, and the status is 3: + +```shell +relix check scripts/broken.relix 2>&1; echo "exit $?" +``` +``` +scripts/broken.relix:1:30: error: Undefined relation: 'Ordrs' +exit 3 +``` + +`--json` writes the diagnostics to standard output instead, as one document for a tool to +read: + +```shell +relix check --json scripts/broken.relix; echo "exit $?" +``` +``` +{"diagnostics":[{"file":"scripts/broken.relix","line":1,"column":30,"severity":"error","message":"Undefined relation: 'Ordrs'"}]} +exit 3 +``` + +[Pipelines](../pipelines.md#lint-every-script) checks every script in a tree. diff --git a/docs/command-examples/completion.md b/docs/command-examples/completion.md new file mode 100644 index 0000000..8317bd8 --- /dev/null +++ b/docs/command-examples/completion.md @@ -0,0 +1,14 @@ +## Examples + +The release archives carry each shell's script in `completions/`, and Homebrew installs +them. To load one by hand, add the line for your shell to its startup file: + +```text +bash source <(relix completion bash) +zsh source <(relix completion zsh) +fish relix completion fish > ~/.config/fish/completions/relix.fish +PowerShell relix completion powershell | Out-String | Invoke-Expression +``` + +The zsh script is also a completion function, so it can be saved as `_relix` in a +directory on `$fpath` instead. diff --git a/docs/command-examples/connectors.md b/docs/command-examples/connectors.md new file mode 100644 index 0000000..8e433cb --- /dev/null +++ b/docs/command-examples/connectors.md @@ -0,0 +1,22 @@ +## Examples + +The connectors that ship with `relix`, and any plugins installed in `~/.relix/connectors`: + +```shell +relix connectors ls +``` +``` +── connectors ────────────────────────────────────────────────────────────────── + type installed + ────── ───────── + clf true + csv true + gedcom true + log true +(4 rows) +``` + +`relix connectors ls --available` adds the connectors that can be installed, which it +fetches from the hosted catalog, and `relix connectors install TYPE` installs one. A +source whose type no installed connector reads — `mongodb`, say — fails with exit status +5, unless `--allow-download` lets the run fetch the connector. diff --git a/docs/command-examples/doc.md b/docs/command-examples/doc.md new file mode 100644 index 0000000..adc6a9b --- /dev/null +++ b/docs/command-examples/doc.md @@ -0,0 +1,38 @@ +## Examples + +A page is found by its glyph, its keyword or its name: + +```shell +relix doc --markdown σ | head -6 +``` +``` +# Name: Selection (σ / SELECT) + +# Syntax: +σ (Relation) +SELECT (Relation) +``` + +With no topic, `relix doc` lists the pages, as rows: + +```shell +relix doc -o tsv | head -4 +``` +``` +topic category title summary +getting-started guide Getting started Your first script — declare data, name a result, run it +glossary guide Glossary The vocabulary — relation, tuple, bag vs set, materialisation, pushdown +σ operator Selection Filter rows by a condition +``` + +`--function` lists, or looks up, the functions only: + +```shell +relix doc --function -o tsv | grep -i '^round' +``` +``` +Round function Round Round to nearest +``` + +On a terminal a page is shown through `$PAGER` (`less` when it is not set); `--no-pager` +writes it straight to standard output. diff --git a/docs/command-examples/drivers.md b/docs/command-examples/drivers.md new file mode 100644 index 0000000..4b202cf --- /dev/null +++ b/docs/command-examples/drivers.md @@ -0,0 +1,20 @@ +## Examples + +```shell +relix drivers ls +``` +``` +── drivers ───────────────────────────────────────────────────────────────────── + name scheme installed + ────────── ──────────────── ───────── + postgresql jdbc:postgresql: false + mysql jdbc:mysql: false +(2 rows) +``` + +`relix drivers install NAME` downloads one of those drivers, checks its checksum, and +installs it in `~/.relix/drivers`, or in `$RELIX_DRIVERS` when that is set. A run that +needs a driver that is not installed fails with exit status 5, unless `--allow-download` +lets it fetch the driver itself. The H2 driver is part of +`relix`, and needs no installing. A driver `relix drivers ls` does not list is installed +by copying its jar into the driver directory. diff --git a/docs/command-examples/explain.md b/docs/command-examples/explain.md new file mode 100644 index 0000000..36bd51d --- /dev/null +++ b/docs/command-examples/explain.md @@ -0,0 +1,20 @@ +## Examples + +```shell +relix explain -e 'GROUP city, COUNT(*) -> n (Customers JOIN Orders)' +``` +``` +Aggregate +└─ Join NATURAL/HASH build=RIGHT + ├─ Scan Customers + └─ Scan Orders +``` + +`--json` writes each query's plan as a JSON object instead: + +```shell +relix explain --json -e 'SELECT amount > 100 (Orders)' | jq -c 'keys' +``` +``` +["plan","query","script"] +``` diff --git a/docs/command-examples/fmt.md b/docs/command-examples/fmt.md new file mode 100644 index 0000000..582132f --- /dev/null +++ b/docs/command-examples/fmt.md @@ -0,0 +1,61 @@ +## Examples + +```shell +cat scripts/messy.relix +``` +``` +-- Orders worth a look +Large:={select amount>100(Orders)}; +query {project order_id,amount (Large)}; +``` + +`relix fmt` prints it canonically — one statement per line, spaced, each operator in one +spelling — keeping its comments: + +```shell +relix fmt scripts/messy.relix +``` +``` +-- Orders worth a look +Large := { σ amount > 100 (Orders) }; +query { π order_id, amount (Large) }; +``` + +`--keywords` writes the operators as ASCII keywords instead of glyphs: + +```shell +relix fmt --keywords scripts/messy.relix +``` +``` +-- Orders worth a look +Large := { SELECT amount > 100 (Orders) }; +query { PROJECT order_id, amount (Large) }; +``` + +`--check` changes nothing and names each file that would change, exiting 1 if there is +one; `-w` rewrites each such file in place: + +```shell +relix fmt --check --keywords scripts/messy.relix scripts/by-city.relix; echo "exit $?" +``` +``` +scripts/messy.relix +exit 1 +``` + +```shell +relix fmt -w --keywords scripts/messy.relix +relix fmt --check --keywords scripts/messy.relix; echo "exit $?" +``` +``` +exit 0 +``` + +With no file, `relix fmt` formats standard input to standard output: + +```shell +echo 'query {select amount>100(Orders)};' | relix fmt +``` +``` +query { σ amount > 100 (Orders) }; +``` diff --git a/docs/command-examples/help.md b/docs/command-examples/help.md new file mode 100644 index 0000000..f32a983 --- /dev/null +++ b/docs/command-examples/help.md @@ -0,0 +1,12 @@ +## Examples + +`relix help COMMAND` prints what `relix COMMAND --help` prints: + +```shell +relix help version | head -3 +``` +``` +Usage: relix version [-hNqVv] [--allow-download] [--remote] [-C=DIR] + [--max-fixpoint-rounds=N] [--max-materialized-rows=N] + [--max-processed-rows=N] [--now=INSTANT] [-P=NAME] +``` diff --git a/docs/command-examples/ir.md b/docs/command-examples/ir.md new file mode 100644 index 0000000..8e2cc5e --- /dev/null +++ b/docs/command-examples/ir.md @@ -0,0 +1,67 @@ +## Examples + +```shell +relix ir -e 'Big := { SELECT amount > 100 (Orders) };' -e 'query Big;' +``` +``` +════════════════════════════════════════════════════════════════════════════════ +RELIX IR namespace=default +════════════════════════════════════════════════════════════════════════════════ + kind: SRC=source INL=inline SYS=system QR=view DB=database + FN=function TVF=table-fn LIT=truth-literal + type: N=number S=string B=boolean ?=any [x]=array {…}=struct + D=date T=time TS=timestamp DUR=duration + mat: [bag]=bag [set]=dedup-set [sort]=sorted (stream=no label) + +── SYMBOLS ───────────────────────────────────────────────────────────────────── + Big QR order_id:N customer_id:N amount:N status:S placed:D + BigSpenders QR name:S total:N + catalog SYS name:S kind:S arity:N schema_code:S + columns SYS relation:S column:S type:S ordinal:N distinct_count:N … + Completed QR order_id:N customer_id:N amount:N status:S placed:D + connections SYS name:S dialect:S + Customers SRC customer_id:N name:S city:S + cycles QR name:S + dependencies QR dependent:S depends_on:S + deps TVF (r:S)→relation + events SYS seq:N stage:S code:S description:S target:S rows:N el… + find TVF (col:S)→relation + funcs TVF (cat:S)→relation + functions SYS name:S category:S arity:N max_arity:N return_type:S pu… + impact TVF (r:S)→relation + keys SYS relation:S key:N ordinal:N column:S + Orders SRC order_id:N customer_id:N amount:N status:S placed:D + plan SYS query:S node_id:N parent_id:N ordinal:N op:S label:S … + Regions INL city:S region:S + relations SYS name:S kind:S namespace:S materialization:S row_count:N… + rules QR seq:N code:S description:S target:S + schema TVF (rel:S)→relation + Spend QR customer_id:N total:N + Stock SRC sku:S on_hand:N + unused QR name:S + version SYS component:S kind:S version:S + +── EXPRESSION TREES ──────────────────────────────────────────────────────────── +Big [QR] order_id:N customer_id:N amount:N status:S placed:D + σ amount > 100 + └─ Orders [SRC] + +BigSpenders [QR] name:S total:N + π name, total + └─ ⋈ + ├─ Customers [SRC] + └─ σ total ≥ 200 + └─ Spend [QR] + +Completed [QR] order_id:N customer_id:N amount:N status:S placed:D + σ status = "completed" + └─ Orders [SRC] + +Spend [QR] customer_id:N total:N + γ [customer_id] SUM(amount)→total [bag] + └─ Completed [QR] + +── ROOT QUERIES ──────────────────────────────────────────────────────────────── + query Big [QR] +════════════════════════════════════════════════════════════════════════════════ +``` diff --git a/docs/command-examples/optimize.md b/docs/command-examples/optimize.md new file mode 100644 index 0000000..130dc2d --- /dev/null +++ b/docs/command-examples/optimize.md @@ -0,0 +1,39 @@ +## Examples + +The rewritten script is printed in the same form the engine runs it in: + +```shell +relix optimize -e "query { SELECT city = 'London' (Customers JOIN Orders) };" +``` +``` +query { (σ city = "London" (Customers)) ⋈ (Orders) }; +``` + +`--report` prints the rules that fired, and each query before and after: + +```shell +relix optimize --report -e "query { SELECT city = 'London' (Customers JOIN Orders) };" +``` +``` +════════════════════════════════════════════════════════════════════════════════ +RELIX OPTIMIZER namespace=default transformations=1 +════════════════════════════════════════════════════════════════════════════════ + +── SUMMARY ───────────────────────────────────────────────────────────────────── + SEL-005 Selection pushed into join input ...................................×1 + +── TREES ─────────────────────────────────────────────────────────────────────── + +── + before: σ city = "London" ((Customers) ⋈ (Orders)) + after: (σ city = "London" (Customers)) ⋈ (Orders) + +── DETAIL ────────────────────────────────────────────────────────────────────── + +── (1 transformation) + [SEL-005] selection pushed into left input of natural join @ :1:9 +════════════════════════════════════════════════════════════════════════════════ +``` + +[Pipelines](../pipelines.md#optimise-a-script-and-review-the-change) compares a script +with its rewrite. diff --git a/docs/command-examples/provenance.md b/docs/command-examples/provenance.md new file mode 100644 index 0000000..8046b59 --- /dev/null +++ b/docs/command-examples/provenance.md @@ -0,0 +1,54 @@ +## Examples + +The default semiring, `counting`, says in how many ways each row is derived. Two London +customers each derive the row `London`: + +```shell +relix provenance -e 'PROJECT city (Customers)' +``` +``` + +------ ------ +city [prov] +------ ------ +London 2 +Leeds 1 +York 1 +------ ------ +(3 tuples; provenance semiring: counting) +``` + +The `lineage` semiring names the source rows each result row came from: + +```shell +relix provenance --semiring=lineage -e 'PROJECT city (SELECT amount > 100 (Customers JOIN Orders))' +``` +``` + +------ ------------------------------------------- +city [prov] +------ ------------------------------------------- +London Customers#1·Orders#1 + Customers#3·Orders#4 +------ ------------------------------------------- +(1 tuple; provenance semiring: lineage) +``` + +In a pipe, or with `-o json`, the rows and their annotations are one JSON document: + +```shell +relix provenance -e 'PROJECT city (Customers)' -o json +``` +``` +{ + "query": "", + "semiring": "counting", + "schema": ["city"], + "tuples": [ + {"row": {"city": "London"}, "provenance": "2"}, + {"row": {"city": "Leeds"}, "provenance": "1"}, + {"row": {"city": "York"}, "provenance": "1"} + ] +} +``` + +The engine's [provenance page](../../provenance.md) explains each semiring. diff --git a/docs/command-examples/relix.md b/docs/command-examples/relix.md new file mode 100644 index 0000000..1dd61b9 --- /dev/null +++ b/docs/command-examples/relix.md @@ -0,0 +1,57 @@ +## Examples + +With no command, `relix` runs scripts, as [`relix run`](run.md) does: + +```shell +relix -e 'PROJECT name, city (Customers)' -o tsv +``` +``` +name city +Alice London +Bob Leeds +Carol London +Dan York +``` + +Global options go before or after the command's name: + +```shell +relix -C reports catalog ls -o tsv --no-header +``` +``` +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/reports/.relix/catalog/10-sample.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +## Environment + +| Variable | Effect | +|---|---| +| `RELIX_OUTPUT` | the output format when `-o` is not given | +| `RELIX_PROFILE` | the profile when `-P` is not given | +| `RELIX_NOW` | the instant `NOW()` returns when `--now` is not given | +| `RELIX_CATALOG_PATH` | catalog files to load, separated as in `PATH`, before the `--catalog` files | +| `RELIX_TRUST_ALL` | `1` or `true` trusts every project's `.relix/` directory | +| `RELIX_DRIVERS` | the directory JDBC drivers are installed in and loaded from, instead of `~/.relix/drivers` | +| `NO_COLOR` | when set, a table on a terminal is not coloured unless `--color=always` | +| `PAGER` | the program [`relix doc`](doc.md) shows a page through on a terminal; `cat` or empty shows it directly | + +A `${NAME}` placeholder in a declaration may also be resolved from the environment; see +[Secrets and profiles](../secrets.md). + +## Files + +| Path | What it holds | +|---|---| +| `~/.relix/catalog/*.relix` | your own catalog, which every directory under your home sees | +| `~/.relix/profiles.json` | your own profiles | +| `~/.relix/trusted` | the projects you have trusted, with a hash of each one's files | +| `~/.relix/drivers/` | installed JDBC drivers | +| `~/.relix/connectors/` | installed connector plugins | +| `~/.relix/cache/files/` | files fetched over `https` by a file connection | +| `.relix/catalog/*.relix`, `.relix/relixrc`, `.relix/profiles.json`, `.relix/root` | a project's catalog, defaults, profiles, and the marker that ends the walk up; see [The catalog](../catalog.md) | diff --git a/docs/command-examples/run.md b/docs/command-examples/run.md new file mode 100644 index 0000000..5da0dd2 --- /dev/null +++ b/docs/command-examples/run.md @@ -0,0 +1,40 @@ +## Examples + +`relix run` is what `relix` does with no command, so these two are the same: + +```shell +relix run -e 'GROUP status, SUM(amount) -> total (Orders)' +``` +``` +── ────────────────────────────────────────────────────────────── + status total + ───────── ───── + completed 400 + pending 50 +(2 rows) +``` + +```shell +relix -e 'GROUP status, SUM(amount) -> total (Orders)' -o csv +``` +``` +status,total +completed,400 +pending,50 +``` + +Each script file is a run of its own, with its own declarations: + +```shell +relix run scripts/top-customers.relix -a city=Leeds scripts/by-city.relix -o tsv +``` +``` +name total +Alice 200 +Carol 200 +name +Bob +``` + +[Binding inputs](../inputs.md), [Output formats](../output.md) and +[Pipelines](../pipelines.md) cover the options in use. diff --git a/docs/command-examples/trace.md b/docs/command-examples/trace.md new file mode 100644 index 0000000..f453d2c --- /dev/null +++ b/docs/command-examples/trace.md @@ -0,0 +1,24 @@ +## Examples + +Each event ends with how long it took, which varies from run to run, so here `cut` +leaves the timings off: + +```shell +relix trace -e 'SELECT amount > 100 (Orders)' | cut -d'(' -f1 +``` +``` +[EXECUTE ] orders SCAN scanned 4 rows +[EXECUTE ] ROWS query delivered 2 rows +``` + +`relix run --trace` writes the same feed to standard error while the rows go to standard +output, and `--trace=FILE` writes it to a file: + +```shell +relix --trace=trace.txt -e 'SELECT amount > 100 (Orders)' -o tsv +``` +``` +order_id customer_id amount status placed +1 1 120 completed 2026-09-02 +4 3 200 completed 2026-10-01 +``` diff --git a/docs/command-examples/version.md b/docs/command-examples/version.md new file mode 100644 index 0000000..e181891 --- /dev/null +++ b/docs/command-examples/version.md @@ -0,0 +1,11 @@ +## Examples + +`relix version`, like `relix --version`, prints one line: the version of the command, +then the version of the Relix engine it runs, as + +```text +relix 1.0.0 (engine 1.0.0) +``` + +The two are numbered independently: a release of the command names the engine release +it was built on. diff --git a/docs/design.md b/docs/design.md index 6ae03dd..1e84099 100644 --- a/docs/design.md +++ b/docs/design.md @@ -407,6 +407,13 @@ version of it. `.relix/` tree and fixture files, comparing stdout and the exit status — so a page cannot be published with an example the command does not honour. +The pages live in `docs/guide/command-line`, laid out as the engine's own manuals are +(a `README.md` index of chapters and pages), with `index.json` as the machine-readable +index. The site renders the section inside the engine's programming guide, at +`guide/command-line/`, so a link out of it is written as if it sat in the engine's +`docs/guide`. Until releases attach the archive, the site pins a commit of this +repository and reads the same tree from it. + ### 7.5 Tests Each command end-to-end through `Main` with captured streams; catalog discovery and diff --git a/docs/guide/command-line/README.md b/docs/guide/command-line/README.md new file mode 100644 index 0000000..1aa3479 --- /dev/null +++ b/docs/guide/command-line/README.md @@ -0,0 +1,65 @@ +# The relix command line + +Using Relix from a shell: the `relix` command, which runs scripts as one stage of a Unix +pipeline, and the commands beside it that check, explain, optimise and format them. + +`relix` writes rows to standard output and everything else to standard error. A script +comes from `-e`, a file or standard input; its data from files, standard input, or the +tables a project declares in its `.relix/` directory; and its rows leave as a table on a +terminal, or tab-separated values, CSV, NDJSON, JSON or Markdown for the next program. + +Every example in this section is run on each build of the command, against a small +project, and the block under it is what it printed. The command reference is generated +from the same model as `relix --help` and the man pages. + +## The command line + +Installing it, and the shape every command shares; then data in, rows out. + +| Page | What it covers | +|---|---| +| [The relix command](getting-started.md) | Installing; a first query; where the script comes from; the commands; diagnostics and exit status | +| [Binding inputs](inputs.md) | `-i` and its formats; standard input; inferred and given headings; inputs that never end; shadowing the catalog; query parameters with `-a` | +| [Output formats](output.md) | `table`, `tsv`, `csv`, `ndjson`, `json` and `markdown`; NULLs and headers; scripts with several queries; `--fail-empty` and `--fail-rows`; messages; closed pipes and interrupts | + +## Projects on the command line + +What a directory gives the commands run in it. + +| Page | What it covers | +|---|---| +| [The catalog](catalog.md) | `.relix/` directories and how they are found; trusting a project; a nearer declaration wins; inspecting the catalog; `--catalog`; `relixrc` | +| [Secrets and profiles](secrets.md) | `${NAME}` placeholders; `profiles.json` and `-P`; the environment and `-D`; private profile files; what is never printed | + +## Command-line recipes + +| Page | What it covers | +|---|---| +| [Pipelines](pipelines.md) | With `jq`, `awk`, `sort` and `head`; with `find` and `xargs`; scripts in here-documents; a data check in CI; optimising a script and reviewing the change | + +## Command reference + +One page per command: what it does, its usage, arguments and options, its exit status, +and examples. + +| Page | What it covers | +|---|---| +| [`relix`](commands/relix.md) | The global options every command takes; the exit statuses; the environment variables and files | +| [`relix run`](commands/run.md) | Runs scripts and writes their queries' rows | +| [`relix check`](commands/check.md) | Analyses scripts without running them, and reports their problems | +| [`relix explain`](commands/explain.md) | Prints each query's physical plan | +| [`relix optimize`](commands/optimize.md) | Prints the script with each query rewritten by the optimiser | +| [`relix trace`](commands/trace.md) | Runs each query and prints what the engine rewrote, planned and ran | +| [`relix bundle`](commands/bundle.md) | Prints each script's JSON bundle of diagnostics, plans and events | +| [`relix ir`](commands/ir.md) | Prints the symbols, views and queries the analyser made of a script | +| [`relix provenance`](commands/provenance.md) | Runs each query and annotates each row with where it came from | +| [`relix fmt`](commands/fmt.md) | Prints scripts canonically, keeping their comments | +| [`relix catalog`](commands/catalog.md) | Lists the catalog's files, relations, columns and declarations; trusts a project | +| [`relix drivers`](commands/drivers.md) | Lists and installs JDBC drivers | +| [`relix connectors`](commands/connectors.md) | Lists and installs connector plugins | +| [`relix doc`](commands/doc.md) | Shows a page of the language reference | +| [`relix help`](commands/help.md) | Prints a command's usage | +| [`relix version`](commands/version.md) | Prints the versions of the command and of the engine | +| [`relix completion`](commands/completion.md) | Prints a completion script for bash, zsh, fish or PowerShell | + +A machine-readable index of these pages is `index.json`, beside this one. diff --git a/docs/guide/command-line/catalog.md b/docs/guide/command-line/catalog.md new file mode 100644 index 0000000..1078175 --- /dev/null +++ b/docs/guide/command-line/catalog.md @@ -0,0 +1,328 @@ +# The catalog + +A project's tables, connections and views are declared once, in its `.relix/` directory, +and every command run inside the project can name them. `relix` finds those directories by +walking up from where it starts, the way `git` finds `.git`, so the same command works +from any subdirectory. This page covers how the directories are laid out and found, how +a nearer declaration replaces a farther one, how to see what a run will load, and the +trust a project's directory needs before anything in it is loaded. + +## The layout + +A `.relix/` directory holds: + +| Path | What it holds | +|---|---| +| `catalog/*.relix` | the declarations: `source`, `connection`, views (`Name := { … };`), `def`, `import` | +| `relixrc` | defaults for the commands run below it: the output format and the profile | +| `profiles.json` | named sets of `${VAR}` values; see [Secrets and profiles](secrets.md) | +| `root` | a marker that stops the walk up at this directory | + +The shop the examples use has one: + +```shell +find .relix -type f | sort +``` +``` +.relix/catalog/00-sources.relix +.relix/catalog/10-views.relix +.relix/catalog/20-warehouse.relix +.relix/profiles.json +``` + +```shell +cat .relix/catalog/00-sources.relix +``` +``` +-- The shop's data. A relative path resolves against this file's directory. +source Customers from csv("../../data/customers.csv") { + header: true, + schema: { customer_id: NUMBER, name: STRING, city: STRING } +}; + +source Orders from csv("../../data/orders.csv") { + header: true, + schema: { order_id: NUMBER, customer_id: NUMBER, amount: NUMBER, status: STRING, placed: DATE } +}; +``` + +```shell +cat .relix/catalog/10-views.relix +``` +``` +Completed := { SELECT status = 'completed' (Orders) }; + +Spend := { GROUP customer_id, SUM(amount) -> total (Completed) }; + +BigSpenders := { PROJECT name, total (Customers JOIN (SELECT total >= 200 (Spend))) }; +``` + +A catalog file holds declarations only; a `query` statement in one is an error. A +relative path in a catalog file resolves against the file's own directory, not the +directory the command runs in, so `../../data/orders.csv` above means `~/shop/data` +from wherever in the project `relix` is started. + +Declaring costs nothing at run time: no file is read and no connection opened until a +query reaches the name. A large catalog costs a parse and an analysis, not a connection. + +## Finding the directories + +A run's catalog comes from these directories, loaded **outermost first**: + +1. `~/.relix`, your own, which every directory under your home sees. +2. Each `.relix/` from the outermost directory down to the one the command starts in, + walking up from it and stopping at your home directory. A `.relix/` that holds a file + named `root` ends the walk there, so the projects of a monorepo do not inherit each + other's declarations. Started outside your home directory, `relix` reads only that + directory's own `.relix/`. +3. The files named by `$RELIX_CATALOG_PATH`, separated as in `PATH`, then those named + with `--catalog FILE`, in order. + +Within a directory, `catalog/*.relix` load in the lexical order of their names, so +`00-sources.relix` comes before `10-views.relix`, as in `/etc/*.d`. + +`-N` (`--no-catalog`) leaves out every `.relix/` directory's catalog and `relixrc`; +`--catalog` files still load. `-C DIR` starts the walk from `DIR`. + +## A project must be trusted + +Loading code because of the directory you are standing in is a risk: a cloned +repository's `.relix/` could declare a source that sends a `${TOKEN}` from your +environment to a server of its author's choosing. So a project's `.relix/` is loaded only +once you have trusted it, as `direnv` does with an `.envrc`. Until then, it is skipped, +with a warning saying how to trust it, and a run that needs one of its names fails with +exit status 5: + +```shell +relix -e 'Customers' 2>&1; echo "exit $?" +``` +``` +relix: warning: skipping ~/shop/.relix, which is not trusted; to load it: relix catalog trust ~/shop +-e:1:1: error: Undefined relation: 'Customers' +relix: Customers is declared in ~/shop/.relix/catalog/00-sources.relix, which is not trusted; to load it: relix catalog trust ~/shop +exit 5 +``` + +`relix catalog trust` trusts the working directory's project, or the one it names. It +records a hash of every file under the `.relix/` directory in `~/.relix/trusted`: + +```shell +relix catalog trust +relix -e 'PROJECT name (Customers)' -o tsv +``` +``` +name +Alice +Bob +Carol +Dan +``` + +Changing, adding or removing any file under a trusted `.relix/` — a catalog file, the +`relixrc`, the `profiles.json` — makes it untrusted again until it is trusted again, so a +`git pull` cannot change what runs without you seeing it. `--list` shows each trusted +project and whether it has changed since: + +```shell +echo "Pending := { SELECT status = 'pending' (Orders) };" > .relix/catalog/30-pending.relix +relix catalog trust --list -o tsv +``` +``` +directory state +~/shop changed +``` + +```shell +relix catalog trust +relix catalog trust --list -o tsv +``` +``` +directory state +~/shop trusted +``` + +`--revoke DIR` removes a project from the record. + +Some things are trusted without asking, because you chose them: `~/.relix`, a script +named on the command line or given with `-e` or on standard input, and `--catalog` and +`$RELIX_CATALOG_PATH` files. `RELIX_TRUST_ALL=1` trusts every directory, for a CI +container whose checkout is the only thing in it. + +## A nearer declaration wins + +When two directories declare the same name, the **nearer** one's declaration replaces the +farther one's. That is an overlay, not an error, and it is what lets a sub-project change +one table while keeping everything else above it. `~/shop/reports` declares its own +`Orders`, a fixed sample to write reports against: + +```shell +cat reports/.relix/catalog/10-sample.relix +``` +``` +-- Reports are written against a fixed sample rather than the live orders. +source Orders from csv("../../sample-orders.csv") { + header: true, + schema: { order_id: NUMBER, customer_id: NUMBER, amount: NUMBER, status: STRING, placed: DATE } +}; +``` + +Every view the shop declares reads whichever `Orders` is nearest, so `BigSpenders` from +inside `reports` is computed over the sample: + +```shell +relix catalog trust reports +``` + +```shell +relix -e 'BigSpenders' +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 200 + Carol 200 +(2 rows) +``` + +```shell +relix -C reports -e 'BigSpenders' +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 500 + Dan 250 +(2 rows) +``` + +Each file is parsed first, the nearest declaration of every name kept, and only then are +the survivors defined together, so the order of the files decides which declaration +wins, never which can see which. A name is replaced only by a declaration of the same +kind: a relation by a relation, a function by a function. + +A file in an untrusted directory neither declares nor replaces anything. + +## Inspecting the catalog + +`relix catalog` answers what a run would load, without running anything: + +| Command | What it lists | +|---|---| +| `relix catalog files` | every catalog file, in load order, whether it is trusted, and the names it declares | +| `relix catalog ls` | every relation: its name, kind, number of columns, and the file whose declaration won | +| `relix catalog schema NAME` | the columns of a relation and their types | +| `relix catalog where NAME` | every declaration of a name, nearest first: the one that won, then those it replaced | + +Each writes rows like any query, as a table on a terminal and in any `-o` format: + +```shell +relix catalog files -o tsv +``` +``` +file trusted declares +~/.relix/catalog/00-regions.relix true Regions +~/shop/.relix/catalog/00-sources.relix true Customers Orders +~/shop/.relix/catalog/10-views.relix true Completed Spend BigSpenders +~/shop/.relix/catalog/20-warehouse.relix true warehouse Stock +~/shop/.relix/catalog/30-pending.relix true Pending +``` + +```shell +relix catalog ls -o tsv +``` +``` +name kind arity file +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/.relix/catalog/00-sources.relix +Pending QR 5 ~/shop/.relix/catalog/30-pending.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +The `kind` is the engine's: `SRC` a source, `INL` an inline table, `QR` a view, `DB` a +table reached through a connection. + +```shell +relix catalog schema Orders +``` +``` +── Orders ────────────────────────────────────────────────────────────────────── + column type + ─────────── ──── + order_id N + customer_id N + amount N + status S + placed D +(5 rows) +``` + +The type codes are the engine's: `N` a number, `S` a string, `B` a boolean, `D` a date, +`T` a time, `TS` a timestamp, `DUR` a duration, `?` any type. + +```shell +relix -C reports catalog where Orders -o tsv +``` +``` +file line kind state +~/shop/reports/.relix/catalog/10-sample.relix 2 source wins +~/shop/.relix/catalog/00-sources.relix 7 source shadowed +``` + +## Catalog files from elsewhere + +`--catalog FILE` adds a file to the catalog for one run, nearer than any `.relix/` +directory's. `$RELIX_CATALOG_PATH` names files the same way for every run. Both are +trusted, since you named them: + +```shell +cat > ~/rates.relix <<'EOF' +Rates := [ +| status | fee | +|-----------|-----| +| completed | 2 | +| pending | 0 | +]; +EOF +relix --catalog ~/rates.relix -e 'PROJECT order_id, fee (Orders JOIN Rates)' -o tsv +``` +``` +order_id fee +1 2 +2 2 +3 0 +4 2 +``` + +## Defaults: relixrc + +A `relixrc` file sets defaults for the commands run below it, as `key = value` lines; a +line starting `#` is a comment. The keys are: + +| Key | Default it sets | Beaten by | +|---|---|---| +| `output` | the output format | `-o`, `$RELIX_OUTPUT` | +| `profile` | the profile `${VAR}` placeholders resolve from | `-P`, `$RELIX_PROFILE` | + +The nearest file's value for a key wins. A key the command does not know, or a line that +is not `key = value`, is reported as a warning and ignored, so a file written for a later +version still works. A `relixrc` is part of its `.relix/` directory, so it is read only +once the directory is trusted: + +```shell +printf 'output = csv\n' > .relix/relixrc +relix catalog trust +relix -e 'PROJECT name (Customers)' +``` +``` +name +Alice +Bob +Carol +Dan +``` diff --git a/docs/guide/command-line/commands/bundle.md b/docs/guide/command-line/commands/bundle.md new file mode 100644 index 0000000..81ef74d --- /dev/null +++ b/docs/guide/command-line/commands/bundle.md @@ -0,0 +1,38 @@ +# relix bundle + +Prints each script's JSON bundle: diagnostics and, per query, the logical plan, the rewrites, the physical plan and the planner's events. A script that does not analyse still has one, holding its diagnostics. + +## Usage + +```text +relix bundle [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +A bundle is one JSON document per script, for a tool such as a query playground: + +```shell +relix bundle -e 'SELECT amount > 100 (Orders)' | jq -c 'keys' +``` +``` +["diagnostics","namespace","queries"] +``` diff --git a/docs/guide/command-line/commands/catalog.md b/docs/guide/command-line/commands/catalog.md new file mode 100644 index 0000000..e8ba414 --- /dev/null +++ b/docs/guide/command-line/commands/catalog.md @@ -0,0 +1,195 @@ +# relix catalog + +Inspects the catalog: the declarations a run takes from the .relix/ directories from ~/.relix down to the working directory, and from --catalog files. A project's .relix/ is loaded only once it is trusted (relix catalog trust). + +## Usage + +```text +relix catalog [OPTIONS] [COMMAND] +``` + +It also takes the [global options](relix.md#global-options). + +## relix catalog files + +Lists every catalog file, in the order they load, with the names each declares and whether it is trusted. An untrusted file is listed but not loaded. + +### Usage + +```text +relix catalog files [OPTIONS] +``` + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## relix catalog ls + +Lists the relations the catalog declares: name, kind, arity and the file whose declaration won. kind is the engine's: SRC a source, INL an inline table, QR a view, DB a connection's table. + +### Usage + +```text +relix catalog ls [OPTIONS] +``` + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## relix catalog schema + +Lists the columns of the relation NAME and their types, in order. + +### Usage + +```text +relix catalog schema [OPTIONS] NAME +``` + +### Arguments + +| Argument | Description | +|---|---| +| `NAME` | A relation the catalog declares. | + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +### Exit status + +| Status | Meaning | +|---|---| +| 0 | success | +| 3 | the catalog declares no relation NAME, or does not analyse | + +## relix catalog where + +Lists every declaration of NAME, nearest first: the one that won, then the ones it shadowed. A declaration in an untrusted directory is listed as untrusted. + +### Usage + +```text +relix catalog where [OPTIONS] NAME +``` + +### Arguments + +| Argument | Description | +|---|---| +| `NAME` | A name the catalog declares. | + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +### Exit status + +| Status | Meaning | +|---|---| +| 0 | success | +| 3 | the catalog declares no NAME | + +## relix catalog trust + +Trusts the .relix/ directory of DIR (default: the working directory), as its files are now: a project's catalog, relixrc and profiles.json are loaded only once it is trusted, and changing, adding or removing a file there means trusting it again. ~/.relix, --catalog files and the script itself are always trusted; RELIX_TRUST_ALL=1 trusts everything, for a CI container. + +### Usage + +```text +relix catalog trust [OPTIONS] [DIR] +``` + +### Arguments + +| Argument | Description | +|---|---| +| `DIR` | The project to trust, or its .relix/ directory. | + +### Options + +| Option | Description | +|---|---| +| `--list` | List the trusted directories and whether each has changed since. | +| `--revoke=DIR` | Stop trusting DIR. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +```shell +relix catalog ls -o tsv +``` +``` +name kind arity file +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/.relix/catalog/00-sources.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +```shell +relix catalog schema Customers -o tsv +``` +``` +column type +customer_id N +name S +city S +``` + +```shell +relix catalog where Orders -o tsv +``` +``` +file line kind state +~/shop/.relix/catalog/00-sources.relix 7 source wins +``` + +```shell +relix catalog trust --list -o tsv +``` +``` +directory state +~/shop trusted +~/shop/reports trusted +``` + +[The catalog](../catalog.md) covers discovery, the overlay and trust. diff --git a/docs/guide/command-line/commands/check.md b/docs/guide/command-line/commands/check.md new file mode 100644 index 0000000..253c6ca --- /dev/null +++ b/docs/guide/command-line/commands/check.md @@ -0,0 +1,70 @@ +# relix check + +Analyses scripts without running them, and reports what is wrong with them on standard error as FILE:LINE:COL: severity: message. + +```text +find . -name '*.relix' -print0 | xargs -0 relix check +``` + +## Usage + +```text +relix check [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--json` | Write the diagnostics to standard output as one JSON document instead. | + +It also takes the [global options](relix.md#global-options). + +## Exit status + +| Status | Meaning | +|---|---| +| 0 | every script analyses | +| 3 | a script does not parse or analyse | + +## Examples + +A script that analyses prints nothing, and exits 0: + +```shell +relix check scripts/top-customers.relix +``` + +A problem is printed on standard error, and the status is 3: + +```shell +relix check scripts/broken.relix 2>&1; echo "exit $?" +``` +``` +scripts/broken.relix:1:30: error: Undefined relation: 'Ordrs' +exit 3 +``` + +`--json` writes the diagnostics to standard output instead, as one document for a tool to +read: + +```shell +relix check --json scripts/broken.relix; echo "exit $?" +``` +``` +{"diagnostics":[{"file":"scripts/broken.relix","line":1,"column":30,"severity":"error","message":"Undefined relation: 'Ordrs'"}]} +exit 3 +``` + +[Pipelines](../pipelines.md#lint-every-script) checks every script in a tree. diff --git a/docs/guide/command-line/commands/completion.md b/docs/guide/command-line/commands/completion.md new file mode 100644 index 0000000..023a3f9 --- /dev/null +++ b/docs/guide/command-line/commands/completion.md @@ -0,0 +1,39 @@ +# relix completion + +Prints a completion script for SHELL: bash, zsh, fish or powershell. + +```text +bash: source <(relix completion bash) +zsh: source <(relix completion zsh) +fish: relix completion fish > ~/.config/fish/completions/relix.fish +powershell: relix completion powershell | Out-String | Invoke-Expression +``` + +## Usage + +```text +relix completion [OPTIONS] SHELL +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SHELL` | bash, zsh, fish or powershell. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +The release archives carry each shell's script in `completions/`, and Homebrew installs +them. To load one by hand, add the line for your shell to its startup file: + +```text +bash source <(relix completion bash) +zsh source <(relix completion zsh) +fish relix completion fish > ~/.config/fish/completions/relix.fish +PowerShell relix completion powershell | Out-String | Invoke-Expression +``` + +The zsh script is also a completion function, so it can be saved as `_relix` in a +directory on `$fpath` instead. diff --git a/docs/guide/command-line/commands/connectors.md b/docs/guide/command-line/commands/connectors.md new file mode 100644 index 0000000..a6b7fc7 --- /dev/null +++ b/docs/guide/command-line/commands/connectors.md @@ -0,0 +1,82 @@ +# relix connectors + +Lists and installs connector plugins, such as mongodb. A source of a type no installed connector reads needs one, installed here ahead of a run, or fetched during one with --allow-download. + +## Usage + +```text +relix connectors [OPTIONS] [COMMAND] +``` + +It also takes the [global options](relix.md#global-options). + +## relix connectors ls + +Lists the connectors installed: those shipped with relix and the plugins installed in ~/.relix/connectors. --available adds the ones the hosted catalog offers, which it fetches. + +### Usage + +```text +relix connectors ls [OPTIONS] +``` + +### Options + +| Option | Description | +|---|---| +| `--available` | Also list the connectors that can be installed, from the hosted catalog. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## relix connectors install + +Downloads the connector plugin TYPE, checks its checksums, and installs it. + +### Usage + +```text +relix connectors install [OPTIONS] TYPE +``` + +### Arguments + +| Argument | Description | +|---|---| +| `TYPE` | A connector type, such as mongodb. | + +It also takes the [global options](relix.md#global-options). + +### Exit status + +| Status | Meaning | +|---|---| +| 0 | installed, or already installed | +| 2 | the catalog has no connector of that type | +| 5 | the download failed | + +## Examples + +The connectors that ship with `relix`, and any plugins installed in `~/.relix/connectors`: + +```shell +relix connectors ls +``` +``` +── connectors ────────────────────────────────────────────────────────────────── + type installed + ────── ───────── + clf true + csv true + gedcom true + log true +(4 rows) +``` + +`relix connectors ls --available` adds the connectors that can be installed, which it +fetches from the hosted catalog, and `relix connectors install TYPE` installs one. A +source whose type no installed connector reads — `mongodb`, say — fails with exit status +5, unless `--allow-download` lets the run fetch the connector. diff --git a/docs/guide/command-line/commands/doc.md b/docs/guide/command-line/commands/doc.md new file mode 100644 index 0000000..5108066 --- /dev/null +++ b/docs/guide/command-line/commands/doc.md @@ -0,0 +1,75 @@ +# relix doc + +Shows the reference page for TOPIC, found by glyph, keyword or name (σ, select, selection), through $PAGER on a terminal. With no TOPIC, lists the pages. + +## Usage + +```text +relix doc [OPTIONS] [TOPIC] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `TOPIC` | A glyph, keyword or name, such as σ, select, join or Len. | + +## Options + +| Option | Description | +|---|---| +| `--function` | Look TOPIC up among the functions only, or list only them. | +| `--markdown` | Print the page's markdown as it is, not rendered. | +| `--no-pager` | Write the page to standard output even on a terminal. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## Exit status + +| Status | Meaning | +|---|---| +| 0 | success | +| 2 | no page has that topic | + +## Examples + +A page is found by its glyph, its keyword or its name: + +```shell +relix doc --markdown σ | head -6 +``` +``` +# Name: Selection (σ / SELECT) + +# Syntax: +σ (Relation) +SELECT (Relation) +``` + +With no topic, `relix doc` lists the pages, as rows: + +```shell +relix doc -o tsv | head -4 +``` +``` +topic category title summary +getting-started guide Getting started Your first script — declare data, name a result, run it +glossary guide Glossary The vocabulary — relation, tuple, bag vs set, materialisation, pushdown +σ operator Selection Filter rows by a condition +``` + +`--function` lists, or looks up, the functions only: + +```shell +relix doc --function -o tsv | grep -i '^round' +``` +``` +Round function Round Round to nearest +``` + +On a terminal a page is shown through `$PAGER` (`less` when it is not set); `--no-pager` +writes it straight to standard output. diff --git a/docs/guide/command-line/commands/drivers.md b/docs/guide/command-line/commands/drivers.md new file mode 100644 index 0000000..de2182d --- /dev/null +++ b/docs/guide/command-line/commands/drivers.md @@ -0,0 +1,79 @@ +# relix drivers + +Lists and installs JDBC drivers. A connection to a database needs its driver, installed here ahead of a run, or fetched during one with --allow-download. + +## Usage + +```text +relix drivers [OPTIONS] [COMMAND] +``` + +It also takes the [global options](relix.md#global-options). + +## relix drivers ls + +Lists the drivers relix can install: name, the JDBC scheme each serves, and whether one is installed. + +### Usage + +```text +relix drivers ls [OPTIONS] +``` + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## relix drivers install + +Downloads the driver NAME, checks its checksum, and installs it. + +### Usage + +```text +relix drivers install [OPTIONS] NAME +``` + +### Arguments + +| Argument | Description | +|---|---| +| `NAME` | A driver relix drivers ls lists, such as postgresql. | + +It also takes the [global options](relix.md#global-options). + +### Exit status + +| Status | Meaning | +|---|---| +| 0 | installed, or already installed | +| 2 | no driver has that name | +| 5 | the download failed | + +## Examples + +```shell +relix drivers ls +``` +``` +── drivers ───────────────────────────────────────────────────────────────────── + name scheme installed + ────────── ──────────────── ───────── + postgresql jdbc:postgresql: false + mysql jdbc:mysql: false +(2 rows) +``` + +`relix drivers install NAME` downloads one of those drivers, checks its checksum, and +installs it in `~/.relix/drivers`, or in `$RELIX_DRIVERS` when that is set. A run that +needs a driver that is not installed fails with exit status 5, unless `--allow-download` +lets it fetch the driver itself. The H2 driver is part of +`relix`, and needs no installing. A driver `relix drivers ls` does not list is installed +by copying its jar into the driver directory. diff --git a/docs/guide/command-line/commands/explain.md b/docs/guide/command-line/commands/explain.md new file mode 100644 index 0000000..4520817 --- /dev/null +++ b/docs/guide/command-line/commands/explain.md @@ -0,0 +1,50 @@ +# relix explain + +Prints the physical plan of each query: how the engine would run it. + +## Usage + +```text +relix explain [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--json` | Write one JSON object per query, {"script", "query", "plan"}, one per line. | +| `--query=NAME` | Explain only the query NAME. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +```shell +relix explain -e 'GROUP city, COUNT(*) -> n (Customers JOIN Orders)' +``` +``` +Aggregate +└─ Join NATURAL/HASH build=RIGHT + ├─ Scan Customers + └─ Scan Orders +``` + +`--json` writes each query's plan as a JSON object instead: + +```shell +relix explain --json -e 'SELECT amount > 100 (Orders)' | jq -c 'keys' +``` +``` +["plan","query","script"] +``` diff --git a/docs/guide/command-line/commands/fmt.md b/docs/guide/command-line/commands/fmt.md new file mode 100644 index 0000000..fdfed97 --- /dev/null +++ b/docs/guide/command-line/commands/fmt.md @@ -0,0 +1,96 @@ +# relix fmt + +Prints scripts canonically, keeping their comments. With no SCRIPT, formats standard input. + +## Usage + +```text +relix fmt [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files; - is standard input, which is also read when none is named. | + +## Options + +| Option | Description | +|---|---| +| `-w` | Rewrite each file that changes in place, instead of printing it. | +| `--check` | Change nothing; print the name of each file that would change, and exit 1 if any would. | +| `--glyphs` | Write operators as glyphs: σ, ⋈, ∧ (the default). | +| `--keywords` | Write operators as ASCII keywords: SELECT, JOIN, AND. | + +It also takes the [global options](relix.md#global-options). + +## Exit status + +| Status | Meaning | +|---|---| +| 0 | success; with --check, no file would change | +| 1 | with --check, a file would change | +| 3 | a script does not parse | + +## Examples + +```shell +cat scripts/messy.relix +``` +``` +-- Orders worth a look +Large:={select amount>100(Orders)}; +query {project order_id,amount (Large)}; +``` + +`relix fmt` prints it canonically — one statement per line, spaced, each operator in one +spelling — keeping its comments: + +```shell +relix fmt scripts/messy.relix +``` +``` +-- Orders worth a look +Large := { σ amount > 100 (Orders) }; +query { π order_id, amount (Large) }; +``` + +`--keywords` writes the operators as ASCII keywords instead of glyphs: + +```shell +relix fmt --keywords scripts/messy.relix +``` +``` +-- Orders worth a look +Large := { SELECT amount > 100 (Orders) }; +query { PROJECT order_id, amount (Large) }; +``` + +`--check` changes nothing and names each file that would change, exiting 1 if there is +one; `-w` rewrites each such file in place: + +```shell +relix fmt --check --keywords scripts/messy.relix scripts/by-city.relix; echo "exit $?" +``` +``` +scripts/messy.relix +exit 1 +``` + +```shell +relix fmt -w --keywords scripts/messy.relix +relix fmt --check --keywords scripts/messy.relix; echo "exit $?" +``` +``` +exit 0 +``` + +With no file, `relix fmt` formats standard input to standard output: + +```shell +echo 'query {select amount>100(Orders)};' | relix fmt +``` +``` +query { σ amount > 100 (Orders) }; +``` diff --git a/docs/guide/command-line/commands/help.md b/docs/guide/command-line/commands/help.md new file mode 100644 index 0000000..9f4359d --- /dev/null +++ b/docs/guide/command-line/commands/help.md @@ -0,0 +1,30 @@ +# relix help + +When no COMMAND is given, the usage help for the main command is displayed. If a COMMAND is specified, the help for that command is shown. + +## Usage + +```text +relix help [OPTIONS] [COMMAND] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `COMMAND` | The COMMAND to display the usage help message for. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +`relix help COMMAND` prints what `relix COMMAND --help` prints: + +```shell +relix help version | head -3 +``` +``` +Usage: relix version [-hNqVv] [--allow-download] [--remote] [-C=DIR] + [--max-fixpoint-rounds=N] [--max-materialized-rows=N] + [--max-processed-rows=N] [--now=INSTANT] [-P=NAME] +``` diff --git a/docs/guide/command-line/commands/ir.md b/docs/guide/command-line/commands/ir.md new file mode 100644 index 0000000..968419d --- /dev/null +++ b/docs/guide/command-line/commands/ir.md @@ -0,0 +1,95 @@ +# relix ir + +Prints the IR report: the symbols, views and queries the analyser made of each script. + +## Usage + +```text +relix ir [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +```shell +relix ir -e 'Big := { SELECT amount > 100 (Orders) };' -e 'query Big;' +``` +``` +════════════════════════════════════════════════════════════════════════════════ +RELIX IR namespace=default +════════════════════════════════════════════════════════════════════════════════ + kind: SRC=source INL=inline SYS=system QR=view DB=database + FN=function TVF=table-fn LIT=truth-literal + type: N=number S=string B=boolean ?=any [x]=array {…}=struct + D=date T=time TS=timestamp DUR=duration + mat: [bag]=bag [set]=dedup-set [sort]=sorted (stream=no label) + +── SYMBOLS ───────────────────────────────────────────────────────────────────── + Big QR order_id:N customer_id:N amount:N status:S placed:D + BigSpenders QR name:S total:N + catalog SYS name:S kind:S arity:N schema_code:S + columns SYS relation:S column:S type:S ordinal:N distinct_count:N … + Completed QR order_id:N customer_id:N amount:N status:S placed:D + connections SYS name:S dialect:S + Customers SRC customer_id:N name:S city:S + cycles QR name:S + dependencies QR dependent:S depends_on:S + deps TVF (r:S)→relation + events SYS seq:N stage:S code:S description:S target:S rows:N el… + find TVF (col:S)→relation + funcs TVF (cat:S)→relation + functions SYS name:S category:S arity:N max_arity:N return_type:S pu… + impact TVF (r:S)→relation + keys SYS relation:S key:N ordinal:N column:S + Orders SRC order_id:N customer_id:N amount:N status:S placed:D + plan SYS query:S node_id:N parent_id:N ordinal:N op:S label:S … + Regions INL city:S region:S + relations SYS name:S kind:S namespace:S materialization:S row_count:N… + rules QR seq:N code:S description:S target:S + schema TVF (rel:S)→relation + Spend QR customer_id:N total:N + Stock SRC sku:S on_hand:N + unused QR name:S + version SYS component:S kind:S version:S + +── EXPRESSION TREES ──────────────────────────────────────────────────────────── +Big [QR] order_id:N customer_id:N amount:N status:S placed:D + σ amount > 100 + └─ Orders [SRC] + +BigSpenders [QR] name:S total:N + π name, total + └─ ⋈ + ├─ Customers [SRC] + └─ σ total ≥ 200 + └─ Spend [QR] + +Completed [QR] order_id:N customer_id:N amount:N status:S placed:D + σ status = "completed" + └─ Orders [SRC] + +Spend [QR] customer_id:N total:N + γ [customer_id] SUM(amount)→total [bag] + └─ Completed [QR] + +── ROOT QUERIES ──────────────────────────────────────────────────────────────── + query Big [QR] +════════════════════════════════════════════════════════════════════════════════ +``` diff --git a/docs/guide/command-line/commands/optimize.md b/docs/guide/command-line/commands/optimize.md new file mode 100644 index 0000000..8fbd381 --- /dev/null +++ b/docs/guide/command-line/commands/optimize.md @@ -0,0 +1,68 @@ +# relix optimize + +Prints the script with each query rewritten by the optimiser, so that it works as a filter: relix optimize slow.relix > fast.relix + +## Usage + +```text +relix optimize [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--report` | Print the rules that fired and each query before and after, instead. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +The rewritten script is printed in the same form the engine runs it in: + +```shell +relix optimize -e "query { SELECT city = 'London' (Customers JOIN Orders) };" +``` +``` +query { (σ city = "London" (Customers)) ⋈ (Orders) }; +``` + +`--report` prints the rules that fired, and each query before and after: + +```shell +relix optimize --report -e "query { SELECT city = 'London' (Customers JOIN Orders) };" +``` +``` +════════════════════════════════════════════════════════════════════════════════ +RELIX OPTIMIZER namespace=default transformations=1 +════════════════════════════════════════════════════════════════════════════════ + +── SUMMARY ───────────────────────────────────────────────────────────────────── + SEL-005 Selection pushed into join input ...................................×1 + +── TREES ─────────────────────────────────────────────────────────────────────── + +── + before: σ city = "London" ((Customers) ⋈ (Orders)) + after: (σ city = "London" (Customers)) ⋈ (Orders) + +── DETAIL ────────────────────────────────────────────────────────────────────── + +── (1 transformation) + [SEL-005] selection pushed into left input of natural join @ :1:9 +════════════════════════════════════════════════════════════════════════════════ +``` + +[Pipelines](../pipelines.md#optimise-a-script-and-review-the-change) compares a script +with its rewrite. diff --git a/docs/guide/command-line/commands/provenance.md b/docs/guide/command-line/commands/provenance.md new file mode 100644 index 0000000..2c4346d --- /dev/null +++ b/docs/guide/command-line/commands/provenance.md @@ -0,0 +1,86 @@ +# relix provenance + +Runs each query and prints its rows, each annotated with where it came from in a provenance semiring. + +## Usage + +```text +relix provenance [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--semiring=NAME` | The semiring: boolean, counting, tropical, security, lineage, cheapest-route, or one an installed library offers (default: counting). | +| `--weight=COLUMN` | The column a weighted closure takes each edge's weight from, such as --semiring tropical --weight cost for shortest paths. | +| `-o`, `--output=FORMAT` | table or json (default: table on a terminal, json in a pipe). | +| `--query=NAME` | Annotate only the query NAME. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +The default semiring, `counting`, says in how many ways each row is derived. Two London +customers each derive the row `London`: + +```shell +relix provenance -e 'PROJECT city (Customers)' +``` +``` + +------ ------ +city [prov] +------ ------ +London 2 +Leeds 1 +York 1 +------ ------ +(3 tuples; provenance semiring: counting) +``` + +The `lineage` semiring names the source rows each result row came from: + +```shell +relix provenance --semiring=lineage -e 'PROJECT city (SELECT amount > 100 (Customers JOIN Orders))' +``` +``` + +------ ------------------------------------------- +city [prov] +------ ------------------------------------------- +London Customers#1·Orders#1 + Customers#3·Orders#4 +------ ------------------------------------------- +(1 tuple; provenance semiring: lineage) +``` + +In a pipe, or with `-o json`, the rows and their annotations are one JSON document: + +```shell +relix provenance -e 'PROJECT city (Customers)' -o json +``` +``` +{ + "query": "", + "semiring": "counting", + "schema": ["city"], + "tuples": [ + {"row": {"city": "London"}, "provenance": "2"}, + {"row": {"city": "Leeds"}, "provenance": "1"}, + {"row": {"city": "York"}, "provenance": "1"} + ] +} +``` + +The engine's [provenance page](../../provenance.md) explains each semiring. diff --git a/docs/guide/command-line/commands/relix.md b/docs/guide/command-line/commands/relix.md new file mode 100644 index 0000000..948ac64 --- /dev/null +++ b/docs/guide/command-line/commands/relix.md @@ -0,0 +1,155 @@ +# relix + +Runs Relix relational-algebra scripts as a stage of a Unix pipeline. Rows go to standard output; everything else goes to standard error. + +## Usage + +```text +relix [OPTIONS] [SCRIPT...] [COMMAND] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Run options + +With no command, `relix` is `relix run`, and takes its options: + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | +| `--line-buffered` | Flush each row as it is written, for a reader that wants it at once. | +| `--query=NAME` | Print only the query NAME. A machine format (tsv, csv, ndjson, json) prints one query: the script's last, unless this names another. | +| `--all` | With ndjson, print every query, each row tagged "_query": NAME. | +| `--fail-empty` | Exit 1 when the queries printed return no rows. | +| `--fail-rows` | Exit 1 when the queries printed return any row. | +| `--trace=FILE` | Write the event feed (what was rewritten, planned and run) to standard error, or to FILE with --trace=FILE, while the rows go to standard output. | + +## Global options + +Every command takes these, before or after its name. + +| Option | Description | +|---|---| +| `-C DIR` | Act as if started in DIR: relative paths and .relix/ discovery start there. | +| `-P`, `--profile=NAME` | The profile ${VAR} placeholders resolve from (default: $RELIX_PROFILE). | +| `-D NAME=VALUE` | Set one ${VAR} for this run; it beats the profile and the environment. Not for secrets: it lands in shell history and ps. | +| `-a`, `--arg=NAME=VALUE` | Bind the query parameter $NAME to VALUE. Repeatable. The value is bound, never pasted into the script, so it cannot change the query. | +| `-N`, `--no-catalog` | Ignore the .relix/ directories' catalogs and relixrc; --catalog files still load. | +| `--catalog=FILE` | Add a catalog file, nearer than the .relix/ directories'. Repeatable. $RELIX_CATALOG_PATH adds files too, separated as in PATH. | +| `-q`, `--quiet` | Print errors only, not warnings. | +| `-v`, `--verbose` | Also print notices (drivers loaded); -vv adds timings. | +| `--now=INSTANT` | Pin NOW() to an ISO-8601 instant, such as 2026-07-22T12:00:00Z (default: $RELIX_NOW). | +| `--max-fixpoint-rounds=N` | Fail a recursive query that needs more than N rounds. | +| `--max-materialized-rows=N` | Fail a query in which one blocking operator buffers more than N rows. | +| `--max-processed-rows=N` | Fail a query that processes more than N rows in all. | +| `--timeout=DURATION` | Fail a query that runs longer than DURATION: 500ms, 30s, 5m, 1h, or ISO-8601 (PT30S). | +| `--sandbox=FILE` | Run under the sandbox FILE describes. | +| `--remote` | Permit http(s) file locations. | +| `--allow-download` | Permit fetching a missing JDBC driver or connector during the run. | +| `-h`, `--help` | Show this help message and exit. | +| `-V`, `--version` | Print version information and exit. | + +## Commands + +| Command | What it does | +|---|---| +| [`relix run`](run.md) | Runs scripts and writes their queries' rows to standard output. | +| [`relix check`](check.md) | Analyses scripts without running them, and reports what is wrong with them on standard error as FILE:LINE:COL: severity: message. | +| [`relix explain`](explain.md) | Prints the physical plan of each query: how the engine would run it. | +| [`relix optimize`](optimize.md) | Prints the script with each query rewritten by the optimiser, so that it works as a filter: relix optimize slow.relix > fast.relix. | +| [`relix trace`](trace.md) | Runs each query and prints its event feed, what the engine rewrote, planned and ran, discarding the rows. | +| [`relix bundle`](bundle.md) | Prints each script's JSON bundle: diagnostics and, per query, the logical plan, the rewrites, the physical plan and the planner's events. | +| [`relix ir`](ir.md) | Prints the IR report: the symbols, views and queries the analyser made of each script. | +| [`relix provenance`](provenance.md) | Runs each query and prints its rows, each annotated with where it came from in a provenance semiring. | +| [`relix fmt`](fmt.md) | Prints scripts canonically, keeping their comments. | +| [`relix catalog`](catalog.md) | Inspects the catalog: the declarations a run takes from the .relix/ directories from ~/.relix down to the working directory, and from --catalog files. | +| [`relix drivers`](drivers.md) | Lists and installs JDBC drivers. | +| [`relix connectors`](connectors.md) | Lists and installs connector plugins, such as mongodb. | +| [`relix doc`](doc.md) | Shows the reference page for TOPIC, found by glyph, keyword or name (σ, select, selection), through $PAGER on a terminal. | +| [`relix help`](help.md) | When no COMMAND is given, the usage help for the main command is displayed. | +| [`relix version`](version.md) | Prints the versions of relix and of the Relix engine it runs. | +| [`relix completion`](completion.md) | Prints a completion script for SHELL: bash, zsh, fish or powershell. | + +## Exit status + +| Status | Meaning | +|---|---| +| 0 | success | +| 1 | an assertion failed (--fail-empty, --fail-rows) | +| 2 | usage error (bad option, nothing to run) | +| 3 | the script did not parse or analyse | +| 4 | execution failed (a data error, a limit hit) | +| 5 | environment (a missing driver, an untrusted catalog, a connection refused) | +| 70 | internal error in relix | +| 130 | interrupted (SIGINT) | +| 141 | standard output closed (SIGPIPE) | + +## Examples + +With no command, `relix` runs scripts, as [`relix run`](run.md) does: + +```shell +relix -e 'PROJECT name, city (Customers)' -o tsv +``` +``` +name city +Alice London +Bob Leeds +Carol London +Dan York +``` + +Global options go before or after the command's name: + +```shell +relix -C reports catalog ls -o tsv --no-header +``` +``` +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/reports/.relix/catalog/10-sample.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +## Environment + +| Variable | Effect | +|---|---| +| `RELIX_OUTPUT` | the output format when `-o` is not given | +| `RELIX_PROFILE` | the profile when `-P` is not given | +| `RELIX_NOW` | the instant `NOW()` returns when `--now` is not given | +| `RELIX_CATALOG_PATH` | catalog files to load, separated as in `PATH`, before the `--catalog` files | +| `RELIX_TRUST_ALL` | `1` or `true` trusts every project's `.relix/` directory | +| `RELIX_DRIVERS` | the directory JDBC drivers are installed in and loaded from, instead of `~/.relix/drivers` | +| `NO_COLOR` | when set, a table on a terminal is not coloured unless `--color=always` | +| `PAGER` | the program [`relix doc`](doc.md) shows a page through on a terminal; `cat` or empty shows it directly | + +A `${NAME}` placeholder in a declaration may also be resolved from the environment; see +[Secrets and profiles](../secrets.md). + +## Files + +| Path | What it holds | +|---|---| +| `~/.relix/catalog/*.relix` | your own catalog, which every directory under your home sees | +| `~/.relix/profiles.json` | your own profiles | +| `~/.relix/trusted` | the projects you have trusted, with a hash of each one's files | +| `~/.relix/drivers/` | installed JDBC drivers | +| `~/.relix/connectors/` | installed connector plugins | +| `~/.relix/cache/files/` | files fetched over `https` by a file connection | +| `.relix/catalog/*.relix`, `.relix/relixrc`, `.relix/profiles.json`, `.relix/root` | a project's catalog, defaults, profiles, and the marker that ends the walk up; see [The catalog](../catalog.md) | diff --git a/docs/guide/command-line/commands/run.md b/docs/guide/command-line/commands/run.md new file mode 100644 index 0000000..5a62bc1 --- /dev/null +++ b/docs/guide/command-line/commands/run.md @@ -0,0 +1,78 @@ +# relix run + +Runs scripts and writes their queries' rows to standard output. + +## Usage + +```text +relix run [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | +| `--line-buffered` | Flush each row as it is written, for a reader that wants it at once. | +| `--query=NAME` | Print only the query NAME. A machine format (tsv, csv, ndjson, json) prints one query: the script's last, unless this names another. | +| `--all` | With ndjson, print every query, each row tagged "_query": NAME. | +| `--fail-empty` | Exit 1 when the queries printed return no rows. | +| `--fail-rows` | Exit 1 when the queries printed return any row. | +| `--trace=FILE` | Write the event feed (what was rewritten, planned and run) to standard error, or to FILE with --trace=FILE, while the rows go to standard output. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +`relix run` is what `relix` does with no command, so these two are the same: + +```shell +relix run -e 'GROUP status, SUM(amount) -> total (Orders)' +``` +``` +── ────────────────────────────────────────────────────────────── + status total + ───────── ───── + completed 400 + pending 50 +(2 rows) +``` + +```shell +relix -e 'GROUP status, SUM(amount) -> total (Orders)' -o csv +``` +``` +status,total +completed,400 +pending,50 +``` + +Each script file is a run of its own, with its own declarations: + +```shell +relix run scripts/top-customers.relix -a city=Leeds scripts/by-city.relix -o tsv +``` +``` +name total +Alice 200 +Carol 200 +name +Bob +``` + +[Binding inputs](../inputs.md), [Output formats](../output.md) and +[Pipelines](../pipelines.md) cover the options in use. diff --git a/docs/guide/command-line/commands/trace.md b/docs/guide/command-line/commands/trace.md new file mode 100644 index 0000000..ab01e12 --- /dev/null +++ b/docs/guide/command-line/commands/trace.md @@ -0,0 +1,53 @@ +# relix trace + +Runs each query and prints its event feed, what the engine rewrote, planned and ran, discarding the rows. relix run --trace keeps the rows too. + +## Usage + +```text +relix trace [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--query=NAME` | Trace only the query NAME. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +Each event ends with how long it took, which varies from run to run, so here `cut` +leaves the timings off: + +```shell +relix trace -e 'SELECT amount > 100 (Orders)' | cut -d'(' -f1 +``` +``` +[EXECUTE ] orders SCAN scanned 4 rows +[EXECUTE ] ROWS query delivered 2 rows +``` + +`relix run --trace` writes the same feed to standard error while the rows go to standard +output, and `--trace=FILE` writes it to a file: + +```shell +relix --trace=trace.txt -e 'SELECT amount > 100 (Orders)' -o tsv +``` +``` +order_id customer_id amount status placed +1 1 120 completed 2026-09-02 +4 3 200 completed 2026-10-01 +``` diff --git a/docs/guide/command-line/commands/version.md b/docs/guide/command-line/commands/version.md new file mode 100644 index 0000000..0419b2e --- /dev/null +++ b/docs/guide/command-line/commands/version.md @@ -0,0 +1,23 @@ +# relix version + +Prints the versions of relix and of the Relix engine it runs. + +## Usage + +```text +relix version [OPTIONS] +``` + +It also takes the [global options](relix.md#global-options). + +## Examples + +`relix version`, like `relix --version`, prints one line: the version of the command, +then the version of the Relix engine it runs, as + +```text +relix 1.0.0 (engine 1.0.0) +``` + +The two are numbered independently: a release of the command names the engine release +it was built on. diff --git a/docs/guide/command-line/getting-started.md b/docs/guide/command-line/getting-started.md new file mode 100644 index 0000000..f5b036d --- /dev/null +++ b/docs/guide/command-line/getting-started.md @@ -0,0 +1,218 @@ +# The relix command + +`relix` runs Relix scripts from a shell. It is built to be one stage of a Unix pipeline: +the script comes from the command line, a file or standard input; data comes from files, +standard input or the tables a project declares; and the rows go to standard output in a +format the next program reads. Everything else — errors, warnings, progress — goes to +standard error, so a pipeline never mistakes a message for a row. + +This page installs the command, runs a first query, and shows the shape every command +shares. The pages after it take one subject each, and the +[command reference](commands/relix.md) lists every command and option. + +## Installing + +`relix` ships as a self-contained archive for macOS, Linux and Windows: a launcher, its +own Java runtime, the man pages and completion scripts for bash, zsh, fish and +PowerShell. Nothing else needs to be installed, and no Java is needed on the machine. + +On macOS and Linux, with Homebrew: + +```text +brew install DarkCollective/relix/relix +``` + +On Windows, with Scoop: + +```text +scoop bucket add relix https://github.com/DarkCollective/scoop-relix +scoop install relix +``` + +Or download the archive for your platform from the +[releases page](https://github.com/DarkCollective/relix-cli/releases), unpack it, and put +its `bin` directory on your `PATH`. Each release lists the archives' SHA-256 checksums +in `checksums.txt`. + +`relix version` prints the version of the command and the version of the Relix engine it +runs, as `relix VERSION (engine VERSION)`. The two are numbered independently. + +## The examples on these pages + +Every example in this section is run on each build of the command, and the block under +it is what it printed. The examples run in `~/shop`, a small project with a customer +file, an order file and a stock database: + +```shell +cat data/customers.csv data/orders.csv +``` +``` +customer_id,name,city +1,Alice,London +2,Bob,Leeds +3,Carol,London +4,Dan,York +order_id,customer_id,amount,status,placed +1,1,120,completed,2026-09-02 +2,1,80,completed,2026-09-14 +3,2,50,pending,2026-09-20 +4,3,200,completed,2026-10-01 +``` + +The project declares its tables once, in `~/shop/.relix/catalog/`, so the examples can +name `Customers` and `Orders` without declaring them. [The catalog](catalog.md) explains +how. A path printed under your home directory is shown as `~`. + +## A first query + +`-e` takes Relix text. A bare expression is run as a query: + +```shell +relix -e 'SELECT amount > 100 (Orders)' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id customer_id amount status placed + ──────── ─────────── ────── ───────── ────────── + 1 1 120 completed 2026-09-02 + 4 3 200 completed 2026-10-01 +(2 rows) +``` + +On a terminal the rows are a table. In a pipe they are tab-separated values, one row per +line under a header, which is what `cut`, `sort`, `awk` and `column` expect: + +```shell +relix -e 'SELECT amount > 100 (Orders)' | cut -f1,3 +``` +``` +order_id amount +1 120 +4 200 +``` + +The language itself — every operator, predicate and function — is the subject of the +[language reference](../../reference/README.md). Operators have a Unicode glyph and an ASCII +keyword, and the command reads both: `σ amount > 100 (Orders)` is the same query. + +## Where the script comes from + +A command reads its script from the first of these that is given: + +1. **`-e TEXT`**, repeatable. The texts are joined, in order, into one script, so one + `-e` can declare a view and the next use it. A text that is a bare relational + expression rather than statements is run as `query { … };`. +2. **Script files**, named as arguments. Each runs in a session of its own, so two files on + one command line never see each other's declarations. `-` names standard input. +3. **Standard input**, when nothing is named and standard input is not a terminal. + +With none of them, the command prints its usage and exits 2. + +```shell +relix -e 'Big := { SELECT amount > 100 (Orders) };' -e 'query { PROJECT order_id (Big) };' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id + ──────── + 1 + 4 +(2 rows) +``` + +A script file holds statements, as a script always does: + +```shell +cat scripts/top-customers.relix +``` +``` +-- The customers who have spent the most, best first. +query { SORT total DESC (PROJECT name, total (Customers JOIN Spend)) }; +``` + +```shell +relix scripts/top-customers.relix +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 200 + Carol 200 +(2 rows) +``` + +and a script can arrive on standard input: + +```shell +echo 'query { PROJECT name (Customers) };' | relix -o csv +``` +``` +name +Alice +Bob +Carol +Dan +``` + +A relative path in a script — a `csv("…")` source, an `import` — resolves against the +script file's own directory, and in `-e` text or standard input against the working +directory. `-C DIR` makes the command act as if it were started in `DIR`. + +## Commands + +`relix` with no command name runs scripts; it is `relix run`. Every other job is a command +of its own, and each writes one thing to standard output: + +| Command | Standard output | +|---|---| +| [`relix run`](commands/run.md) | the rows of the queries (the default) | +| [`relix check`](commands/check.md) | nothing; problems go to standard error, or `--json` to standard output | +| [`relix explain`](commands/explain.md) | the physical plan of each query | +| [`relix optimize`](commands/optimize.md) | the script, rewritten by the optimiser | +| [`relix trace`](commands/trace.md) | what the engine rewrote, planned and ran | +| [`relix bundle`](commands/bundle.md) | one JSON document per script: diagnostics, plans and events | +| [`relix ir`](commands/ir.md) | the symbols, views and queries the analyser made of the script | +| [`relix provenance`](commands/provenance.md) | the rows, each with where it came from | +| [`relix fmt`](commands/fmt.md) | the script, printed canonically | +| [`relix catalog`](commands/catalog.md) | the catalog's files, relations, columns and trust | +| [`relix drivers`](commands/drivers.md), [`relix connectors`](commands/connectors.md) | the JDBC drivers and connector plugins, and installing them | +| [`relix doc`](commands/doc.md) | a page of the language reference | +| [`relix completion`](commands/completion.md) | a completion script for a shell | +| [`relix help`](commands/help.md), [`relix version`](commands/version.md) | usage, and the versions | + +Options follow the usual conventions: short options cluster (`-qN`), a long option takes +its value as `--output=csv` or `--output csv`, `--` ends the options, and `-` is standard +input. The [global options](commands/relix.md#global-options) — the profile, the limits, +`-C` and the rest — are accepted by every command, before or after its name. + +## When something goes wrong + +A problem in a script is reported on standard error as `FILE:LINE:COLUMN: severity: +message`, the shape compilers use, so an editor or a CI annotator can jump to it: + +```shell +relix check scripts/broken.relix 2>&1; echo "exit $?" +``` +``` +scripts/broken.relix:1:30: error: Undefined relation: 'Ordrs' +exit 3 +``` + +The exit status says what kind of failure it was, so a script can tell a typo from a +database that is down: + +| Status | Meaning | +|---|---| +| 0 | success | +| 1 | an assertion asked for failed: `--fail-empty` or `--fail-rows` | +| 2 | a usage error: a bad option, nothing to run | +| 3 | the script did not parse or analyse | +| 4 | the query failed while it ran: a data error, a limit hit | +| 5 | the environment: a missing driver, an untrusted catalog, a connection refused | +| 70 | an internal error in `relix`; the stack trace goes to standard error | +| 130 | interrupted with Ctrl-C (`SIGINT`) | +| 141 | standard output was closed by the reader (`SIGPIPE`) | + +`-q` keeps only the errors, `-v` adds notices such as the catalog files and drivers +loaded, and `-vv` adds timings. diff --git a/docs/guide/command-line/index.json b/docs/guide/command-line/index.json new file mode 100644 index 0000000..7830f5e --- /dev/null +++ b/docs/guide/command-line/index.json @@ -0,0 +1,28 @@ +{ + "title": "The relix command line", + "pages": [ + {"path": "getting-started.md", "chapter": "The command line", "title": "The relix command", "summary": "Installing; a first query; where the script comes from; the commands; diagnostics and exit status"}, + {"path": "inputs.md", "chapter": "The command line", "title": "Binding inputs", "summary": "`-i` and its formats; standard input; inferred and given headings; inputs that never end; shadowing the catalog; query parameters with `-a`"}, + {"path": "output.md", "chapter": "The command line", "title": "Output formats", "summary": "`table`, `tsv`, `csv`, `ndjson`, `json` and `markdown`; NULLs and headers; scripts with several queries; `--fail-empty` and `--fail-rows`; messages; closed pipes and interrupts"}, + {"path": "catalog.md", "chapter": "Projects on the command line", "title": "The catalog", "summary": "`.relix/` directories and how they are found; trusting a project; a nearer declaration wins; inspecting the catalog; `--catalog`; `relixrc`"}, + {"path": "secrets.md", "chapter": "Projects on the command line", "title": "Secrets and profiles", "summary": "`${NAME}` placeholders; `profiles.json` and `-P`; the environment and `-D`; private profile files; what is never printed"}, + {"path": "pipelines.md", "chapter": "Command-line recipes", "title": "Pipelines", "summary": "With `jq`, `awk`, `sort` and `head`; with `find` and `xargs`; scripts in here-documents; a data check in CI; optimising a script and reviewing the change"}, + {"path": "commands/relix.md", "chapter": "Command reference", "title": "relix", "summary": "The global options every command takes; the exit statuses; the environment variables and files", "command": "relix"}, + {"path": "commands/run.md", "chapter": "Command reference", "title": "relix run", "summary": "Runs scripts and writes their queries' rows", "command": "relix run"}, + {"path": "commands/check.md", "chapter": "Command reference", "title": "relix check", "summary": "Analyses scripts without running them, and reports their problems", "command": "relix check"}, + {"path": "commands/explain.md", "chapter": "Command reference", "title": "relix explain", "summary": "Prints each query's physical plan", "command": "relix explain"}, + {"path": "commands/optimize.md", "chapter": "Command reference", "title": "relix optimize", "summary": "Prints the script with each query rewritten by the optimiser", "command": "relix optimize"}, + {"path": "commands/trace.md", "chapter": "Command reference", "title": "relix trace", "summary": "Runs each query and prints what the engine rewrote, planned and ran", "command": "relix trace"}, + {"path": "commands/bundle.md", "chapter": "Command reference", "title": "relix bundle", "summary": "Prints each script's JSON bundle of diagnostics, plans and events", "command": "relix bundle"}, + {"path": "commands/ir.md", "chapter": "Command reference", "title": "relix ir", "summary": "Prints the symbols, views and queries the analyser made of a script", "command": "relix ir"}, + {"path": "commands/provenance.md", "chapter": "Command reference", "title": "relix provenance", "summary": "Runs each query and annotates each row with where it came from", "command": "relix provenance"}, + {"path": "commands/fmt.md", "chapter": "Command reference", "title": "relix fmt", "summary": "Prints scripts canonically, keeping their comments", "command": "relix fmt"}, + {"path": "commands/catalog.md", "chapter": "Command reference", "title": "relix catalog", "summary": "Lists the catalog's files, relations, columns and declarations; trusts a project", "command": "relix catalog", "subcommands": [{"command": "relix catalog files", "anchor": "relix-catalog-files"}, {"command": "relix catalog ls", "anchor": "relix-catalog-ls"}, {"command": "relix catalog schema", "anchor": "relix-catalog-schema"}, {"command": "relix catalog where", "anchor": "relix-catalog-where"}, {"command": "relix catalog trust", "anchor": "relix-catalog-trust"}]}, + {"path": "commands/drivers.md", "chapter": "Command reference", "title": "relix drivers", "summary": "Lists and installs JDBC drivers", "command": "relix drivers", "subcommands": [{"command": "relix drivers ls", "anchor": "relix-drivers-ls"}, {"command": "relix drivers install", "anchor": "relix-drivers-install"}]}, + {"path": "commands/connectors.md", "chapter": "Command reference", "title": "relix connectors", "summary": "Lists and installs connector plugins", "command": "relix connectors", "subcommands": [{"command": "relix connectors ls", "anchor": "relix-connectors-ls"}, {"command": "relix connectors install", "anchor": "relix-connectors-install"}]}, + {"path": "commands/doc.md", "chapter": "Command reference", "title": "relix doc", "summary": "Shows a page of the language reference", "command": "relix doc"}, + {"path": "commands/help.md", "chapter": "Command reference", "title": "relix help", "summary": "Prints a command's usage", "command": "relix help"}, + {"path": "commands/version.md", "chapter": "Command reference", "title": "relix version", "summary": "Prints the versions of the command and of the engine", "command": "relix version"}, + {"path": "commands/completion.md", "chapter": "Command reference", "title": "relix completion", "summary": "Prints a completion script for bash, zsh, fish or PowerShell", "command": "relix completion"} + ] +} diff --git a/docs/guide/command-line/inputs.md b/docs/guide/command-line/inputs.md new file mode 100644 index 0000000..6f0dc12 --- /dev/null +++ b/docs/guide/command-line/inputs.md @@ -0,0 +1,291 @@ +# Binding inputs + +A script names the relations it reads. Most of the time those are declared somewhere — in +the script, or in the project's [catalog](catalog.md) — but a pipeline often has data that +exists only for this run: a file another program wrote, the output of `jq`, a log being +followed. `-i` binds a relation name to such data for one run, without a declaration. +`-a` does the same for a single value, a query parameter. + +## `-i NAME=[FORMAT:]LOCATION` + +`-i` binds the relation `NAME` to the data at `LOCATION`, which is a file, `-` for +standard input, or an `http(s)` URL. It is repeatable, one relation per `-i`: + +```shell +cat data/returns.ndjson +``` +``` +{"order_id": 2, "reason": "damaged", "refund": {"amount": 80, "currency": "GBP"}} +{"order_id": 4, "reason": "late", "refund": {"amount": 20, "currency": "GBP"}} +``` + +```shell +relix -i Returns=data/returns.ndjson -e 'PROJECT order_id, reason (Returns)' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id reason + ──────── ─────── + 2 damaged + 4 late +(2 rows) +``` + +A bound relation is used like any other, and joins with the catalog's tables: + +```shell +relix -i Returns=data/returns.ndjson -e 'PROJECT order_id, amount, reason (Orders JOIN Returns)' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id amount reason + ──────── ────── ─────── + 2 80 damaged + 4 200 late +(2 rows) +``` + +A relative path resolves against the working directory, or against `-C DIR`. The same name +may not be bound twice. + +### Formats + +`FORMAT` is one of: + +| Format | What it reads | +|---|---| +| `csv` | comma-separated values, RFC 4180, with a header row | +| `tsv` | tab-separated values, with a header row | +| `json` | one JSON array of objects | +| `ndjson` | one JSON object per line, the format `jq -c` writes | + +Without a format, the file's extension decides: `.csv`, `.tsv` or `.tab`, `.json`, and +`.ndjson` or `.jsonl`. A file with any other extension needs its format named: + +```shell +cp data/orders.csv orders.txt +relix -i O=orders.txt -e 'O' 2>&1; echo "exit $?" +``` +``` +relix: -i O=orders.txt: cannot tell the format from the extension .txt; name it, as NAME=csv:LOCATION (csv, tsv, json or ndjson) +exit 2 +``` + +```shell +relix -i O=csv:orders.txt -e 'GROUP status, COUNT(*) -> n (O)' +``` +``` +── ────────────────────────────────────────────────────────────── + status n + ───────── ─ + completed 3 + pending 1 +(2 rows) +``` + +A nested JSON object becomes a nested value, which the language's dotted paths and +`UNNEST` take apart; [Nested data](../nested.md) covers them. + +### Standard input + +`-` binds standard input. Standard input has no extension, so it always names its format: + +```shell +jq -c 'select(.refund.amount > 50)' data/returns.ndjson \ + | relix -i Returns=ndjson:- -e 'PROJECT name, order_id, reason (Customers JOIN Orders JOIN Returns)' +``` +``` +── ────────────────────────────────────────────────────────────── + name order_id reason + ───── ──────── ─────── + Alice 2 damaged +(1 row) +``` + +Only one input can be standard input, and when one is, the script cannot be read from it +too: it comes from `-e` or a file. + +```shell +relix -i A=csv:- -i B=csv:- -e 'A' 2>&1 < data/orders.csv; echo "exit $?" +``` +``` +relix: standard input (-) can be bound to one input only +exit 2 +``` + +### The heading of an input + +The columns of a bound input and their types are inferred. For `csv` and `tsv` the names +are the header row's; for `json` and `ndjson` they are the keys of the objects. The types +are inferred from the values in the first 1000 records, or in as many as `--infer-rows=N` +says, so `order_id` above is a number and `placed` a date. + +`--schema NAME='{ … }'` gives the heading instead, in the syntax a `source` declaration's +`schema` uses. Here the `amount` column is read as a string rather than a number: + +```shell +relix -i O=data/orders.csv --schema O='{ order_id: NUMBER, amount: STRING }' -e 'PROJECT order_id, amount (O)' -o json +``` +``` +[ + {"order_id":1,"amount":"120"}, + {"order_id":2,"amount":"80"}, + {"order_id":3,"amount":"50"}, + {"order_id":4,"amount":"200"} +] +``` + +`--schema` and `--unbounded` name an input bound with `-i`; naming any other is a usage +error. + +### Inputs that never end + +An input such as `tail -f`'s never ends. `--unbounded=NAME` declares that, and two things +follow. The rows stream: each is filtered, joined against bounded tables and written as +it arrives, rather than after the input ends. And a query that would have to read all of +the input before writing anything — a `GROUP`, a `SORT`, a `DISTINCT` — is refused before +it starts, rather than waiting forever: + +```shell +printf '{"level":"error","msg":"disk"}\n{"level":"info","msg":"ok"}\n' | relix -i Log=ndjson:- --unbounded=Log -e "SELECT level = 'error' (Log)" -o ndjson +``` +``` +{"level":"error","msg":"disk"} +``` + +```shell +printf '{"level":"error"}\n' | relix -i Log=ndjson:- --unbounded=Log -e 'GROUP level, COUNT(*) -> n (Log)' -o ndjson 2>&1; echo "exit $?" +``` +``` +relix: -e: cannot materialise unbounded relation for blocking operator γ (GROUP); add a bound (e.g. λ n) below it +exit 3 +``` + +A `table` needs every row before it can align the columns, so an unbounded input is +written in a format that streams — `tsv`, `csv`, `ndjson` or `json` — and on a terminal +that means naming one with `-o`. A `json` input is one array, read whole, so it cannot be +unbounded. Write one object per +line and bind it as `ndjson`. With `--line-buffered` each row is written out as soon as it +is produced, for a reader that wants it at once; see [Output formats](output.md). + +### Files on the web + +A location that is an `http://` or `https://` URL is fetched, and only with `--remote`, +which a run must give for anything to be fetched. Without it, binding a URL is a usage +error. The format comes from the URL's path, or is named. + +### Shadowing the catalog + +A bound input replaces a relation of the same name that the catalog declares, for this +run. Every view that reads the name reads the input instead, so a view written against the +real `Orders` can be run over a sample file without changing it: + +```shell +relix -e 'BigSpenders' +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 200 + Carol 200 +(2 rows) +``` + +```shell +relix -i Orders=reports/sample-orders.csv -e 'BigSpenders' +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 500 + Dan 250 +(2 rows) +``` + +## `-a NAME=VALUE`: query parameters + +A script can leave a value to be supplied when it runs, as a parameter: `$city` in + +```shell +cat scripts/by-city.relix +``` +``` +-- The customers of one city, named when the script runs: relix -a city=London +query { PROJECT name (SELECT city = $city (Customers)) }; +``` + +`-a city=London` binds it (the long form is `--arg`): + +```shell +relix -a city=London scripts/by-city.relix +``` +``` +── ────────────────────────────────────────────────────────────── + name + ───── + Alice + Carol +(2 rows) +``` + +```shell +relix --arg city=Leeds scripts/by-city.relix -o tsv +``` +``` +name +Bob +``` + +The value is bound to the parameter, never pasted into the script's text, so whatever it +holds — quotes, a semicolon, a fragment of Relix — it is only ever a value, and cannot +change what the query does: + +```shell +relix -a "city=London' OR 1 = 1 --" scripts/by-city.relix -o tsv +``` +``` +name +``` + +A parameter takes the type of what it is compared with, and the text `-a` gives is read +as that type: a number, a date, `true` or `false`. Text that is not of the type is an +error naming the parameter: + +```shell +relix -a since=2026-09-15 -e 'PROJECT order_id, placed (SELECT placed >= $since (Orders))' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id placed + ──────── ────────── + 3 2026-09-20 + 4 2026-10-01 +(2 rows) +``` + +```shell +relix -a since=yesterday -e 'SELECT placed >= $since (Orders)' 2>&1; echo "exit $?" +``` +``` +relix: -e: parameter $since is compared with a DATE, so it cannot be bound to the text 'yesterday' +exit 3 +``` + +A query cannot run until every parameter it reaches has a value: + +```shell +relix scripts/by-city.relix 2>&1; echo "exit $?" +``` +``` +relix: scripts/by-city.relix: parameter $city is not bound; give it a value with -a city=VALUE +exit 3 +``` + +The [parameter reference page](../../reference/language/parameter.md) has the rules in +full. + +`-a` binds a value a query compares with. A value a *declaration* needs — a database's +address, a password, a token — is a placeholder, `${NAME}`, given by a profile, the +environment or `-D`, and the subject of [Secrets and profiles](secrets.md). diff --git a/docs/guide/command-line/output.md b/docs/guide/command-line/output.md new file mode 100644 index 0000000..580ac4e --- /dev/null +++ b/docs/guide/command-line/output.md @@ -0,0 +1,269 @@ +# Output formats + +Rows go to standard output and nothing else does. Errors, warnings, notices and +progress go to standard error, so the rows can be piped, redirected or captured without +anything to strip out of them. This page covers the formats rows are written in, which +queries of a script are written, how a run can assert something about its rows, and how +a run ends when its reader stops reading. + +## Choosing a format + +`-o FORMAT` (`--output=FORMAT`) chooses one of: + +| Format | Shape | Streams | +|---|---|---| +| `table` | aligned columns under a rule, with a row count | no | +| `tsv` | tab-separated values under a header row | yes | +| `csv` | comma-separated values under a header row, RFC 4180 | yes | +| `ndjson` | one JSON object per row, one per line | yes | +| `json` | one JSON array of objects | yes | +| `markdown` | a Markdown table | no | + +Without `-o`, the format is `$RELIX_OUTPUT`, else the `output` a project's +[`relixrc`](catalog.md#defaults-relixrc) sets, else `table` when standard output is a +terminal and `tsv` when it is not. So the same command prints a table to a person and +tab-separated values to a pipe. + +```shell +relix -e 'Customers' -o csv +``` +``` +customer_id,name,city +1,Alice,London +2,Bob,Leeds +3,Carol,London +4,Dan,York +``` + +```shell +relix -e 'Customers' -o ndjson +``` +``` +{"customer_id":1,"name":"Alice","city":"London"} +{"customer_id":2,"name":"Bob","city":"Leeds"} +{"customer_id":3,"name":"Carol","city":"London"} +{"customer_id":4,"name":"Dan","city":"York"} +``` + +```shell +relix -e 'PROJECT name, city (Customers)' -o json +``` +``` +[ + {"name":"Alice","city":"London"}, + {"name":"Bob","city":"Leeds"}, + {"name":"Carol","city":"London"}, + {"name":"Dan","city":"York"} +] +``` + +```shell +relix -e 'PROJECT name, city (Customers)' -o markdown +``` +``` +### + +| name | city | +| --- | --- | +| Alice | London | +| Bob | Leeds | +| Carol | London | +| Dan | York | +``` + +```shell +RELIX_OUTPUT=tsv relix -e 'PROJECT name (Customers)' +``` +``` +name +Alice +Bob +Carol +Dan +``` + +`tsv` writes a tab, a newline, a carriage return or a backslash inside a value as `\t`, +`\n`, `\r` or `\\`, so that every line is one row and every tab separates two columns; +this is the form `cut`, `awk` and `sort -t` read. `csv` quotes a value holding a comma, a +quote or a line break instead. `json` and `ndjson` write numbers as JSON numbers, booleans +as `true` and `false`, NULL as `null`, and dates, times and durations as ISO-8601 +strings. + +A format that streams writes each row as it is produced, so a result larger than memory +goes straight through and the first rows arrive before the last are computed. `table` and +`markdown` align their columns, which needs every row first. In a table a value longer +than 30 characters is cut short, ending `…`; the machine formats write every value whole. + +### Headers, NULLs and colour + +| Option | Effect | +|---|---| +| `--no-header` | leave out the header row of `csv` and `tsv` | +| `--null=STRING` | write NULL as `STRING`; by default it is empty in `csv`, `tsv` and `markdown`, and `NULL` in a table | +| `--color=WHEN` | colour a table `auto`, `always` or `never`; `auto` colours on a terminal unless `$NO_COLOR` is set | + +```shell +relix -e 'PROJECT name, order_id (Customers LJOIN Customers.customer_id = Orders.customer_id Orders)' -o csv --null=- +``` +``` +name,order_id +Alice,1 +Alice,2 +Bob,3 +Carol,4 +Dan,- +``` + +```shell +relix -e 'PROJECT name (Customers)' -o tsv --no-header +``` +``` +Alice +Bob +Carol +Dan +``` + +## Scripts with several queries + +A script may hold several `query` statements. A table prints each of them, under its +name: the view's name for `query Name;`, and ``, its place among the +script's queries, for `query { … };`. + +```shell +relix -e 'query Completed;' -e 'query BigSpenders;' +``` +``` +── Completed ─────────────────────────────────────────────────────────────────── + order_id customer_id amount status placed + ──────── ─────────── ────── ───────── ────────── + 1 1 120 completed 2026-09-02 + 2 1 80 completed 2026-09-14 + 4 3 200 completed 2026-10-01 +(3 rows) +── BigSpenders ───────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 200 + Carol 200 +(2 rows) +``` + +A machine format holds one result, since a CSV file with two headers is not a CSV file. It +prints the script's **last** query, as a shell function returns its last command's status, +or the one `--query=NAME` names: + +```shell +relix -e 'query Completed;' -e 'query BigSpenders;' -o tsv +``` +``` +name total +Alice 200 +Carol 200 +``` + +```shell +relix -e 'query Completed;' -e 'query BigSpenders;' -o tsv --query=Completed +``` +``` +order_id customer_id amount status placed +1 1 120 completed 2026-09-02 +2 1 80 completed 2026-09-14 +4 3 200 completed 2026-10-01 +``` + +`--all` prints every query in `ndjson`, each row tagged with the query it belongs to in a +`"_query"` field, so one stream can carry several results and `jq` can take them apart: + +```shell +relix -e 'query Spend;' -e 'query BigSpenders;' -o ndjson --all +``` +``` +{"_query":"Spend","customer_id":1,"total":200} +{"_query":"Spend","customer_id":3,"total":200} +{"_query":"BigSpenders","name":"Alice","total":200} +{"_query":"BigSpenders","name":"Carol","total":200} +``` + +With any other machine format `--all` is a usage error. + +Several script files on one command line are independent runs, and their output is +written one after another in the order the files were named. + +## Asserting something about the rows + +`--fail-empty` makes a run exit 1 when the queries it printed returned no rows, and +`--fail-rows` when they returned any. The rows are printed either way. This is the +`grep -q` idiom, for a data check in CI or in a shell `if`: + +```shell +relix --fail-rows -e 'SELECT amount < 0 (Orders)' -o tsv; echo "exit $?" +``` +``` +order_id customer_id amount status placed +exit 0 +``` + +```shell +relix --fail-rows -e "SELECT status = 'pending' (Orders)" -o tsv > /dev/null || echo "orders are still pending" +``` +``` +orders are still pending +``` + +[Pipelines](pipelines.md#a-data-check-in-ci) shows one in a CI job. + +## Where messages go + +A problem in a script is reported on standard error as `FILE:LINE:COLUMN: severity: +message`. `FILE` is the script's path, `-e` (or `-e#2` for the second of several `-e` +texts), or ``; `LINE` and `COLUMN` count within that text, so a message about an +`-e` points into what was typed. A problem with the run itself — an option, a file that +is not there, a database that refuses the connection — is one line beginning `relix:`. + +```shell +relix -e 'SELECT amount > 100 (Ordrs)' 2>&1; echo "exit $?" +``` +``` +-e:1:22: error: Undefined relation: 'Ordrs' +exit 3 +``` + +Warnings are printed too, unless `-q` is given. `-v` adds notices — each catalog file +read, each JDBC driver loaded — and `-vv` adds how long each stage took. `relix check +--json` writes the diagnostics to standard output as one JSON document instead, for a +tool to read; see [`relix check`](commands/check.md). + +The [exit status](getting-started.md#when-something-goes-wrong) says which kind of +failure it was. + +## When the reader stops reading + +A program at the other end of a pipe can stop reading at any time: `head` does after its +last line. When it does, `relix` stops: the query in flight is cancelled — a plan stops +pulling rows, a database cursor is closed — and the command exits 141, the status a shell +gives a program ended by `SIGPIPE`, without printing anything. So a query over a large or +endless relation can be cut short with `head`: + +```shell +relix -e 'Orders' -o tsv | head -3 +``` +``` +order_id customer_id amount status placed +1 1 120 completed 2026-09-02 +2 1 80 completed 2026-09-14 +``` + +Ctrl-C (`SIGINT`) likewise cancels the query and closes the session, and the command exits +130. + +Standard output is written in blocks and flushed at the end of each result. With +`--line-buffered` each row is flushed as it is written, for a reader that wants each row +at once — an input following a log, read by something that reacts to each line. + +## Following what the engine did + +`--trace` writes the run's event feed — what the optimiser rewrote, the plan it chose, +how many rows each step produced — to standard error while the rows go to standard +output; `--trace=FILE` writes it to `FILE` instead. [`relix trace`](commands/trace.md) +prints the same feed on standard output, without the rows. diff --git a/docs/guide/command-line/pipelines.md b/docs/guide/command-line/pipelines.md new file mode 100644 index 0000000..8f4fe90 --- /dev/null +++ b/docs/guide/command-line/pipelines.md @@ -0,0 +1,192 @@ +# Pipelines + +`relix` reads text and writes text, so it works with the programs a shell already has. +This page is a set of recipes: `relix` with `jq` for JSON, with `find` and `xargs` for +many files, with `awk`, `sort` and `head` for its rows, and as a check in CI. + +The rule that makes all of them work: **standard output carries rows and nothing else.** +Messages go to standard error, the format in a pipe is tab-separated values unless `-o` +says otherwise, and the [exit status](getting-started.md#when-something-goes-wrong) says +what happened. + +## With jq + +`jq` is the tool for shaping JSON, and `relix` the one for joining and aggregating it. A +pipeline passes JSON between them as NDJSON, one object per line, which `jq -c` writes, +`-i NAME=ndjson:-` reads and `-o ndjson` writes. + +Here `jq` flattens the returns file, `relix` joins it to the customers and orders, and +`jq` formats the result: + +```shell +jq -c '{order_id, reason, refund: .refund.amount}' data/returns.ndjson \ + | relix -i Returns=ndjson:- -e 'PROJECT name, order_id, refund (Customers JOIN Orders JOIN Returns)' -o ndjson \ + | jq -r '"\(.name) was refunded \(.refund) on order \(.order_id)"' +``` +``` +Alice was refunded 80 on order 2 +Carol was refunded 20 on order 4 +``` + +`relix` can also hand `jq` a single JSON document, an array, with `-o json`: + +```shell +relix -e 'GROUP city, COUNT(*) -> customers (Customers)' -o json | jq -c 'map({(.city): .customers}) | add' +``` +``` +{"London":2,"Leeds":1,"York":1} +``` + +## With awk, sort and cut + +Tab-separated values are what the classic text tools split on. The header is the first +line; `--no-header` leaves it out. + +```shell +relix -e 'Orders' --no-header | awk -F'\t' '{ total[$4] += $3 } END { for (s in total) print s, total[s] }' | sort +``` +``` +completed 400 +pending 50 +``` + +## Stopping early + +When the reader of a pipe stops, `relix` stops too: it cancels the query rather than +computing rows nobody will read. `head` can cut a large result short: + +```shell +relix -e 'SORT amount DESC (Orders)' | head -3 +``` +``` +order_id customer_id amount status placed +4 3 200 completed 2026-10-01 +1 1 120 completed 2026-09-02 +``` + +## With find and xargs + +### Lint every script + +`relix check` analyses scripts without running them, reporting each problem as +`FILE:LINE:COLUMN: severity: message` and exiting 3 if any script has one. Over every +script in a tree: + +```shell +find scripts -name '*.relix' | sort | xargs relix check 2>&1 || echo "a script has a problem" +``` +``` +scripts/broken.relix:1:30: error: Undefined relation: 'Ordrs' +a script has a problem +``` + +`xargs` fails when any run of the command it starts fails, so the pipeline's status says +whether every script passed. Use `find … -print0 | xargs -0` when a file name may hold a +space. + +### Run a query over each of many files + +`-i` binds one file per run, so `xargs -I{}` runs a query over each file in turn. Here +each CSV file's row count, with its name: + +```shell +find data reports -name '*.csv' | sort | xargs -I{} sh -c 'printf "%s\t" {}; relix -i T={} -e "GROUP COUNT(*) -> n (T)" -o tsv --no-header' +``` +``` +data/customers.csv 4 +data/orders.csv 4 +reports/sample-orders.csv 2 +``` + +### Check every script's formatting + +`relix fmt --check` prints the name of each file that is not formatted canonically, and +exits 1 if there is one; `relix fmt -w` rewrites them in place. The shop's scripts spell +operators as ASCII keywords, so they are checked against `--keywords`: + +```shell +find scripts -name '*.relix' | sort | xargs relix fmt --check --keywords || echo "a script needs formatting" +``` +``` +scripts/messy.relix +scripts/top-customers.relix +a script needs formatting +``` + +## Scripts written in the shell + +A script can be written inline with a here-document, which keeps a longer query readable +and still needs no file: + +```shell +relix -o csv <<'EOF' +Late := { SELECT placed >= DATE '2026-09-15' (Orders) }; +query { PROJECT name, order_id (Customers JOIN Late) }; +EOF +``` +``` +name,order_id +Bob,3 +Carol,4 +``` + +A value the shell knows goes in as a [parameter](inputs.md#a-name-value-query-parameters), +never pasted into the text: + +```shell +city=London +relix -a city="$city" scripts/by-city.relix -o tsv +``` +``` +name +Alice +Carol +``` + +## A data check in CI + +`--fail-rows` exits 1 when the query returns any row, and `--fail-empty` when it returns +none, so a data assertion is one line of a CI job: + +```shell +relix --fail-rows -e 'SELECT amount <= 0 (Orders)' -o tsv > /dev/null && echo "every order has a positive amount" +``` +``` +every order has a positive amount +``` + +```shell +relix --fail-rows -e "SELECT status = 'pending' (Orders)" -o tsv || echo "pending orders found" +``` +``` +order_id customer_id amount status placed +3 2 50 pending 2026-09-20 +pending orders found +``` + +A CI job runs in a fresh container where the project's `.relix/` cannot have been +trusted by hand; `RELIX_TRUST_ALL=1` trusts it, and a profile or the job's secret +environment variables supply the [placeholders](secrets.md). + +## Optimise a script, and review the change + +`relix optimize` prints a script with each query rewritten by the optimiser, so it works +as a filter, and `diff` shows what it changed: + +```shell +cat > slow.relix <<'EOF' +query { SELECT city = 'London' (Customers JOIN Orders) }; +EOF +relix optimize slow.relix | diff slow.relix -; echo "exit $?" +``` +``` +1c1 +< query { SELECT city = 'London' (Customers JOIN Orders) }; +--- +> query { (σ city = "London" (Customers)) ⋈ (Orders) }; +exit 1 +``` + +`diff` exits 1 when the files differ. To keep the rewritten script, redirect it: +`relix optimize slow.relix > fast.relix`. [`relix optimize --report`](commands/optimize.md) +lists the rules that fired instead. diff --git a/docs/guide/command-line/secrets.md b/docs/guide/command-line/secrets.md new file mode 100644 index 0000000..2e2d57c --- /dev/null +++ b/docs/guide/command-line/secrets.md @@ -0,0 +1,184 @@ +# Secrets and profiles + +A connection to a database needs an address and a password; a source that calls an API +needs a token. None of them belongs in a script or a catalog file that is committed and +shared. They are written as **placeholders**, `${NAME}`, and the command supplies the +values when a query runs: from `-D`, from a named **profile**, or from the environment. + +## Placeholders + +The shop's stock lives in a database, declared in the catalog with placeholders where +its address and credentials go: + +```shell +cat .relix/catalog/20-warehouse.relix +``` +``` +-- The stock database. Its address and credentials come from placeholders. +connection warehouse from database { + url: "${WAREHOUSE_URL}", + user: "${WAREHOUSE_USER}", + password: "${WAREHOUSE_PASSWORD}" +}; + +source Stock from warehouse { + table: "STOCK", + schema: { sku: STRING, on_hand: NUMBER } +}; +``` + +A placeholder may stand in any string value of a declaration: a URL, a file path, a +table name, a header, a credential. The engine's +[Secrets in declarations](../secrets.md) page has the rules. + +A placeholder is resolved each time a query runs, for the declarations that query +reaches, and only then. A query that never reaches the warehouse needs none of its +values, and one that does fails before anything is sent when a value is missing, naming +the placeholder; [below](#the-environment-and-d) is an example. + +Each `${NAME}` is looked up in three places, and the first that has it wins: + +1. **`-D NAME=VALUE`** on the command line. +2. **The selected profile**, from the `profiles.json` files. +3. **The environment** of the process. + +## Profiles + +A profile is a named set of values, kept in a `profiles.json` file in a `.relix/` +directory: one JSON object whose members are profiles, each an object of string values. +The shop's project keeps the parts of its `dev` profile that are not secret: + +```shell +cat .relix/profiles.json +``` +``` +{ + "dev": { + "WAREHOUSE_URL": "jdbc:h2:~/shop/data/warehouse", + "WAREHOUSE_USER": "shop" + } +} +``` + +and the password is in your own `~/.relix/profiles.json`, outside the project and its +repository: + +```shell +cat ~/.relix/profiles.json +``` +``` +{ + "dev": { "WAREHOUSE_PASSWORD": "s3cret" } +} +``` + +`-P NAME` (`--profile=NAME`) selects a profile. Its values are merged across every +`profiles.json` the run reads, outermost first, so a nearer file's value for a name +replaces a farther one's, and a profile may be split between your own file and the +project's, as `dev` is here: + +```shell +relix -P dev -e 'Stock' +``` +``` +── ────────────────────────────────────────────────────────────── + sku on_hand + ──── ─────── + mug 12 + tee 0 + tote 31 +(3 rows) +``` + +Without `-P`, the profile is `$RELIX_PROFILE`, else the `profile` a +[`relixrc`](catalog.md#defaults-relixrc) names. With none of them, no profile is +selected and placeholders come from `-D` and the environment alone. + +```shell +RELIX_PROFILE=dev relix -e 'PROJECT sku (SELECT on_hand = 0 (Stock))' -o tsv +``` +``` +sku +tee +``` + +Naming a profile no file defines is a usage error, which lists the ones that are defined: + +```shell +relix -P production -e 'Stock' 2>&1; echo "exit $?" +``` +``` +relix: no profile 'production' (defined: dev) +exit 2 +``` + +A project's `profiles.json` is part of its `.relix/` directory, so it is read only once +the project is [trusted](catalog.md#a-project-must-be-trusted); `~/.relix/profiles.json` +is always read. + +### A profile file must be private + +A profile may hold a password, so a `profiles.json` that its group or others can read is +refused, as `ssh` refuses a private key anyone can read. The error names the `chmod` that +fixes it: + +```shell +chmod 644 ~/.relix/profiles.json +relix -P dev -e 'Stock' 2>&1; echo "exit $?" +chmod 600 ~/.relix/profiles.json +``` +``` +relix: ~/.relix/profiles.json is readable by group or others, and a profile may hold credentials; make it private with: chmod 600 ~/.relix/profiles.json +exit 5 +``` + +Where the file system has no POSIX permissions, as on Windows, the check is not made. + +## The environment, and `-D` + +A value not in the selected profile comes from the environment, which is what a CI job's +secrets and a twelve-factor deployment use. Here no profile is selected: `-D` gives the +address and the user, and the environment the password: + +```shell +WAREHOUSE_PASSWORD=s3cret relix -D WAREHOUSE_URL=jdbc:h2:$HOME/shop/data/warehouse -D WAREHOUSE_USER=shop -e 'Stock' -o tsv +``` +``` +sku on_hand +mug 12 +tee 0 +tote 31 +``` + +Without the password, the query fails before the connection is opened, and the error +names what is missing. The status is 3, as for any script that cannot be analysed: + +```shell +relix -D WAREHOUSE_URL=jdbc:h2:$HOME/shop/data/warehouse -D WAREHOUSE_USER=shop -e 'Stock' 2>&1; echo "exit $?" +``` +``` +relix: -e: placeholder ${WAREHOUSE_PASSWORD} in connection 'warehouse' has no value: the session's placeholder resolver returned none +exit 3 +``` + +`-D NAME=VALUE` sets one value for one run, and beats the profile and the environment. +It is for values that are not secret: a command line is kept in your shell's history and +shown by `ps` to everyone on the machine. A secret belongs in a private `profiles.json`, +or in the environment. + +## What is never printed + +The placeholder stays in the declaration, and the value goes only to the connector that +needs it, for the run that needs it. Commands that print a script or a plan — +`relix fmt`, `relix ir`, `relix bundle`, `relix explain`, `relix catalog` — print +`${WAREHOUSE_PASSWORD}`, never the password, and so does an error message: + +```shell +relix fmt .relix/catalog/20-warehouse.relix +``` +``` +-- The stock database. Its address and credentials come from placeholders. +connection warehouse from jdbc { password: "${WAREHOUSE_PASSWORD}", url: "${WAREHOUSE_URL}", user: "${WAREHOUSE_USER}" }; + +source Stock from warehouse { table: "STOCK", schema: { sku: string, on_hand: number } }; +``` diff --git a/src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java b/src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java new file mode 100644 index 0000000..cff1f97 --- /dev/null +++ b/src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java @@ -0,0 +1,420 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.packaging; + +import com.darkcollective.relix.cli.command.RelixCommand; +import com.darkcollective.relix.cli.io.Host; +import com.darkcollective.relix.cli.io.Interruption; +import picocli.CommandLine; +import picocli.CommandLine.Help; +import picocli.CommandLine.Model.ArgSpec; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Model.OptionSpec; +import picocli.CommandLine.Model.PositionalParamSpec; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Writes the command reference of the programming guide's command-line section: a + * Markdown page per command, from the same picocli model {@code --help} and the man pages + * are printed from, so the three cannot drift (design §7.4), and {@code index.json}, the + * section's machine-readable index. + * + *

A page is the command's description, usage, arguments, options, subcommands and exit + * status, then the hand-written part from {@code docs/command-examples/.md}: its + * examples, which the examples test runs like every other page's. The root command's page, + * {@code relix.md}, holds the global options every command takes; the others link to it + * rather than repeat them. + * + *

The pages are checked in, so that a change to the command's options shows up in its + * documentation in the same diff; {@code CommandPagesTest} fails when they are stale, and + * {@code ./gradlew commandPages} rewrites them. + */ +public final class CommandPages { + + /** The section's pages, relative to the repository. */ + public static final String GUIDE = "docs/guide/command-line"; + + /** The hand-written part of each command page, relative to the repository. */ + public static final String EXAMPLES = "docs/command-examples"; + + /** Where the command pages go, relative to the section. */ + public static final String COMMANDS = "commands"; + + /** The section's machine-readable index, relative to the section. */ + public static final String INDEX = "index.json"; + + /** The width the usage synopsis is wrapped to. */ + private static final int WIDTH = 80; + + private static final Pattern INDEX_CHAPTER = Pattern.compile("^##\\s+(.+?)\\s*$"); + private static final Pattern INDEX_TITLE = Pattern.compile("^#\\s+(.+?)\\s*$"); + private static final Pattern LINK = Pattern.compile("\\[([^\\]]+)]\\(([^)#\\s]+\\.md)\\)"); + + private CommandPages() { + } + + /** + * Writes the pages and the index. + * + * @param args the repository's directory + * @throws IOException if one cannot be written + */ + public static void main(String[] args) throws IOException { + if (args.length != 1) { + throw new IllegalArgumentException("usage: CommandPages REPOSITORY"); + } + Path repository = Path.of(args[0]); + Path guide = repository.resolve(GUIDE); + Map files = generate(repository.resolve(EXAMPLES), guide); + Path commands = guide.resolve(COMMANDS); + Files.createDirectories(commands); + try (var stale = Files.list(commands)) { + for (Path file : stale.toList()) { + if (!files.containsKey(COMMANDS + "/" + file.getFileName())) { + Files.delete(file); + } + } + } + for (Map.Entry file : files.entrySet()) { + Files.writeString(guide.resolve(file.getKey()), file.getValue(), StandardCharsets.UTF_8); + } + } + + /** + * Every file this writes, by its path relative to the section. + * + * @param examples the directory of the command pages' hand-written parts + * @param guide the section, whose {@code README.md} the index is read from + * @return the command pages and {@code index.json} + */ + public static Map generate(Path examples, Path guide) { + CommandSpec root = new CommandLine(new RelixCommand(Host.system(), new Interruption())).getCommandSpec(); + Map files = new LinkedHashMap<>(); + List commands = new ArrayList<>(); + commands.add(root); + commands.addAll(visible(root)); + for (CommandSpec command : commands) { + String name = fileName(command); + files.put(COMMANDS + "/" + name, page(command, root, examples.resolve(name))); + } + files.put(INDEX, index(guide.resolve("README.md"), commands)); + return files; + } + + // ── a command's page ───────────────────────────────────────────────────────────── + + private static String page(CommandSpec command, CommandSpec root, Path examples) { + boolean isRoot = command == root; + StringBuilder md = new StringBuilder(); + md.append("# ").append(command.qualifiedName()).append("\n\n"); + description(md, command); + section(md, command, "##", isRoot); + if (isRoot) { + md.append("## Global options\n\n"); + md.append("Every command takes these, before or after its name.\n\n"); + options(md, global(root)); + md.append("## Commands\n\n"); + md.append("| Command | What it does |\n|---|---|\n"); + for (CommandSpec sub : visible(root)) { + md.append("| [`").append(sub.qualifiedName()).append("`](").append(fileName(sub)).append(") | ") + .append(cell(firstSentence(sub))).append(" |\n"); + } + md.append('\n'); + exitStatus(md, root, "##"); + } + for (CommandSpec sub : visible(command)) { + if (isRoot) { + break; + } + md.append("## ").append(sub.qualifiedName()).append("\n\n"); + description(md, sub); + section(md, sub, "###", false); + } + String handWritten = read(examples); + if (handWritten != null) { + md.append(handWritten.strip()).append('\n'); + } + // picocli wraps the synopsis with the platform's line separator; a page has LF + // endings everywhere, as it is checked in. + return md.toString().replace("\r\n", "\n").replaceAll("\n{3,}", "\n\n").strip() + "\n"; + } + + /** Usage, arguments, options and exit status, under headings of {@code level}. */ + private static void section(StringBuilder md, CommandSpec command, String level, boolean isRoot) { + md.append(level).append(" Usage").append("\n\n```text\n") + .append(synopsis(command)).append("```\n\n"); + List positionals = command.positionalParameters().stream() + .filter(p -> !p.hidden()).toList(); + if (!positionals.isEmpty()) { + md.append(level).append(" Arguments\n\n| Argument | Description |\n|---|---|\n"); + for (PositionalParamSpec p : positionals) { + md.append("| `").append(p.paramLabel()).append(p.arity().max() > 1 ? "...`" : "`").append(" | ") + .append(cell(description(p))).append(" |\n"); + } + md.append('\n'); + } + List own = command.options().stream() + .filter(o -> !o.hidden() && !o.inherited() && !o.usageHelp() && !o.versionHelp()) + .filter(o -> !isRoot || !global(command).contains(o)) + .toList(); + if (!own.isEmpty()) { + md.append(level).append(isRoot ? " Run options" : " Options").append("\n\n"); + if (isRoot) { + md.append("With no command, `relix` is `relix run`, and takes its options:\n\n"); + } + options(md, own); + } + if (!isRoot) { + md.append("It also takes the [global options](relix.md#global-options).\n\n"); + } + if (!isRoot) { + exitStatus(md, command, level); + } + } + + private static void exitStatus(StringBuilder md, CommandSpec command, String level) { + Map exits = command.usageMessage().exitCodeList(); + if (!exits.isEmpty()) { + md.append(level).append(" Exit status\n\n| Status | Meaning |\n|---|---|\n"); + exits.forEach((code, meaning) -> md.append("| ").append(code.strip()).append(" | ") + .append(cell(meaning)).append(" |\n")); + md.append('\n'); + } + } + + private static void options(StringBuilder md, List options) { + md.append("| Option | Description |\n|---|---|\n"); + for (OptionSpec o : options) { + md.append("| ").append(optionNames(o)).append(" | ").append(cell(description(o))).append(" |\n"); + } + md.append('\n'); + } + + private static String optionNames(OptionSpec option) { + String label = option.typeInfo().isBoolean() || option.arity().max() == 0 ? "" : option.paramLabel(); + List names = new ArrayList<>(); + for (String name : option.names()) { + boolean last = name.equals(option.longestName()); + if (label.isEmpty() || !last) { + names.add("`" + name + "`"); + } else { + String separator = name.startsWith("--") ? "=" : " "; + names.add("`" + name + separator + label + "`"); + } + } + return String.join(", ", names); + } + + private static List global(CommandSpec root) { + CommandSpec mixin = root.mixins().get("options"); + Set names = mixin.options().stream().map(OptionSpec::longestName).collect(Collectors.toSet()); + List global = new ArrayList<>(); + for (OptionSpec o : root.options()) { + if (names.contains(o.longestName()) || o.usageHelp() || o.versionHelp()) { + global.add(o); + } + } + return global; + } + + private static void description(StringBuilder md, CommandSpec command) { + List code = new ArrayList<>(); + List prose = new ArrayList<>(); + for (String line : command.usageMessage().description()) { + for (String part : line.split("%n", -1)) { + if (part.startsWith(" ") && !part.isBlank()) { + flush(md, prose); + code.add(part.strip()); + continue; + } + if (!code.isEmpty()) { + md.append("```text\n").append(String.join("\n", code)).append("\n```\n\n"); + code.clear(); + } + if (part.isBlank()) { + flush(md, prose); + } else { + prose.add(part.strip()); + } + } + } + flush(md, prose); + if (!code.isEmpty()) { + md.append("```text\n").append(String.join("\n", code)).append("\n```\n\n"); + } + } + + private static void flush(StringBuilder md, List prose) { + if (!prose.isEmpty()) { + md.append(String.join(" ", prose)).append("\n\n"); + prose.clear(); + } + } + + private static String synopsis(CommandSpec command) { + command.usageMessage().autoWidth(false).width(WIDTH).abbreviateSynopsis(true); + Help help = new Help(command, Help.defaultColorScheme(Help.Ansi.OFF)); + return help.synopsis(0); + } + + private static String description(ArgSpec arg) { + return String.join(" ", arg.description()).replaceAll("\\s+", " ").strip(); + } + + private static String firstSentence(CommandSpec command) { + String text = String.join(" ", command.usageMessage().description()) + .replace("%n", " ").replaceAll("\\s+", " ").strip(); + int stop = text.indexOf(". "); + String sentence = stop < 0 ? text : text.substring(0, stop + 1); + return sentence.endsWith(".") ? sentence : sentence + "."; + } + + private static String cell(String text) { + return text.replace("|", "\\|"); + } + + private static List visible(CommandSpec command) { + return command.subcommands().entrySet().stream() + .filter(e -> e.getKey().equals(e.getValue().getCommandSpec().name())) + .map(e -> e.getValue().getCommandSpec()) + .filter(c -> !c.usageMessage().hidden()) + .toList(); + } + + /** {@code relix.md} for the root, {@code run.md} for {@code relix run}. */ + private static String fileName(CommandSpec command) { + return command.name() + ".md"; + } + + // ── index.json ─────────────────────────────────────────────────────────────────── + + /** + * The section's index as JSON: each page in reading order, with its chapter, its title + * and what the index says it covers, and for a command page the command and its + * subcommands, each with the anchor of its section. + */ + private static String index(Path readme, List commands) { + StringBuilder json = new StringBuilder(); + json.append("{\n \"title\": ").append(string(title(readme))).append(",\n \"pages\": [\n"); + List entries = new ArrayList<>(); + Map byFile = new LinkedHashMap<>(); + for (CommandSpec command : commands) { + byFile.put(COMMANDS + "/" + fileName(command), command); + } + String chapter = null; + for (String line : lines(readme)) { + Matcher heading = INDEX_CHAPTER.matcher(line); + if (heading.matches()) { + chapter = heading.group(1); + continue; + } + Matcher link = LINK.matcher(line); + if (chapter == null || !line.startsWith("|") || !link.find()) { + continue; + } + String path = link.group(2); + String title = link.group(1).replace("`", ""); + String[] cells = line.strip().split("(? 2 ? cells[2].strip().replace("\\|", "|") : ""; + StringBuilder entry = new StringBuilder(); + entry.append(" {\"path\": ").append(string(path)) + .append(", \"chapter\": ").append(string(chapter)) + .append(", \"title\": ").append(string(title)) + .append(", \"summary\": ").append(string(summary)); + CommandSpec command = byFile.get(path); + if (command != null) { + entry.append(", \"command\": ").append(string(command.qualifiedName())); + List subs = visible(command).stream() + .filter(c -> command.parent() != null) + .map(c -> "{\"command\": " + string(c.qualifiedName()) + ", \"anchor\": " + + string(slug(c.qualifiedName())) + "}") + .toList(); + if (!subs.isEmpty()) { + entry.append(", \"subcommands\": [").append(String.join(", ", subs)).append(']'); + } + } + entry.append('}'); + entries.add(entry.toString()); + } + json.append(String.join(",\n", entries)).append("\n ]\n}\n"); + return json.toString(); + } + + private static String title(Path readme) { + for (String line : lines(readme)) { + Matcher title = INDEX_TITLE.matcher(line); + if (title.matches()) { + return title.group(1); + } + } + throw new IllegalStateException(readme + " has no title"); + } + + private static String slug(String text) { + return text.toLowerCase(java.util.Locale.ROOT).replaceAll("[^a-z0-9]+", "-").replaceAll("(^-|-$)", ""); + } + + private static String string(String text) { + StringBuilder out = new StringBuilder("\""); + for (char c : text.toCharArray()) { + switch (c) { + case '"' -> out.append("\\\""); + case '\\' -> out.append("\\\\"); + case '\n' -> out.append("\\n"); + default -> { + if (c < 0x20) { + out.append(String.format("\\u%04x", (int) c)); + } else { + out.append(c); + } + } + } + } + return out.append('"').toString(); + } + + private static List lines(Path file) { + try { + return Files.readAllLines(file, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException("cannot read " + file, e); + } + } + + private static String read(Path file) { + if (!Files.isRegularFile(file)) { + return null; + } + try { + return Files.readString(file, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException("cannot read " + file, e); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java b/src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java new file mode 100644 index 0000000..09032fb --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java @@ -0,0 +1,294 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import com.darkcollective.relix.cli.Main; +import com.darkcollective.relix.cli.io.Host; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.PosixFilePermissions; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +/** + * Runs the guide's examples where a reader would: in {@code ~/shop}, a copy of the fixture + * tree under {@code src/test/resources/docs-fixture/home}, which is the home directory. + * + *

The fixture is the shop the pages describe: a user-level {@code ~/.relix} with a + * table and a profile, the project {@code ~/shop} with its catalog, data files, scripts and + * profiles, and {@code ~/shop/reports}, a sub-project that overlays {@code Orders}. Setting + * it up does what a fresh copy cannot carry: {@code @HOME@} in a profile becomes the home + * directory's path, every {@code profiles.json} is made private, and the H2 database the + * {@code warehouse} connection opens is written, with user {@code shop} and the password + * {@code ~/.relix/profiles.json} holds. + * + *

An example that is a plain {@code relix} command ({@link ShellCommand}) runs in this + * JVM, with standard output a terminal and standard input a terminal unless the example + * redirects it, as a reader typing it would have them. Anything else runs through + * {@code bash}, with a {@code relix} on its {@code PATH} that starts the command in a JVM + * of its own; there standard output is a pipe, as it is for a reader's pipeline. Both get + * the same environment: the fixture's {@code HOME}, {@code NO_COLOR=1}, {@code TZ=UTC}, + * and nothing of the user running the build. + */ +final class ExampleRunner implements AutoCloseable { + + private static final String FIXTURE = "/docs-fixture/home"; + + /** The H2 database's credentials, as the fixture's profiles give them. */ + private static final String WAREHOUSE_USER = "shop"; + private static final String WAREHOUSE_PASSWORD = "s3cret"; + + private final Path home; + private final Path shop; + private final Path bin; + private final Map environment; + private final Path stdin; + private final Path stdout; + + /** + * A fresh copy of the fixture. + * + * @param root an empty directory to put it in + * @param trusted whether {@code ~/shop} and {@code ~/shop/reports} start trusted + */ + ExampleRunner(Path root, boolean trusted) { + try { + this.home = Files.createDirectories(root.resolve("home")).toRealPath(); + copy(fixture(), home); + this.shop = home.resolve("shop"); + for (Path profiles : List.of(home.resolve(".relix/profiles.json"), shop.resolve(".relix/profiles.json"))) { + Files.writeString(profiles, Files.readString(profiles).replace("@HOME@", home.toString())); + Files.setPosixFilePermissions(profiles, PosixFilePermissions.fromString("rw-------")); + } + warehouse(shop.resolve("data/warehouse")); + this.bin = Files.createDirectories(root.resolve("bin")); + this.environment = environment(home, bin); + this.stdin = root.resolve("stdin.txt"); + this.stdout = root.resolve("stdout.txt"); + shim(bin.resolve("relix"), home); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + if (trusted) { + for (Path project : List.of(shop, shop.resolve("reports"))) { + Output trust = runInProcess(new ShellCommand(Map.of(), List.of("catalog", "trust", project.toString()), + null, null, false, false)); + if (trust.status() != 0) { + throw new IllegalStateException("cannot trust " + project + ": " + trust); + } + } + } + } + + /** {@return the fixture's home directory} */ + Path home() { + return home; + } + + /** + * What an example did. + * + * @param status its exit status + * @param out what it wrote to standard output, with the home directory written {@code ~} + * @param err what it wrote to standard error + * @param shell whether it ran through {@code bash} + */ + record Output(int status, String out, String err, boolean shell) { + @Override + public String toString() { + return (shell ? "bash" : "in-process") + ", exit " + status + "\n── stdout ──\n" + out + + "── stderr ──\n" + err; + } + } + + /** + * Runs an example. + * + * @param command the example's text + * @return what it did + */ + Output run(String command) { + return ShellCommand.parse(command).map(this::runInProcess).orElseGet(() -> runInBash(command)); + } + + private Output runInProcess(ShellCommand command) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayOutputStream err = new ByteArrayOutputStream(); + OutputStream errTarget = command.mergeStderr() ? out : err; + InputStream in; + try { + in = command.heredoc() != null + ? new ByteArrayInputStream(command.heredoc().getBytes(StandardCharsets.UTF_8)) + : command.stdinFile() != null ? Files.newInputStream(shop.resolve(command.stdinFile())) + : new ByteArrayInputStream(new byte[0]); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + Map env = new HashMap<>(environment); + env.putAll(command.environment()); + boolean stdinIsTerminal = command.heredoc() == null && command.stdinFile() == null; + Host host = new Host(in, out, new PrintStream(errTarget, true, StandardCharsets.UTF_8), env, shop, home, + stdinIsTerminal, true); + // The engine finds its driver directory and file cache under the JVM's user.home, + // not $HOME, so the fixture is made that too for the run, as the bash shim does. + String userHome = System.getProperty("user.home"); + int status; + try { + System.setProperty("user.home", home.toString()); + status = Main.run(host, command.arguments().toArray(String[]::new)); + } finally { + System.setProperty("user.home", userHome); + } + String stdout = out.toString(StandardCharsets.UTF_8); + if (command.echoStatus()) { + stdout += "exit " + status + "\n"; + status = 0; + } + return new Output(status, tilde(stdout), err.toString(StandardCharsets.UTF_8), false); + } + + private Output runInBash(String command) { + // Descriptors 8 and 9 are the example's own standard input and output, which the + // shim compares a stage's streams with to tell where a reader's terminal would be. + ProcessBuilder builder = new ProcessBuilder("bash", "-c", "exec 8<&0 9>&1\n" + command) + .directory(shop.toFile()); + builder.environment().clear(); + builder.environment().putAll(environment); + try { + Files.write(stdin, new byte[0]); + Files.write(stdout, new byte[0]); + Path err = Files.createTempFile(bin.getParent(), "stderr", ".txt"); + Process process = builder.redirectInput(stdin.toFile()).redirectOutput(stdout.toFile()) + .redirectError(err.toFile()).start(); + if (!process.waitFor(2, TimeUnit.MINUTES)) { + process.destroyForcibly(); + throw new IllegalStateException("timed out: " + command); + } + return new Output(process.exitValue(), tilde(Files.readString(stdout)), Files.readString(err), true); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + + /** The home directory's path, which differs on every run, written as a reader's shell abbreviates it. */ + private String tilde(String text) { + return text.replace(home.toString(), "~"); + } + + @Override + public void close() { + // Nothing stays open: each example's JVM or session ends with it. + } + + private static Map environment(Path home, Path bin) { + Map env = new HashMap<>(); + // The relix under test first; then where jq, find and xargs are on a build machine. + env.put("PATH", bin + ":/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin"); + env.put("HOME", home.toString()); + env.put("NO_COLOR", "1"); + env.put("TZ", "UTC"); + // A UTF-8 locale, as a reader's terminal has: on Linux the JVM decodes its + // arguments by the locale, and `relix doc σ` would otherwise not say σ. + env.put("LC_ALL", System.getProperty("os.name").startsWith("Mac") ? "en_US.UTF-8" : "C.UTF-8"); + env.put("PAGER", "cat"); + return env; + } + + /** + * A {@code relix} for {@code bash}: this JVM's class path and {@link TerminalMain}, the + * fixture as the home directory a JDBC driver's {@code ~} means too, and standard input + * and output taken for terminals when they are the example's own — descriptors 8 and + * 9 — as they would be a reader's. + */ + private static void shim(Path file, Path home) throws IOException { + String java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + String script = "#!/bin/sh\n" + + "in=false; [ /dev/fd/0 -ef /dev/fd/8 ] && in=true\n" + + "out=false; [ /dev/fd/1 -ef /dev/fd/9 ] && out=true\n" + + "exec " + quote(java) + " -XX:TieredStopAtLevel=1 -Xshare:auto" + // The launcher's streams are UTF-8 whatever the locale, so a glyph in a + // message reads as it does on a reader's terminal. + + " -Dstdout.encoding=UTF-8 -Dstderr.encoding=UTF-8" + + " -Duser.home=" + quote(home.toString()) + + " -Drelix.docs.stdinIsTerminal=$in -Drelix.docs.stdoutIsTerminal=$out" + + " -cp " + quote(System.getProperty("java.class.path")) + + " " + TerminalMain.class.getName() + " \"$@\"\n"; + Files.writeString(file, script); + Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rwxr-xr-x")); + } + + private static String quote(String word) { + return "'" + word.replace("'", "'\\''") + "'"; + } + + private static Path fixture() { + URL url = ExampleRunner.class.getResource(FIXTURE); + if (url == null) { + throw new IllegalStateException("no " + FIXTURE + " on the test class path"); + } + try { + return Path.of(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalStateException(e); + } + } + + private static void copy(Path from, Path to) throws IOException { + try (Stream walk = Files.walk(from)) { + for (Path source : walk.toList()) { + Path target = to.resolve(from.relativize(source).toString()); + if (Files.isDirectory(source)) { + Files.createDirectories(target); + } else { + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + } + } + } + } + + /** The stock database the fixture's {@code warehouse} connection opens. */ + private static void warehouse(Path file) { + try (Connection connection = DriverManager.getConnection("jdbc:h2:" + file, WAREHOUSE_USER, WAREHOUSE_PASSWORD); + Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE STOCK (SKU VARCHAR(16) PRIMARY KEY, ON_HAND INT)"); + statement.execute("INSERT INTO STOCK VALUES ('mug', 12), ('tee', 0), ('tote', 31)"); + } catch (SQLException e) { + throw new IllegalStateException("cannot write the warehouse database: " + e.getMessage(), e); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/GuideExamplesTest.java b/src/test/java/com/darkcollective/relix/cli/docs/GuideExamplesTest.java new file mode 100644 index 0000000..0b43af0 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/GuideExamplesTest.java @@ -0,0 +1,134 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Runs every example in the command-line section of the programming guide and compares + * what it writes to standard output, and its exit status, with what the page shows + * (design §7.4), so that no page can be published with an example the command does not + * honour. + * + *

An example is a {@code shell} block; the plain block on the line after it is its + * standard output, and an example with none must print nothing. Its exit status must be + * 0, unless it ends {@code ; echo "exit $?"}, which prints the status for the page to + * show. Standard error is not compared, except where an example sends it to standard + * output with {@code 2>&1}. + * + *

Each page runs in a fresh copy of the fixture ({@link ExampleRunner}), its examples + * in order, so that one may build on what an earlier one did. The project starts trusted, + * except on the pages listed in {@link #UNTRUSTED}, which show trusting it. + * + *

Comparison forgives trailing whitespace on a line, which a table pads with and an + * editor strips, and the path of the fixture's home directory, written {@code ~}. Nothing + * else. + * + *

The examples run where {@code bash} is a POSIX shell, so not on Windows; the pages + * are written for one. Every other platform the build runs on runs them all. + */ +@DisabledOnOs(OS.WINDOWS) +@DisplayName("The command-line guide's examples print what the pages show") +class GuideExamplesTest { + + /** Pages whose project starts untrusted, because they show trusting it. */ + private static final Set UNTRUSTED = Set.of("catalog.md"); + + @TempDir + Path root; + + @TestFactory + List everyExample() { + Path directory = GuidePages.directory(); + List pages = new ArrayList<>(); + for (String page : GuidePages.files(directory)) { + List examples = GuidePages.examples(GuidePages.lines(directory.resolve(page))); + if (examples.isEmpty()) { + continue; + } + AtomicReference runner = new AtomicReference<>(); + List tests = new ArrayList<>(); + for (GuidePages.Example example : examples) { + tests.add(DynamicTest.dynamicTest(page + ":" + example.line(), () -> { + if (runner.get() == null) { + Path fixture = Files.createDirectories(root.resolve(page.replace('/', '_'))); + runner.set(new ExampleRunner(fixture, !UNTRUSTED.contains(page))); + } + check(page, example, runner.get().run(example.command())); + })); + } + pages.add(DynamicContainer.dynamicContainer(page, tests)); + } + assertThat(pages).as("pages with examples").isNotEmpty(); + return pages; + } + + private static void check(String page, GuidePages.Example example, ExampleRunner.Output output) { + String where = page + ":" + example.line() + "\n" + example.command() + "\n"; + assertThat(output.status()) + .as(where + "exits 0 (end it with ; echo \"exit $?\" to show another status)\n" + output) + .isZero(); + assertThat(normalise(output.out())) + .as(where + output) + .isEqualTo(example.expected() == null ? "" : normalise(example.expected())); + } + + private static String normalise(String text) { + return text.lines().map(String::stripTrailing).reduce((a, b) -> a + "\n" + b).orElse("").stripTrailing(); + } + + @Test + @DisplayName("every command a page shows a reader is a shell example, which runs") + void noUnrunCommandBlocks() { + Path directory = GuidePages.directory(); + List found = new ArrayList<>(); + for (String page : GuidePages.files(directory)) { + for (GuidePages.Block block : GuidePages.blocks(GuidePages.lines(directory.resolve(page)))) { + if (Set.of("bash", "sh", "console", "zsh", "shell-session").contains(block.lang())) { + found.add(page + ":" + block.line() + " (" + block.lang() + ")"); + } + } + } + assertThat(found).as("blocks the examples test would not run; write them as ```shell").isEmpty(); + } + + static String read(Path file) { + try { + return Files.readString(file); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/GuidePages.java b/src/test/java/com/darkcollective/relix/cli/docs/GuidePages.java new file mode 100644 index 0000000..46cfcf6 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/GuidePages.java @@ -0,0 +1,219 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +/** + * The command-line section of the programming guide, {@code docs/guide/command-line}: its + * pages, and the fenced blocks and links in each. + * + *

The pages are Markdown in the dialect the site and the engine's manuals share. A + * fenced block is an example when its language is {@code shell}; the plain fence + * immediately after it, if there is one, is what the example writes to standard output. + */ +final class GuidePages { + + /** The section's directory, relative to the repository. */ + static final Path DIRECTORY = Path.of("docs/guide/command-line"); + + /** The language of a block the examples test runs. */ + static final String EXAMPLE = "shell"; + + private static final Pattern FENCE = Pattern.compile("^```\\s*([A-Za-z0-9_-]*)\\s*$"); + private static final Pattern LINK = Pattern.compile("\\[([^\\]]*)]\\(([^)\\s]+)\\)"); + private static final Pattern HEADING = Pattern.compile("^(#{1,6})\\s+(.+?)\\s*$"); + + private GuidePages() { + } + + /** + * A fenced block. + * + * @param lang its language, empty for a plain fence + * @param body its lines, joined + * @param line the line of its opening fence, from 1 + */ + record Block(String lang, String body, int line) { + } + + /** + * An example and what it prints. + * + * @param line the line of its opening fence + * @param command the block's text + * @param expected the plain block after it, or {@code null} when it prints nothing + */ + record Example(int line, String command, String expected) { + } + + /** + * A link written in a page. + * + * @param line the line it is on + * @param href its target, as written + */ + record Link(int line, String href) { + } + + /** {@return the section's directory: {@code relix.cli.guide}, or the repository's} */ + static Path directory() { + String property = System.getProperty("relix.cli.guide"); + return property != null ? Path.of(property) : DIRECTORY; + } + + /** + * Every Markdown file of the section, as paths relative to its directory. + * + * @param directory the section's directory + * @return the files, in path order + */ + static List files(Path directory) { + try (Stream walk = Files.walk(directory)) { + return walk.filter(p -> p.toString().endsWith(".md")) + .map(p -> directory.relativize(p).toString().replace('\\', '/')) + .sorted() + .toList(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + static List lines(Path file) { + try { + return Files.readAllLines(file, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * The fenced blocks of a page. + * + * @param lines the page + * @return its blocks, in order + */ + static List blocks(List lines) { + List blocks = new ArrayList<>(); + for (int i = 0; i < lines.size(); i++) { + Matcher open = FENCE.matcher(lines.get(i)); + if (!open.matches()) { + continue; + } + int start = i; + List body = new ArrayList<>(); + for (i++; i < lines.size() && !lines.get(i).strip().equals("```"); i++) { + body.add(lines.get(i)); + } + blocks.add(new Block(open.group(1), String.join("\n", body), start + 1)); + } + return blocks; + } + + /** + * The examples of a page: each {@code shell} block, paired with the plain block that + * follows on the very next line. + * + * @param lines the page + * @return its examples, in order + */ + static List examples(List lines) { + List blocks = blocks(lines); + List examples = new ArrayList<>(); + for (int i = 0; i < blocks.size(); i++) { + Block block = blocks.get(i); + if (!block.lang().equals(EXAMPLE)) { + continue; + } + String expected = null; + if (i + 1 < blocks.size()) { + Block next = blocks.get(i + 1); + int closing = block.line() + (int) block.body().lines().count() + 1; + if (next.lang().isEmpty() && next.line() == closing + 1) { + expected = next.body(); + i++; + } + } + examples.add(new Example(block.line(), block.body(), expected)); + } + return examples; + } + + /** + * The links of a page, outside its fenced blocks and code spans. + * + * @param lines the page + * @return its links, in order + */ + static List links(List lines) { + List links = new ArrayList<>(); + boolean fenced = false; + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i); + if (FENCE.matcher(line).matches()) { + fenced = !fenced; + continue; + } + if (fenced) { + continue; + } + Matcher link = LINK.matcher(line.replaceAll("`[^`]*`", "")); + while (link.find()) { + links.add(new Link(i + 1, link.group(2))); + } + } + return links; + } + + /** + * The anchors a page's headings get on the site: the heading's text, lower-cased, with + * each run of anything but a letter or digit made a hyphen. + * + * @param lines the page + * @return its anchors + */ + static List anchors(List lines) { + List anchors = new ArrayList<>(); + boolean fenced = false; + for (String line : lines) { + if (FENCE.matcher(line).matches()) { + fenced = !fenced; + continue; + } + Matcher heading = HEADING.matcher(line); + if (!fenced && heading.matches()) { + anchors.add(slug(heading.group(2).replace("`", ""))); + } + } + return anchors; + } + + static String slug(String text) { + String slug = text.toLowerCase(java.util.Locale.ROOT) + .replaceAll("[^a-z0-9]+", "-") + .replaceAll("(^-|-$)", ""); + return slug.isEmpty() ? "section" : slug; + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/GuideStructureTest.java b/src/test/java/com/darkcollective/relix/cli/docs/GuideStructureTest.java new file mode 100644 index 0000000..0e9a0d9 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/GuideStructureTest.java @@ -0,0 +1,169 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import com.darkcollective.relix.cli.packaging.CommandPages; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Holds the command-line section of the programming guide together: every command has a + * page and the pages are what the command model makes of it now, the index lists every + * page, and no link between pages, or into the engine's manuals, is broken. + * + *

The section is rendered by the site inside the engine's programming guide, at + * {@code guide/command-line/}, so a link out of it is written as if it sat in the + * engine's {@code docs/guide/command-line}: {@code ../secrets.md} is a guide page and + * {@code ../../reference/README.md} the language reference. Such a link is resolved here + * against the pinned engine's manuals, which the build unpacks. + */ +@DisplayName("The command-line guide") +class GuideStructureTest { + + private static final Pattern INDEX_ROW_LINK = Pattern.compile("^\\|\\s*\\[[^\\]]+]\\(([^)#]+\\.md)\\)"); + + private final Path guide = GuidePages.directory(); + + @Test + @DisplayName("has the command pages and index the command model makes now") + void commandPagesAreCurrent() { + Map generated = CommandPages.generate(examples(), guide); + List stale = new ArrayList<>(); + generated.forEach((path, text) -> { + Path file = guide.resolve(path); + if (!Files.isRegularFile(file) || !GuideExamplesTest.read(file).equals(text)) { + stale.add(path); + } + }); + for (String page : GuidePages.files(guide)) { + if (page.startsWith(CommandPages.COMMANDS + "/") && !generated.containsKey(page)) { + stale.add(page + " (no such command)"); + } + } + assertThat(stale).as("stale command pages; run ./gradlew commandPages").isEmpty(); + } + + @Test + @DisplayName("gives every hand-written part of a command page a command") + void everyExamplesFileHasACommand() { + Set commands = new LinkedHashSet<>(); + for (String path : CommandPages.generate(examples(), guide).keySet()) { + if (path.startsWith(CommandPages.COMMANDS + "/")) { + commands.add(path.substring(CommandPages.COMMANDS.length() + 1)); + } + } + List orphans = new ArrayList<>(); + for (String file : GuidePages.files(examples())) { + if (!commands.contains(file)) { + orphans.add(file); + } + } + assertThat(orphans).as("files in docs/command-examples naming no command").isEmpty(); + } + + @Test + @DisplayName("lists every page in its index, and links nothing that is not there") + void indexListsEveryPage() { + Set listed = new LinkedHashSet<>(); + for (String line : GuidePages.lines(guide.resolve("README.md"))) { + Matcher row = INDEX_ROW_LINK.matcher(line); + if (row.find()) { + listed.add(row.group(1)); + } + } + Set pages = new LinkedHashSet<>(GuidePages.files(guide)); + pages.remove("README.md"); + assertThat(listed).as("pages the index lists").containsExactlyInAnyOrderElementsOf(pages); + } + + @Test + @DisplayName("has no broken link, to a page or to a heading") + void noBrokenLinks() { + Path engine = engineDocs(); + List broken = new ArrayList<>(); + for (String page : GuidePages.files(guide)) { + List lines = GuidePages.lines(guide.resolve(page)); + for (GuidePages.Link link : GuidePages.links(lines)) { + String problem = problem(page, link.href(), engine); + if (problem != null) { + broken.add(page + ":" + link.line() + ": " + link.href() + ": " + problem); + } + } + } + assertThat(broken).isEmpty(); + } + + /** What is wrong with a link written on {@code page}, or {@code null} when nothing is. */ + private String problem(String page, String href, Path engine) { + if (href.startsWith("https://") || href.startsWith("http://") || href.startsWith("mailto:")) { + return null; + } + int hash = href.indexOf('#'); + String target = hash < 0 ? href : href.substring(0, hash); + String anchor = hash < 0 ? null : href.substring(hash + 1); + Path file; + if (target.isEmpty()) { + file = guide.resolve(page); + } else { + if (!target.endsWith(".md")) { + return "not a page"; + } + // The section as it sits in the engine's tree: docs/guide/command-line/. + Path virtual = Path.of("docs/guide/command-line").resolve(page).resolveSibling(target).normalize(); + String path = virtual.toString().replace('\\', '/'); + if (path.equals("docs/guide/command-line/README.md")) { + return "the index is not a page of its own on the site; link a page"; + } + if (path.startsWith("docs/guide/command-line/")) { + file = guide.resolve(path.substring("docs/guide/command-line/".length())); + } else if (path.startsWith("docs/")) { + file = engine.resolve(path); + } else { + return "outside the manuals"; + } + } + if (!Files.isRegularFile(file)) { + return "no such page"; + } + if (anchor != null && !GuidePages.anchors(GuidePages.lines(file)).contains(anchor)) { + return "no such heading"; + } + return null; + } + + private static Path examples() { + String property = System.getProperty("relix.cli.commandExamples"); + return Path.of(property != null ? property : CommandPages.EXAMPLES); + } + + private static Path engineDocs() { + String property = System.getProperty("relix.engine.docs"); + assertThat(property).as("relix.engine.docs, the unpacked engine manuals").isNotBlank(); + return Path.of(property); + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/ShellCommand.java b/src/test/java/com/darkcollective/relix/cli/docs/ShellCommand.java new file mode 100644 index 0000000..544ff67 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/ShellCommand.java @@ -0,0 +1,223 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * An example that is one {@code relix} command and nothing a shell would have to do for + * it, so that it can run inside the test's JVM rather than through {@code bash}. + * + *

The forms recognised are the ones the guide's pages write: + * + * 

+ * [NAME=VALUE ...] relix ARG ... [< FILE | <<'EOF' ... EOF] [2>&1 | 2>/dev/null] [; echo "exit $?"]
+ *
+ * + *

An argument may be quoted with single or double quotes or escaped with a backslash, + * and a line may continue with a trailing backslash. Anything else a shell would act on + * — a pipe, an expansion, a glob, a {@code ~}, a second command — makes the example not + * one of these, and it runs through {@code bash}. The test errs that way: an example + * this class wrongly took for a plain command would run differently from how a reader's + * shell runs it, while one it wrongly passes to {@code bash} is only slower. + * + * @param environment the {@code NAME=VALUE} words before {@code relix} + * @param arguments the words after it + * @param stdinFile the file standard input is redirected from, or {@code null} + * @param heredoc the here-document standard input holds, or {@code null} + * @param mergeStderr whether {@code 2>&1} sends standard error to standard output + * @param echoStatus whether the command is followed by {@code echo "exit $?"} + */ +record ShellCommand(Map environment, List arguments, String stdinFile, + String heredoc, boolean mergeStderr, boolean echoStatus) { + + /** What prints the exit status after an example, so the page can show it. */ + static final String ECHO_STATUS = "; echo \"exit $?\""; + + private static final Pattern ASSIGNMENT = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*=.*", Pattern.DOTALL); + private static final Pattern HEREDOC = Pattern.compile("<<-?'([A-Za-z_]+)'"); + + /** + * Reads an example as a plain {@code relix} command, if it is one. + * + * @param text the example, as the page's {@code shell} block holds it + * @return the command, or empty when a shell has to run it + */ + static Optional parse(String text) { + String body = text.strip(); + String heredoc = null; + int newline = body.indexOf('\n'); + Matcher here = HEREDOC.matcher(newline < 0 ? body : body.substring(0, newline)); + if (here.find()) { + // relix ... <<'EOF' on the first line, the document after it, EOF alone last. + String delimiter = here.group(1); + List lines = body.lines().toList(); + if (lines.size() < 2 || !lines.getLast().equals(delimiter)) { + return Optional.empty(); + } + heredoc = String.join("\n", lines.subList(1, lines.size() - 1)) + "\n"; + body = lines.getFirst().substring(0, here.start()) + lines.getFirst().substring(here.end()); + } else { + body = body.replace("\\\n", " "); + if (body.contains("\n")) { + return Optional.empty(); + } + } + + boolean echoStatus = false; + if (body.endsWith(ECHO_STATUS)) { + echoStatus = true; + body = body.substring(0, body.length() - ECHO_STATUS.length()); + } + + List words = words(body); + if (words == null) { + return Optional.empty(); + } + Map environment = new LinkedHashMap<>(); + int i = 0; + while (i < words.size() && ASSIGNMENT.matcher(words.get(i)).matches()) { + String word = words.get(i++); + int equals = word.indexOf('='); + environment.put(word.substring(0, equals), word.substring(equals + 1)); + } + if (i >= words.size() || !words.get(i).equals("relix")) { + return Optional.empty(); + } + i++; + + List arguments = new ArrayList<>(); + String stdinFile = null; + boolean mergeStderr = false; + for (; i < words.size(); i++) { + String word = words.get(i); + switch (word) { + case "\u0000<" -> { + if (i + 1 >= words.size() || stdinFile != null || heredoc != null) { + return Optional.empty(); + } + stdinFile = words.get(++i); + } + case "\u00002>&1" -> mergeStderr = true; + case "\u00002>/dev/null" -> { + // Standard error is not compared, so discarding it changes nothing. + } + default -> { + if (word.startsWith("\u0000")) { + return Optional.empty(); + } + arguments.add(word); + } + } + } + return Optional.of(new ShellCommand(environment, arguments, stdinFile, heredoc, mergeStderr, echoStatus)); + } + + /** + * Splits a line into words as a shell would, or returns {@code null} when the line + * holds anything a shell would do more with than quote. The redirections this class + * knows come back as words of their own, marked with a leading NUL that no word typed + * on a page can hold. + */ + private static List words(String line) { + List words = new ArrayList<>(); + StringBuilder word = null; + int i = 0; + while (i < line.length()) { + char c = line.charAt(i); + if (c == ' ' || c == '\t') { + if (word != null) { + words.add(word.toString()); + word = null; + } + i++; + continue; + } + if (word == null) { + if (line.startsWith("2>&1", i) || line.startsWith("2>/dev/null", i)) { + String redirect = line.startsWith("2>&1", i) ? "2>&1" : "2>/dev/null"; + int end = i + redirect.length(); + if (end == line.length() || line.charAt(end) == ' ') { + words.add("\u0000" + redirect); + i = end; + continue; + } + } + if (c == '<') { + words.add("\u0000<"); + i++; + continue; + } + if (c == '~' || c == '#') { + return null; + } + word = new StringBuilder(); + } + switch (c) { + case '\'' -> { + int close = line.indexOf('\'', i + 1); + if (close < 0) { + return null; + } + word.append(line, i + 1, close); + i = close + 1; + } + case '"' -> { + i++; + while (i < line.length() && line.charAt(i) != '"') { + char d = line.charAt(i); + if (d == '$' || d == '`' || d == '!') { + return null; + } + if (d == '\\' && i + 1 < line.length() && "\"\\".indexOf(line.charAt(i + 1)) >= 0) { + d = line.charAt(++i); + } + word.append(d); + i++; + } + if (i >= line.length()) { + return null; + } + i++; + } + case '\\' -> { + if (i + 1 >= line.length()) { + return null; + } + word.append(line.charAt(i + 1)); + i += 2; + } + default -> { + if ("|&;<>()$`*?[]{}!".indexOf(c) >= 0) { + return null; + } + word.append(c); + i++; + } + } + } + if (word != null) { + words.add(word.toString()); + } + return words; + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/TerminalMain.java b/src/test/java/com/darkcollective/relix/cli/docs/TerminalMain.java new file mode 100644 index 0000000..a722193 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/TerminalMain.java @@ -0,0 +1,54 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import com.darkcollective.relix.cli.Main; +import com.darkcollective.relix.cli.io.Host; +import com.darkcollective.relix.cli.io.Interruption; + +import java.io.FileDescriptor; +import java.io.FileOutputStream; +import java.nio.file.Path; + +/** + * The {@code relix} an example's {@code bash} runs: the command's own entry point, over a + * host whose standard streams are terminals where a reader's would be. + * + *

A reader typing {@code relix -e … | jq} has a terminal on {@code relix}'s standard + * input, and one on {@code jq}'s standard output; the example's run has files there, which + * {@link ExampleRunner} gives the shim. The shim compares each stream with them and says + * which is "the terminal" in two system properties, so that the command chooses the + * format, and whether to read a script from standard input, as it would for the reader. + */ +public final class TerminalMain { + + private TerminalMain() { + } + + /** + * Runs the command and exits with its status. + * + * @param args the command line + */ + public static void main(String[] args) { + Interruption interruption = new Interruption(); + Runtime.getRuntime().addShutdownHook(new Thread(interruption::interrupt, "relix-interrupt")); + Host host = new Host(System.in, new FileOutputStream(FileDescriptor.out), System.err, System.getenv(), + Path.of(System.getProperty("user.dir")), Path.of(System.getProperty("user.home")), + Boolean.getBoolean("relix.docs.stdinIsTerminal"), Boolean.getBoolean("relix.docs.stdoutIsTerminal")); + System.exit(Main.run(host, interruption, args)); + } +} diff --git a/src/test/resources/docs-fixture/home/.relix/catalog/00-regions.relix b/src/test/resources/docs-fixture/home/.relix/catalog/00-regions.relix new file mode 100644 index 0000000..b22f3d6 --- /dev/null +++ b/src/test/resources/docs-fixture/home/.relix/catalog/00-regions.relix @@ -0,0 +1,8 @@ +-- Your own tables: every directory under your home sees them. +Regions := [ +| city | region | +|--------|------------| +| London | South East | +| Leeds | North | +| York | North | +]; diff --git a/src/test/resources/docs-fixture/home/.relix/profiles.json b/src/test/resources/docs-fixture/home/.relix/profiles.json new file mode 100644 index 0000000..c289ff4 --- /dev/null +++ b/src/test/resources/docs-fixture/home/.relix/profiles.json @@ -0,0 +1,3 @@ +{ + "dev": { "WAREHOUSE_PASSWORD": "s3cret" } +} diff --git a/src/test/resources/docs-fixture/home/shop/.relix/catalog/00-sources.relix b/src/test/resources/docs-fixture/home/shop/.relix/catalog/00-sources.relix new file mode 100644 index 0000000..bc6ca4f --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/.relix/catalog/00-sources.relix @@ -0,0 +1,10 @@ +-- The shop's data. A relative path resolves against this file's directory. +source Customers from csv("../../data/customers.csv") { + header: true, + schema: { customer_id: NUMBER, name: STRING, city: STRING } +}; + +source Orders from csv("../../data/orders.csv") { + header: true, + schema: { order_id: NUMBER, customer_id: NUMBER, amount: NUMBER, status: STRING, placed: DATE } +}; diff --git a/src/test/resources/docs-fixture/home/shop/.relix/catalog/10-views.relix b/src/test/resources/docs-fixture/home/shop/.relix/catalog/10-views.relix new file mode 100644 index 0000000..8a7cd7f --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/.relix/catalog/10-views.relix @@ -0,0 +1,5 @@ +Completed := { SELECT status = 'completed' (Orders) }; + +Spend := { GROUP customer_id, SUM(amount) -> total (Completed) }; + +BigSpenders := { PROJECT name, total (Customers JOIN (SELECT total >= 200 (Spend))) }; diff --git a/src/test/resources/docs-fixture/home/shop/.relix/catalog/20-warehouse.relix b/src/test/resources/docs-fixture/home/shop/.relix/catalog/20-warehouse.relix new file mode 100644 index 0000000..cb4e96c --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/.relix/catalog/20-warehouse.relix @@ -0,0 +1,11 @@ +-- The stock database. Its address and credentials come from placeholders. +connection warehouse from database { + url: "${WAREHOUSE_URL}", + user: "${WAREHOUSE_USER}", + password: "${WAREHOUSE_PASSWORD}" +}; + +source Stock from warehouse { + table: "STOCK", + schema: { sku: STRING, on_hand: NUMBER } +}; diff --git a/src/test/resources/docs-fixture/home/shop/.relix/profiles.json b/src/test/resources/docs-fixture/home/shop/.relix/profiles.json new file mode 100644 index 0000000..3ee415f --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/.relix/profiles.json @@ -0,0 +1,6 @@ +{ + "dev": { + "WAREHOUSE_URL": "jdbc:h2:@HOME@/shop/data/warehouse", + "WAREHOUSE_USER": "shop" + } +} diff --git a/src/test/resources/docs-fixture/home/shop/data/customers.csv b/src/test/resources/docs-fixture/home/shop/data/customers.csv new file mode 100644 index 0000000..1124ade --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/data/customers.csv @@ -0,0 +1,5 @@ +customer_id,name,city +1,Alice,London +2,Bob,Leeds +3,Carol,London +4,Dan,York diff --git a/src/test/resources/docs-fixture/home/shop/data/orders.csv b/src/test/resources/docs-fixture/home/shop/data/orders.csv new file mode 100644 index 0000000..aa5f79c --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/data/orders.csv @@ -0,0 +1,5 @@ +order_id,customer_id,amount,status,placed +1,1,120,completed,2026-09-02 +2,1,80,completed,2026-09-14 +3,2,50,pending,2026-09-20 +4,3,200,completed,2026-10-01 diff --git a/src/test/resources/docs-fixture/home/shop/data/returns.ndjson b/src/test/resources/docs-fixture/home/shop/data/returns.ndjson new file mode 100644 index 0000000..1131b81 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/data/returns.ndjson @@ -0,0 +1,2 @@ +{"order_id": 2, "reason": "damaged", "refund": {"amount": 80, "currency": "GBP"}} +{"order_id": 4, "reason": "late", "refund": {"amount": 20, "currency": "GBP"}} diff --git a/src/test/resources/docs-fixture/home/shop/reports/.relix/catalog/10-sample.relix b/src/test/resources/docs-fixture/home/shop/reports/.relix/catalog/10-sample.relix new file mode 100644 index 0000000..aa0a3b4 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/reports/.relix/catalog/10-sample.relix @@ -0,0 +1,5 @@ +-- Reports are written against a fixed sample rather than the live orders. +source Orders from csv("../../sample-orders.csv") { + header: true, + schema: { order_id: NUMBER, customer_id: NUMBER, amount: NUMBER, status: STRING, placed: DATE } +}; diff --git a/src/test/resources/docs-fixture/home/shop/reports/sample-orders.csv b/src/test/resources/docs-fixture/home/shop/reports/sample-orders.csv new file mode 100644 index 0000000..5ce152b --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/reports/sample-orders.csv @@ -0,0 +1,3 @@ +order_id,customer_id,amount,status,placed +101,1,500,completed,2026-01-05 +102,4,250,completed,2026-01-06 diff --git a/src/test/resources/docs-fixture/home/shop/scripts/broken.relix b/src/test/resources/docs-fixture/home/shop/scripts/broken.relix new file mode 100644 index 0000000..995f15d --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/scripts/broken.relix @@ -0,0 +1 @@ +query { SELECT amount > 100 (Ordrs) }; diff --git a/src/test/resources/docs-fixture/home/shop/scripts/by-city.relix b/src/test/resources/docs-fixture/home/shop/scripts/by-city.relix new file mode 100644 index 0000000..bfb5427 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/scripts/by-city.relix @@ -0,0 +1,2 @@ +-- The customers of one city, named when the script runs: relix -a city=London +query { PROJECT name (SELECT city = $city (Customers)) }; diff --git a/src/test/resources/docs-fixture/home/shop/scripts/messy.relix b/src/test/resources/docs-fixture/home/shop/scripts/messy.relix new file mode 100644 index 0000000..a533d40 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/scripts/messy.relix @@ -0,0 +1,3 @@ +-- Orders worth a look +Large:={select amount>100(Orders)}; +query {project order_id,amount (Large)}; diff --git a/src/test/resources/docs-fixture/home/shop/scripts/top-customers.relix b/src/test/resources/docs-fixture/home/shop/scripts/top-customers.relix new file mode 100644 index 0000000..a2fff29 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/scripts/top-customers.relix @@ -0,0 +1,2 @@ +-- The customers who have spent the most, best first. +query { SORT total DESC (PROJECT name, total (Customers JOIN Spend)) };