diff --git a/.gitattributes b/.gitattributes index 4a97af3..58365f1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -7,6 +7,12 @@ gradlew text eol=lf *.bat text eol=crlf *.cmd text eol=crlf *.ps1 text eol=crlf +# Relix scripts: the tests compare what fmt and optimize write with them byte for byte. +*.relix text eol=lf +# The command-line guide: its command pages are generated, and compared with the +# generator's output byte for byte. +docs/guide/command-line/** text eol=lf +docs/command-examples/** text eol=lf # Binary files. *.jar binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4dcd33..64b8cc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ -# The gate: `./gradlew clean build` on Linux, macOS and Windows, for every pull request -# and every push to main or develop. +# The gate: `./gradlew clean build`, then the platform archive's tests, on Linux, macOS +# and Windows, for every pull request and every push to main or develop. name: CI on: @@ -20,17 +20,25 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-java@v6 with: distribution: corretto java-version: '21' # Also validates the checked-in wrapper jar against Gradle's published checksums. - - uses: gradle/actions/setup-gradle@v4 + - uses: gradle/actions/setup-gradle@v6 + # The completion scripts are syntax-checked by each shell that is installed; the + # runners have bash and PowerShell, and Linux gets zsh and fish here. + - name: Install zsh and fish + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y zsh fish - run: ./gradlew clean build + # The platform archive, run as it ships: a pipeline through its launcher, the CDS + # archive mapped, the man pages rendered. + - run: ./gradlew packageTest - name: Upload test and coverage reports if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: reports-${{ matrix.os }} path: build/reports/ diff --git a/.github/workflows/close-issues.yml b/.github/workflows/close-issues.yml index c83f591..4bbd26f 100644 --- a/.github/workflows/close-issues.yml +++ b/.github/workflows/close-issues.yml @@ -21,7 +21,7 @@ jobs: if: github.event.pull_request.merged == true runs-on: ubuntu-latest steps: - - uses: actions/github-script@v7 + - uses: actions/github-script@v9 with: script: | const pr = context.payload.pull_request; diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..2f2a6ac --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,205 @@ +# Releases the command (design §7.3): a platform archive for each of macOS, Linux and +# Windows, attached with their checksums to this repository's GitHub Release with the +# command-line guide's archive (relix--docs.zip, design §7.4), and the +# Homebrew formula and Scoop manifest that install them, pushed to the tap +# (DarkCollective/homebrew-relix) and the bucket (DarkCollective/scoop-relix). +# +# A tag vX.Y.Z publishes X.Y.Z. A tag with a suffix (v1.2.0-rc1) publishes a pre-release +# and leaves the tap and the bucket alone. +# +# Run from the Actions tab, it is a dry run: it builds and tests every archive, writes the +# formula and the manifest and checks their checksums against the archives, and uploads +# them as the run's `release-files` artifact. It publishes nothing, and since it may run +# on develop, whose engine pin is a snapshot, it skips checkEnginePin. +name: Release + +on: + push: + tags: ['v[0-9]*.[0-9]*.[0-9]*'] + workflow_dispatch: + inputs: + version: + description: 'The version to build, without the v (a dry run publishes nothing)' + required: true + default: '0.0.0-dryrun' + +permissions: + contents: read + +jobs: + version: + name: Version + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + publish: ${{ steps.version.outputs.publish }} + stable: ${{ steps.version.outputs.stable }} + steps: + - id: version + env: + EVENT: ${{ github.event_name }} + TAG: ${{ github.ref_name }} + INPUT: ${{ inputs.version }} + run: | + if [ "$EVENT" = push ]; then + version="${TAG#v}" + publish=true + else + version="$INPUT" + publish=false + fi + if ! printf '%s' "$version" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$'; then + echo "::error::'$version' is not a version such as 1.2.0 or 1.2.0-rc1" + exit 1 + fi + case "$version" in *-*) stable=false ;; *) stable=true ;; esac + { + echo "version=$version" + echo "publish=$publish" + echo "stable=$stable" + } >> "$GITHUB_OUTPUT" + + package: + name: Package (${{ matrix.platform }}) + needs: version + strategy: + fail-fast: true + matrix: + include: + - { os: macos-latest, platform: macos-aarch64 } + - { os: macos-15-intel, platform: macos-x86_64 } + - { os: ubuntu-latest, platform: linux-x86_64 } + - { os: ubuntu-24.04-arm, platform: linux-aarch64 } + - { os: windows-latest, platform: windows-x86_64 } + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + env: + VERSION: ${{ needs.version.outputs.version }} + # A dry run may build develop, whose engine pin is a snapshot; a release may not. + SKIP_PIN: ${{ needs.version.outputs.publish != 'true' && '-x checkEnginePin' || '' }} + steps: + - uses: actions/checkout@v7 + # Corretto's JDK carries the jmods jlink links from. + - uses: actions/setup-java@v6 + with: + distribution: corretto + java-version: '21' + - uses: gradle/actions/setup-gradle@v6 + # The gate, then the archive, then the archive's own tests: a pipeline through its + # launcher, the CDS archive mapped, the man pages rendered. + - name: Build, package and test the archive + shell: bash + run: ./gradlew clean build releaseArchive packageTest -PcliVersion="$VERSION" $SKIP_PIN + # The command-line guide, the same on every platform, so it is built once. Its + # examples have just run in the build above. + - name: Build the command-line guide's archive + if: matrix.platform == 'linux-x86_64' + shell: bash + run: ./gradlew docsArchive -PcliVersion="$VERSION" + - name: Check the archive is this platform's + shell: bash + run: | + case "${{ matrix.platform }}" in windows-*) ext=zip ;; *) ext=tar.gz ;; esac + test -f "build/distributions/relix-$VERSION-${{ matrix.platform }}.$ext" + - uses: actions/upload-artifact@v6 + with: + name: archive-${{ matrix.platform }} + path: build/distributions/relix-* + if-no-files-found: error + - name: Upload test reports + if: failure() + uses: actions/upload-artifact@v6 + with: + name: reports-${{ matrix.platform }} + path: build/reports/ + if-no-files-found: ignore + + release-files: + name: Checksums, formula and manifest + needs: [version, package] + runs-on: ubuntu-latest + env: + VERSION: ${{ needs.version.outputs.version }} + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v7 + with: + pattern: archive-* + path: dist + merge-multiple: true + # Fails unless every archive the formula and the manifest name is in dist with the + # checksum they give it. + - run: python3 packaging/release.py "$VERSION" dist release-files + - uses: actions/upload-artifact@v6 + with: + name: release-files + path: release-files/ + if-no-files-found: error + + publish: + name: Publish the release + needs: [version, release-files] + if: needs.version.outputs.publish == 'true' + runs-on: ubuntu-latest + permissions: + contents: write + env: + VERSION: ${{ needs.version.outputs.version }} + STABLE: ${{ needs.version.outputs.stable }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/download-artifact@v7 + with: + pattern: archive-* + path: dist + merge-multiple: true + - uses: actions/download-artifact@v7 + with: + name: release-files + path: release-files + - name: Create the GitHub Release + run: | + prerelease=() + if [ "$STABLE" != true ]; then prerelease=(--prerelease); fi + gh release create "v$VERSION" --repo "$GITHUB_REPOSITORY" --verify-tag \ + --title "relix $VERSION" --generate-notes "${prerelease[@]}" \ + dist/relix-"$VERSION"-* release-files/checksums.txt + + tap: + name: Update the Homebrew tap and the Scoop bucket + needs: [version, publish] + if: needs.version.outputs.stable == 'true' + runs-on: ubuntu-latest + env: + VERSION: ${{ needs.version.outputs.version }} + steps: + - uses: actions/download-artifact@v7 + with: + name: release-files + path: release-files + - name: Push the formula to DarkCollective/homebrew-relix + env: + TOKEN: ${{ secrets.HOMEBREW_TAP_PAT }} + run: | + git clone "https://x-access-token:${TOKEN}@github.com/DarkCollective/homebrew-relix.git" tap + mkdir -p tap/Formula + cp release-files/relix.rb tap/Formula/relix.rb + cd tap + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add Formula/relix.rb + git commit -m "relix $VERSION" + git push + - name: Push the manifest to DarkCollective/scoop-relix + env: + TOKEN: ${{ secrets.SCOOP_BUCKET_PAT }} + run: | + git clone "https://x-access-token:${TOKEN}@github.com/DarkCollective/scoop-relix.git" bucket + mkdir -p bucket/bucket + cp release-files/relix.json bucket/bucket/relix.json + cd bucket + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add bucket/relix.json + git commit -m "relix $VERSION" + git push diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c3da769..939e03a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,9 +106,11 @@ below. You need no Java 21 installed beforehand: Gradle provisions it. source set, which is a module; `EngineBoundaryTest` enforces it for every source set, tests included. Run it alone with `./gradlew test --tests '*EngineBoundaryTest'`. - **The engine pin.** `relixEngineVersion` in `build.gradle` is the one place the - engine's version is named. On `develop` it is `1.0.0-SNAPSHOT`, resolved from - Central's snapshot repository, which relix-core publishes to on every push to its - own `develop`. `checkEnginePin` fails a release build that still pins a snapshot: + engine's version is named. On `develop` it is a released engine while one has + everything the command needs, and `1.0.0-SNAPSHOT` while it needs engine work that + is not yet released, resolved from Central's snapshot repository, which relix-core + publishes to on every push to its own `develop`. `checkEnginePin` fails a release + build that still pins a snapshot: `./gradlew checkEnginePin -PcliVersion=`. - **One module, no library.** The module exports nothing (`ModuleShapeTest`) and the build publishes no Maven artifact (`checkNoPublication`). @@ -118,6 +120,22 @@ below. You need no Java 21 installed beforehand: Gradle provisions it. the build unpacks into `build/engine-docs`. A page that disagrees is fixed in relix-core. +### The release archive + +The command ships as a platform archive: a jlink image with an AppCDS archive, the man +pages and the completion scripts (design §7.3). CI also runs its tests on each +platform, after `build`: + +```bash +./gradlew packageTest +``` + +It builds the archive (`releaseArchive`, a minute or so), unpacks it with the system's +`tar`, and runs the launcher it holds: a script on standard input over a catalog in a +temporary tree, OPTIMIZE and `relix doc` (each finds a provider that jlink would leave +out unnoticed), `-Xshare:on` (which fails unless the CDS archive is used), and `man` over +every page. The man pages need `man` installed, and are not checked on Windows. + ### Building against a local engine checkout To try an engine change before it is published, build against your relix-core @@ -133,6 +151,29 @@ together with the engine-internal module it implements, so the runtime class pat holds some engine classes twice. That does no harm on a class path, but do not judge a module-path launch by it. +The checkout's build runs under this repository's Gradle, so it needs a relix-core +that builds on Gradle 9. An older one fails while configuring, in a plugin that +Gradle 9 broke (for example `info.solidsoft.pitest`, which reports an unknown +property `baseDir`). + +### Cutting a release (maintainers) + +1. Pin a released engine in `build.gradle` (`checkEnginePin` refuses a snapshot) and + promote to `main`. +2. Run the **Release** workflow from the Actions tab with the version. It is a dry run: + it builds and tests the five platform archives, writes the Homebrew formula and the + Scoop manifest from [`packaging/`](packaging), checks their checksums against the + archives, and uploads them as the `release-files` artifact. Nothing is published. +3. Tag `main`: `git tag -a v1.2.0 -m "relix 1.2.0" && git push origin v1.2.0`. The + workflow runs again, attaches the archives and `checksums.txt` to the GitHub Release, + and pushes the formula to `DarkCollective/homebrew-relix` and the manifest to + `DarkCollective/scoop-relix`, with the `HOMEBREW_TAP_PAT` and `SCOOP_BUCKET_PAT` + secrets. A tag with a suffix (`v1.2.0-rc1`) makes a pre-release and leaves the tap and + the bucket alone. + +The formula and the manifest are written whole from `packaging/` on every release, so a +change to either is made there, not in the tap or the bucket. + ## Branches and commits Work on a branch named with a prefix — `feat/`, `fix/`, `docs/`, `chore/` or @@ -150,6 +191,27 @@ because that is the interface a script or a pipeline depends on. A change to a command, an option or an exit status updates its help text, which is also what the manual page and the site's command reference are generated from. +The command-line section of the programming guide is in +[`docs/guide/command-line`](docs/guide/command-line/README.md); the site renders it, +and each release attaches it as `relix--docs.zip`. Three things keep it true: + +- **The command pages are generated.** `docs/guide/command-line/commands/*.md` and + `index.json` come from the command's picocli model, followed by the hand-written + examples in `docs/command-examples/.md`. After changing an option or one of + those files, run `./gradlew commandPages` and commit what it writes; + `GuideStructureTest` fails while they are stale. +- **Every example runs.** A ```` ```shell ```` block is an example, and the plain block on + the line after it is what it prints on standard output; `GuideExamplesTest` runs it in + a copy of the fixture under `src/test/resources/docs-fixture` (the `~/shop` the pages + describe) and compares. An example exits 0 unless it ends `; echo "exit $?"`, which + prints its status for the page to show. Write commands a reader would type in a + ```` ```shell ```` block, never `bash` or `console`, and keep anything else — an install + command, a syntax summary — in a ```` ```text ```` block. +- **Every link resolves**, to a page or a heading, here or in the engine's manuals: + the section sits at `docs/guide/command-line` in the engine's tree, so + `../secrets.md` is the guide's page and `../../reference/README.md` the language + reference. A new page is listed in the section's `README.md`. + ## Questions Ask in [Discussions](https://github.com/DarkCollective/relix-cli/discussions). diff --git a/README.md b/README.md index dd79954..bb05783 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,10 @@ It is a front end to the Relix engine, [relix-core](https://github.com/DarkCollective/relix-core), and reaches it only through the engine's published API. +Its documentation is the command-line section of the Relix programming guide, in +[`docs/guide/command-line`](docs/guide/command-line/README.md), published on the +[Relix site](https://relix.darkcollective.com/guide/index.html). + ## Status Under development. There is no release yet, and the command line described above is @@ -28,6 +32,27 @@ Requires Java 21, which the Gradle wrapper provisions if it is missing: [CONTRIBUTING.md](CONTRIBUTING.md#the-build-gate) describes what the build checks. +`./gradlew releaseArchive` builds this platform's archive in `build/distributions`: a +self-contained image (no Java needed to run it) with `bin/relix`, the man pages in +`man/man1` and the completion scripts in `completions/`. + +## Installing + +Once there is a release: + +```bash +brew install DarkCollective/relix/relix +``` + +```powershell +scoop bucket add relix https://github.com/DarkCollective/scoop-relix +scoop install relix +``` + +Or download the archive for your platform from the +[releases](https://github.com/DarkCollective/relix-cli/releases), check it against +`checksums.txt`, unpack it and put its `bin/` on your `PATH`. + ## Licence [Apache License 2.0](LICENSE.txt). See [CONTRIBUTING.md](CONTRIBUTING.md) to take diff --git a/build.gradle b/build.gradle index ad4b9af..67fbfdc 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,11 @@ plugins { id 'java' + id 'application' id 'jacoco' + // The platform image (design §7.3). forceMerge folds the jars that are not proper + // modules into one that is, which jlink needs. + id 'org.beryx.jlink' version '4.1.1' } group = 'com.darkcollective.relix' @@ -16,10 +20,11 @@ version = findProperty('cliVersion') ?: '0.1.0-SNAPSHOT' // The engine pin: the one place the engine's version is named. // -// `develop` tracks the engine's own develop branch, which relix-core publishes to Central's -// snapshot repository on every push, so that what this command needs reaches it before the -// next release candidate is cut. A release pins a released engine (checkEnginePin). -ext.relixEngineVersion = '1.0.0-SNAPSHOT' +// `develop` pins a released engine while one has everything this command needs. When the +// command needs engine work that is not yet released, it tracks the engine's own develop +// branch instead (1.0.0-SNAPSHOT), which relix-core publishes to Central's snapshot +// repository on every push. A release pins a released engine (checkEnginePin). +ext.relixEngineVersion = '1.0.0-rc9' def engineIsSnapshot = relixEngineVersion.endsWith('-SNAPSHOT') @@ -44,7 +49,26 @@ java { } } +sourceSets { + // What generates the man pages and completion scripts the release archive carries. + // It runs in the build and ships in nothing. + packaging { + compileClasspath += sourceSets.main.output + runtimeClasspath += sourceSets.main.output + } + // The tests of the packaged command: they run its launcher, not its classes. + packageTest + // CommandPagesTest regenerates the command pages to compare them with the checked-in ones. + test { + compileClasspath += sourceSets.packaging.output + runtimeClasspath += sourceSets.packaging.output + } +} + configurations { + packagingImplementation.extendsFrom implementation + packagingRuntimeOnly.extendsFrom runtimeOnly + // The pinned engine's manuals, which the documentation guards read. engineManuals { canBeConsumed = false @@ -52,6 +76,9 @@ configurations { } } +// The H2 driver: the image bundles it, and the package tests write their database with it. +def h2Driver = 'com.h2database:h2:2.5.252' + dependencies { // The engine, for the command and the library its embed API is. implementation "com.darkcollective.relix:relix:${relixEngineVersion}" @@ -68,11 +95,27 @@ dependencies { implementation 'info.picocli:picocli:4.7.7' annotationProcessor 'info.picocli:picocli-codegen:4.7.7' - testImplementation platform('org.junit:junit-bom:5.13.4') + // The one JDBC driver the image carries: the database that needs no setup. The + // engine finds it through DriverManager and nothing here names it; every other + // driver is installed with `relix drivers install`. + runtimeOnly h2Driver + + testImplementation platform('org.junit:junit-bom:6.1.3') testImplementation 'org.junit.jupiter:junit-jupiter' - testImplementation 'org.assertj:assertj-core:3.27.4' + testImplementation 'org.assertj:assertj-core:3.27.7' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' + packagingImplementation 'info.picocli:picocli-codegen:4.7.7' + packagingImplementation 'org.asciidoctor:asciidoctorj:3.0.1' + + packageTestImplementation platform('org.junit:junit-bom:6.1.3') + packageTestImplementation 'org.junit.jupiter:junit-jupiter' + packageTestImplementation 'org.assertj:assertj-core:3.27.7' + packageTestRuntimeOnly 'org.junit.platform:junit-platform-launcher' + // The database the pipeline test's catalog connects to is written with the driver + // the image bundles. + packageTestRuntimeOnly h2Driver + engineManuals "com.darkcollective.relix:relix:${relixEngineVersion}:manuals@zip" } @@ -138,6 +181,12 @@ test { dependsOn unpackEngineDocs inputs.dir(engineDocsDir).withPropertyName('engineDocs') systemProperty 'relix.engine.docs', engineDocsDir.get().asFile.absolutePath + // The command-line guide's pages, which the docs tests run and check: without these + // inputs a docs-only change would leave the tests UP-TO-DATE, passing without running. + inputs.dir('docs/guide/command-line').withPropertyName('guide') + inputs.dir('docs/command-examples').withPropertyName('commandExamples') + systemProperty 'relix.cli.guide', file('docs/guide/command-line').absolutePath + systemProperty 'relix.cli.commandExamples', file('docs/command-examples').absolutePath finalizedBy tasks.named('jacocoTestReport') } @@ -196,3 +245,245 @@ jar { 'Relix-Engine-Version': relixEngineVersion) } } + +// ── The release archive (design §7.3) ───────────────────────────────────────────────── +// +// ./gradlew releaseArchive build/distributions/relix--.tar.gz +// (.zip on Windows), for the platform the build runs on +// ./gradlew packageTest runs the archive's launcher: a pipeline, the CDS +// archive, the man pages +// +// The archive holds the jlink image (bin/relix, the runtime and an AppCDS archive), +// man/man1, completions/ and the licence. Each platform's is built on that platform. + +application { + applicationName = 'relix' + mainModule = 'com.darkcollective.relix.cli' + mainClass = 'com.darkcollective.relix.cli.Main' +} + +def os = org.gradle.internal.os.OperatingSystem.current() +def arch = System.getProperty('os.arch') in ['aarch64', 'arm64'] ? 'aarch64' : 'x86_64' +def platform = "${os.isMacOsX() ? 'macos' : os.isWindows() ? 'windows' : 'linux'}-${arch}" +def releaseName = "relix-${version}" +def imageDir = layout.buildDirectory.dir('image') +def cdsArchive = 'lib/relix.jsa' + +// Each stage of a pipeline starts a JVM, so the image carries an AppCDS archive of the +// classes a run loads, and the launcher maps it. The JVM's warnings go to standard error, +// not to standard output where they would join the rows. +def launcherJvmArgs = ['-XX:SharedArchiveFile={{BIN_DIR}}/../' + cdsArchive, + '-Xlog:all=off', '-Xlog:all=warning:stderr'] + +jlink { + options = ['--strip-debug', '--compress', 'zip-6', '--no-header-files', '--no-man-pages', + // The JDK's own CDS archive, which the application's extends. + '--generate-cds-archive', + // The solver is a provider nothing `requires`, so jlink would leave it out + // and OPTIMIZE would find no solver. It is named as a root. + '--add-modules', 'com.darkcollective.relix.solver.ojalgo'] + launcher { + name = 'relix' + jvmArgs = launcherJvmArgs + } + // H2 is not a named module, and picocli is one only from Java 9's versioned entries; + // jlink takes neither, so both are merged into one module that is... + forceMerge('picocli', 'h2') + // ...which re-declares H2's driver, for DriverManager to find it in the image, and + // reads the JDK modules H2 may touch (JNDI data sources, JMX, logging). + mergedModule { + requires 'java.sql' + requires 'java.naming' + requires 'java.security.sasl' + requires 'java.management' + requires 'java.logging' + provides('java.sql.Driver').with('org.h2.Driver') + } +} + +// The script the CDS archive is trained on: a run that parses, analyses and executes a +// join and an aggregate over a bound CSV file, and prints tsv. The classes it loads are +// the ones every run loads. +def trainingDir = layout.buildDirectory.dir('cds-training') +def writeTrainingRun = tasks.register('writeTrainingRun') { + description = 'Writes the script and data the AppCDS archive is trained on.' + def dir = trainingDir + outputs.dir(dir) + doLast { + def root = dir.get().asFile + root.deleteDir() + new File(root, 'home').mkdirs() + new File(root, 'people.csv').text = 'id,name,city\n1,ada,london\n2,bob,paris\n3,cy,london\n' + new File(root, 'training.relix').text = '''\ +Cities := [ +| city | country | +|--------|---------| +| london | uk | +| paris | fr | +]; +query { τ country (γ country, COUNT(*) → people (σ id > 0 (People ⋈ Cities))) }; +''' + } +} + +tasks.named('jlink').configure { + dependsOn writeTrainingRun + def image = imageDir + def training = trainingDir + def jvmArgs = launcherJvmArgs + def archive = cdsArchive + def module = "${application.mainModule.get()}/${application.mainClass.get()}" + doLast { + def root = image.get().asFile + // The generated launcher resolves a symbolic link before finding the runtime + // beside it: Homebrew runs it through one, which is relative to its directory. + def script = new File(root, 'bin/relix') + if (script.exists()) { + script.text = script.text.replace('DIR="${0%/*}"', '''\ +LINK="$0" +while [ -h "$LINK" ]; do + TARGET=$(readlink "$LINK") + case "$TARGET" in + /*) LINK="$TARGET" ;; + *) LINK="$(dirname "$LINK")/$TARGET" ;; + esac +done +DIR="$(cd "$(dirname "$LINK")" && pwd)"''') + } + // The Windows launcher would run from its own directory, which takes relative + // paths and .relix/ discovery away from the user's. + def batch = new File(root, 'bin/relix.bat') + if (batch.exists()) { + batch.text = batch.text.readLines() + .findAll { !(it.trim() in ['pushd %DIR%', 'popd']) } + .join('\r\n') + '\r\n' + } + + // The training run, with the launcher's options but writing the archive instead + // of mapping it. + def dir = training.get().asFile + def java = new File(root, "bin/java${os.isWindows() ? '.exe' : ''}").absolutePath + def command = [java, "-XX:ArchiveClassesAtExit=${new File(root, archive)}"] + + jvmArgs.findAll { !it.startsWith('-XX:SharedArchiveFile') } + + ["-Duser.home=${new File(dir, 'home')}", + '-m', module, '-o', 'tsv', '-i', 'People=csv:people.csv', 'training.relix'] + def process = new ProcessBuilder(command*.toString()).directory(dir).redirectErrorStream(true).start() + def output = process.inputStream.getText('UTF-8') + if (process.waitFor() != 0 || !output.contains('uk\t2')) { + throw new GradleException("The AppCDS training run failed:\n${output}") + } + if (!new File(root, archive).isFile()) { + throw new GradleException("The AppCDS training run wrote no ${archive}:\n${output}") + } + } +} + +// The man pages and completion scripts, from the command's picocli model. +def releaseFilesDir = layout.buildDirectory.dir('release-files') +def releaseFiles = tasks.register('releaseFiles', JavaExec) { + description = 'Generates the man pages and completion scripts from the command model.' + // The jar, not the classes: the man pages' version comes from its manifest. + classpath = files(tasks.named('jar')) + sourceSets.packaging.runtimeClasspath - sourceSets.main.output + mainClass = 'com.darkcollective.relix.cli.packaging.ReleaseFiles' + def dir = releaseFilesDir + outputs.dir(dir) + inputs.property('version', version.toString()) + doFirst { dir.get().asFile.deleteDir() } + argumentProviders.add({ + def root = dir.get().asFile + [new File(root, 'adoc').path, new File(root, 'man/man1').path, new File(root, 'completions').path] + } as CommandLineArgumentProvider) +} + +// ── The command-line guide (design §7.4) ────────────────────────────────────────────── +// +// ./gradlew commandPages rewrites docs/guide/command-line/commands/*.md and +// index.json from the command's picocli model +// ./gradlew docsArchive build/distributions/relix--docs.zip, the +// section the site renders in its programming guide +tasks.register('commandPages', JavaExec) { + group = 'documentation' + description = 'Rewrites the command pages of docs/guide/command-line from the command model.' + classpath = sourceSets.packaging.runtimeClasspath + mainClass = 'com.darkcollective.relix.cli.packaging.CommandPages' + args = [projectDir.absolutePath] +} + +tasks.register('docsArchive', Zip) { + group = 'documentation' + description = 'Builds the command-line guide archive the site renders from.' + from('docs/guide/command-line') { + into 'command-line' + } + archiveFileName = "relix-${version}-docs.zip" + destinationDirectory = layout.buildDirectory.dir('distributions') + preserveFileTimestamps = false + reproducibleFileOrder = true +} + +def stageRelease = tasks.register('stageRelease', Sync) { + description = 'Lays out the release archive\'s contents in build/release.' + dependsOn tasks.named('jlink') + from(imageDir) + from(releaseFiles) { + include 'man/**', 'completions/**' + } + from(layout.projectDirectory) { + include 'LICENSE.txt', 'NOTICE', 'README.md' + } + def staged = layout.buildDirectory.dir("release/${releaseName}") + into staged + // The JDK's CDS archives are read-only, which a copy over them fails on. + doFirst { staged.get().asFile.deleteDir() } +} + +def releaseArchive = tasks.register('releaseArchive', os.isWindows() ? Zip : Tar) { + group = 'distribution' + description = "Builds the release archive for this platform (${platform})." + from(stageRelease) { + into releaseName + } + archiveFileName = "${releaseName}-${platform}.${os.isWindows() ? 'zip' : 'tar.gz'}" + destinationDirectory = layout.buildDirectory.dir('distributions') + if (!os.isWindows()) { + compression = Compression.GZIP + } + preserveFileTimestamps = false + reproducibleFileOrder = true + // The launcher, the runtime's binaries and jspawnhelper must stay executable; an + // archive otherwise gives every file the same mode. + useFileSystemPermissions() +} + +// The archive, unpacked by the system's tar as a user or Homebrew unpacks it, so that +// the tests run what ships rather than what was staged. +def unpackedRelease = layout.buildDirectory.dir('package-test') +def unpackRelease = tasks.register('unpackRelease', Exec) { + description = 'Unpacks the release archive into build/package-test with tar.' + def archive = releaseArchive.flatMap { it.archiveFile } + def dir = unpackedRelease + inputs.file(archive) + outputs.dir(dir) + doFirst { + def root = dir.get().asFile + root.deleteDir() + root.mkdirs() + } + // Windows' own tar is bsdtar, which reads a zip; a GNU tar earlier on the PATH does not. + executable os.isWindows() ? "${System.getenv('SystemRoot')}\\System32\\tar.exe" : 'tar' + argumentProviders.add({ + ['-xf', archive.get().asFile.absolutePath, '-C', dir.get().asFile.absolutePath] + } as CommandLineArgumentProvider) +} + +tasks.register('packageTest', Test) { + group = 'verification' + description = 'Runs the release archive\'s launcher: a pipeline, the CDS archive, the man pages.' + testClassesDirs = sourceSets.packageTest.output.classesDirs + classpath = sourceSets.packageTest.runtimeClasspath + useJUnitPlatform() + dependsOn unpackRelease + def unpacked = unpackedRelease.map { it.dir(releaseName) } + inputs.dir(unpackedRelease).withPropertyName('release') + systemProperty 'relix.release', unpacked.get().asFile.absolutePath +} diff --git a/docs/command-examples/bundle.md b/docs/command-examples/bundle.md new file mode 100644 index 0000000..8abaf2e --- /dev/null +++ b/docs/command-examples/bundle.md @@ -0,0 +1,10 @@ +## Examples + +A bundle is one JSON document per script, for a tool such as a query playground: + +```shell +relix bundle -e 'SELECT amount > 100 (Orders)' | jq -c 'keys' +``` +``` +["diagnostics","namespace","queries"] +``` diff --git a/docs/command-examples/catalog.md b/docs/command-examples/catalog.md new file mode 100644 index 0000000..17b0281 --- /dev/null +++ b/docs/command-examples/catalog.md @@ -0,0 +1,44 @@ +## Examples + +```shell +relix catalog ls -o tsv +``` +``` +name kind arity file +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/.relix/catalog/00-sources.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +```shell +relix catalog schema Customers -o tsv +``` +``` +column type +customer_id N +name S +city S +``` + +```shell +relix catalog where Orders -o tsv +``` +``` +file line kind state +~/shop/.relix/catalog/00-sources.relix 7 source wins +``` + +```shell +relix catalog trust --list -o tsv +``` +``` +directory state +~/shop trusted +~/shop/reports trusted +``` + +[The catalog](../catalog.md) covers discovery, the overlay and trust. diff --git a/docs/command-examples/check.md b/docs/command-examples/check.md new file mode 100644 index 0000000..a38dafa --- /dev/null +++ b/docs/command-examples/check.md @@ -0,0 +1,30 @@ +## Examples + +A script that analyses prints nothing, and exits 0: + +```shell +relix check scripts/top-customers.relix +``` + +A problem is printed on standard error, and the status is 3: + +```shell +relix check scripts/broken.relix 2>&1; echo "exit $?" +``` +``` +scripts/broken.relix:1:30: error: Undefined relation: 'Ordrs' +exit 3 +``` + +`--json` writes the diagnostics to standard output instead, as one document for a tool to +read: + +```shell +relix check --json scripts/broken.relix; echo "exit $?" +``` +``` +{"diagnostics":[{"file":"scripts/broken.relix","line":1,"column":30,"severity":"error","message":"Undefined relation: 'Ordrs'"}]} +exit 3 +``` + +[Pipelines](../pipelines.md#lint-every-script) checks every script in a tree. diff --git a/docs/command-examples/completion.md b/docs/command-examples/completion.md new file mode 100644 index 0000000..8317bd8 --- /dev/null +++ b/docs/command-examples/completion.md @@ -0,0 +1,14 @@ +## Examples + +The release archives carry each shell's script in `completions/`, and Homebrew installs +them. To load one by hand, add the line for your shell to its startup file: + +```text +bash source <(relix completion bash) +zsh source <(relix completion zsh) +fish relix completion fish > ~/.config/fish/completions/relix.fish +PowerShell relix completion powershell | Out-String | Invoke-Expression +``` + +The zsh script is also a completion function, so it can be saved as `_relix` in a +directory on `$fpath` instead. diff --git a/docs/command-examples/connectors.md b/docs/command-examples/connectors.md new file mode 100644 index 0000000..8e433cb --- /dev/null +++ b/docs/command-examples/connectors.md @@ -0,0 +1,22 @@ +## Examples + +The connectors that ship with `relix`, and any plugins installed in `~/.relix/connectors`: + +```shell +relix connectors ls +``` +``` +── connectors ────────────────────────────────────────────────────────────────── + type installed + ────── ───────── + clf true + csv true + gedcom true + log true +(4 rows) +``` + +`relix connectors ls --available` adds the connectors that can be installed, which it +fetches from the hosted catalog, and `relix connectors install TYPE` installs one. A +source whose type no installed connector reads — `mongodb`, say — fails with exit status +5, unless `--allow-download` lets the run fetch the connector. diff --git a/docs/command-examples/doc.md b/docs/command-examples/doc.md new file mode 100644 index 0000000..adc6a9b --- /dev/null +++ b/docs/command-examples/doc.md @@ -0,0 +1,38 @@ +## Examples + +A page is found by its glyph, its keyword or its name: + +```shell +relix doc --markdown σ | head -6 +``` +``` +# Name: Selection (σ / SELECT) + +# Syntax: +σ (Relation) +SELECT (Relation) +``` + +With no topic, `relix doc` lists the pages, as rows: + +```shell +relix doc -o tsv | head -4 +``` +``` +topic category title summary +getting-started guide Getting started Your first script — declare data, name a result, run it +glossary guide Glossary The vocabulary — relation, tuple, bag vs set, materialisation, pushdown +σ operator Selection Filter rows by a condition +``` + +`--function` lists, or looks up, the functions only: + +```shell +relix doc --function -o tsv | grep -i '^round' +``` +``` +Round function Round Round to nearest +``` + +On a terminal a page is shown through `$PAGER` (`less` when it is not set); `--no-pager` +writes it straight to standard output. diff --git a/docs/command-examples/drivers.md b/docs/command-examples/drivers.md new file mode 100644 index 0000000..4b202cf --- /dev/null +++ b/docs/command-examples/drivers.md @@ -0,0 +1,20 @@ +## Examples + +```shell +relix drivers ls +``` +``` +── drivers ───────────────────────────────────────────────────────────────────── + name scheme installed + ────────── ──────────────── ───────── + postgresql jdbc:postgresql: false + mysql jdbc:mysql: false +(2 rows) +``` + +`relix drivers install NAME` downloads one of those drivers, checks its checksum, and +installs it in `~/.relix/drivers`, or in `$RELIX_DRIVERS` when that is set. A run that +needs a driver that is not installed fails with exit status 5, unless `--allow-download` +lets it fetch the driver itself. The H2 driver is part of +`relix`, and needs no installing. A driver `relix drivers ls` does not list is installed +by copying its jar into the driver directory. diff --git a/docs/command-examples/explain.md b/docs/command-examples/explain.md new file mode 100644 index 0000000..36bd51d --- /dev/null +++ b/docs/command-examples/explain.md @@ -0,0 +1,20 @@ +## Examples + +```shell +relix explain -e 'GROUP city, COUNT(*) -> n (Customers JOIN Orders)' +``` +``` +Aggregate +└─ Join NATURAL/HASH build=RIGHT + ├─ Scan Customers + └─ Scan Orders +``` + +`--json` writes each query's plan as a JSON object instead: + +```shell +relix explain --json -e 'SELECT amount > 100 (Orders)' | jq -c 'keys' +``` +``` +["plan","query","script"] +``` diff --git a/docs/command-examples/fmt.md b/docs/command-examples/fmt.md new file mode 100644 index 0000000..582132f --- /dev/null +++ b/docs/command-examples/fmt.md @@ -0,0 +1,61 @@ +## Examples + +```shell +cat scripts/messy.relix +``` +``` +-- Orders worth a look +Large:={select amount>100(Orders)}; +query {project order_id,amount (Large)}; +``` + +`relix fmt` prints it canonically — one statement per line, spaced, each operator in one +spelling — keeping its comments: + +```shell +relix fmt scripts/messy.relix +``` +``` +-- Orders worth a look +Large := { σ amount > 100 (Orders) }; +query { π order_id, amount (Large) }; +``` + +`--keywords` writes the operators as ASCII keywords instead of glyphs: + +```shell +relix fmt --keywords scripts/messy.relix +``` +``` +-- Orders worth a look +Large := { SELECT amount > 100 (Orders) }; +query { PROJECT order_id, amount (Large) }; +``` + +`--check` changes nothing and names each file that would change, exiting 1 if there is +one; `-w` rewrites each such file in place: + +```shell +relix fmt --check --keywords scripts/messy.relix scripts/by-city.relix; echo "exit $?" +``` +``` +scripts/messy.relix +exit 1 +``` + +```shell +relix fmt -w --keywords scripts/messy.relix +relix fmt --check --keywords scripts/messy.relix; echo "exit $?" +``` +``` +exit 0 +``` + +With no file, `relix fmt` formats standard input to standard output: + +```shell +echo 'query {select amount>100(Orders)};' | relix fmt +``` +``` +query { σ amount > 100 (Orders) }; +``` diff --git a/docs/command-examples/help.md b/docs/command-examples/help.md new file mode 100644 index 0000000..f32a983 --- /dev/null +++ b/docs/command-examples/help.md @@ -0,0 +1,12 @@ +## Examples + +`relix help COMMAND` prints what `relix COMMAND --help` prints: + +```shell +relix help version | head -3 +``` +``` +Usage: relix version [-hNqVv] [--allow-download] [--remote] [-C=DIR] + [--max-fixpoint-rounds=N] [--max-materialized-rows=N] + [--max-processed-rows=N] [--now=INSTANT] [-P=NAME] +``` diff --git a/docs/command-examples/ir.md b/docs/command-examples/ir.md new file mode 100644 index 0000000..8e2cc5e --- /dev/null +++ b/docs/command-examples/ir.md @@ -0,0 +1,67 @@ +## Examples + +```shell +relix ir -e 'Big := { SELECT amount > 100 (Orders) };' -e 'query Big;' +``` +``` +════════════════════════════════════════════════════════════════════════════════ +RELIX IR namespace=default +════════════════════════════════════════════════════════════════════════════════ + kind: SRC=source INL=inline SYS=system QR=view DB=database + FN=function TVF=table-fn LIT=truth-literal + type: N=number S=string B=boolean ?=any [x]=array {…}=struct + D=date T=time TS=timestamp DUR=duration + mat: [bag]=bag [set]=dedup-set [sort]=sorted (stream=no label) + +── SYMBOLS ───────────────────────────────────────────────────────────────────── + Big QR order_id:N customer_id:N amount:N status:S placed:D + BigSpenders QR name:S total:N + catalog SYS name:S kind:S arity:N schema_code:S + columns SYS relation:S column:S type:S ordinal:N distinct_count:N … + Completed QR order_id:N customer_id:N amount:N status:S placed:D + connections SYS name:S dialect:S + Customers SRC customer_id:N name:S city:S + cycles QR name:S + dependencies QR dependent:S depends_on:S + deps TVF (r:S)→relation + events SYS seq:N stage:S code:S description:S target:S rows:N el… + find TVF (col:S)→relation + funcs TVF (cat:S)→relation + functions SYS name:S category:S arity:N max_arity:N return_type:S pu… + impact TVF (r:S)→relation + keys SYS relation:S key:N ordinal:N column:S + Orders SRC order_id:N customer_id:N amount:N status:S placed:D + plan SYS query:S node_id:N parent_id:N ordinal:N op:S label:S … + Regions INL city:S region:S + relations SYS name:S kind:S namespace:S materialization:S row_count:N… + rules QR seq:N code:S description:S target:S + schema TVF (rel:S)→relation + Spend QR customer_id:N total:N + Stock SRC sku:S on_hand:N + unused QR name:S + version SYS component:S kind:S version:S + +── EXPRESSION TREES ──────────────────────────────────────────────────────────── +Big [QR] order_id:N customer_id:N amount:N status:S placed:D + σ amount > 100 + └─ Orders [SRC] + +BigSpenders [QR] name:S total:N + π name, total + └─ ⋈ + ├─ Customers [SRC] + └─ σ total ≥ 200 + └─ Spend [QR] + +Completed [QR] order_id:N customer_id:N amount:N status:S placed:D + σ status = "completed" + └─ Orders [SRC] + +Spend [QR] customer_id:N total:N + γ [customer_id] SUM(amount)→total [bag] + └─ Completed [QR] + +── ROOT QUERIES ──────────────────────────────────────────────────────────────── + query Big [QR] +════════════════════════════════════════════════════════════════════════════════ +``` diff --git a/docs/command-examples/optimize.md b/docs/command-examples/optimize.md new file mode 100644 index 0000000..130dc2d --- /dev/null +++ b/docs/command-examples/optimize.md @@ -0,0 +1,39 @@ +## Examples + +The rewritten script is printed in the same form the engine runs it in: + +```shell +relix optimize -e "query { SELECT city = 'London' (Customers JOIN Orders) };" +``` +``` +query { (σ city = "London" (Customers)) ⋈ (Orders) }; +``` + +`--report` prints the rules that fired, and each query before and after: + +```shell +relix optimize --report -e "query { SELECT city = 'London' (Customers JOIN Orders) };" +``` +``` +════════════════════════════════════════════════════════════════════════════════ +RELIX OPTIMIZER namespace=default transformations=1 +════════════════════════════════════════════════════════════════════════════════ + +── SUMMARY ───────────────────────────────────────────────────────────────────── + SEL-005 Selection pushed into join input ...................................×1 + +── TREES ─────────────────────────────────────────────────────────────────────── + +── + before: σ city = "London" ((Customers) ⋈ (Orders)) + after: (σ city = "London" (Customers)) ⋈ (Orders) + +── DETAIL ────────────────────────────────────────────────────────────────────── + +── (1 transformation) + [SEL-005] selection pushed into left input of natural join @ :1:9 +════════════════════════════════════════════════════════════════════════════════ +``` + +[Pipelines](../pipelines.md#optimise-a-script-and-review-the-change) compares a script +with its rewrite. diff --git a/docs/command-examples/provenance.md b/docs/command-examples/provenance.md new file mode 100644 index 0000000..8046b59 --- /dev/null +++ b/docs/command-examples/provenance.md @@ -0,0 +1,54 @@ +## Examples + +The default semiring, `counting`, says in how many ways each row is derived. Two London +customers each derive the row `London`: + +```shell +relix provenance -e 'PROJECT city (Customers)' +``` +``` + +------ ------ +city [prov] +------ ------ +London 2 +Leeds 1 +York 1 +------ ------ +(3 tuples; provenance semiring: counting) +``` + +The `lineage` semiring names the source rows each result row came from: + +```shell +relix provenance --semiring=lineage -e 'PROJECT city (SELECT amount > 100 (Customers JOIN Orders))' +``` +``` + +------ ------------------------------------------- +city [prov] +------ ------------------------------------------- +London Customers#1·Orders#1 + Customers#3·Orders#4 +------ ------------------------------------------- +(1 tuple; provenance semiring: lineage) +``` + +In a pipe, or with `-o json`, the rows and their annotations are one JSON document: + +```shell +relix provenance -e 'PROJECT city (Customers)' -o json +``` +``` +{ + "query": "", + "semiring": "counting", + "schema": ["city"], + "tuples": [ + {"row": {"city": "London"}, "provenance": "2"}, + {"row": {"city": "Leeds"}, "provenance": "1"}, + {"row": {"city": "York"}, "provenance": "1"} + ] +} +``` + +The engine's [provenance page](../../provenance.md) explains each semiring. diff --git a/docs/command-examples/relix.md b/docs/command-examples/relix.md new file mode 100644 index 0000000..1dd61b9 --- /dev/null +++ b/docs/command-examples/relix.md @@ -0,0 +1,57 @@ +## Examples + +With no command, `relix` runs scripts, as [`relix run`](run.md) does: + +```shell +relix -e 'PROJECT name, city (Customers)' -o tsv +``` +``` +name city +Alice London +Bob Leeds +Carol London +Dan York +``` + +Global options go before or after the command's name: + +```shell +relix -C reports catalog ls -o tsv --no-header +``` +``` +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/reports/.relix/catalog/10-sample.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +## Environment + +| Variable | Effect | +|---|---| +| `RELIX_OUTPUT` | the output format when `-o` is not given | +| `RELIX_PROFILE` | the profile when `-P` is not given | +| `RELIX_NOW` | the instant `NOW()` returns when `--now` is not given | +| `RELIX_CATALOG_PATH` | catalog files to load, separated as in `PATH`, before the `--catalog` files | +| `RELIX_TRUST_ALL` | `1` or `true` trusts every project's `.relix/` directory | +| `RELIX_DRIVERS` | the directory JDBC drivers are installed in and loaded from, instead of `~/.relix/drivers` | +| `NO_COLOR` | when set, a table on a terminal is not coloured unless `--color=always` | +| `PAGER` | the program [`relix doc`](doc.md) shows a page through on a terminal; `cat` or empty shows it directly | + +A `${NAME}` placeholder in a declaration may also be resolved from the environment; see +[Secrets and profiles](../secrets.md). + +## Files + +| Path | What it holds | +|---|---| +| `~/.relix/catalog/*.relix` | your own catalog, which every directory under your home sees | +| `~/.relix/profiles.json` | your own profiles | +| `~/.relix/trusted` | the projects you have trusted, with a hash of each one's files | +| `~/.relix/drivers/` | installed JDBC drivers | +| `~/.relix/connectors/` | installed connector plugins | +| `~/.relix/cache/files/` | files fetched over `https` by a file connection | +| `.relix/catalog/*.relix`, `.relix/relixrc`, `.relix/profiles.json`, `.relix/root` | a project's catalog, defaults, profiles, and the marker that ends the walk up; see [The catalog](../catalog.md) | diff --git a/docs/command-examples/run.md b/docs/command-examples/run.md new file mode 100644 index 0000000..5da0dd2 --- /dev/null +++ b/docs/command-examples/run.md @@ -0,0 +1,40 @@ +## Examples + +`relix run` is what `relix` does with no command, so these two are the same: + +```shell +relix run -e 'GROUP status, SUM(amount) -> total (Orders)' +``` +``` +── ────────────────────────────────────────────────────────────── + status total + ───────── ───── + completed 400 + pending 50 +(2 rows) +``` + +```shell +relix -e 'GROUP status, SUM(amount) -> total (Orders)' -o csv +``` +``` +status,total +completed,400 +pending,50 +``` + +Each script file is a run of its own, with its own declarations: + +```shell +relix run scripts/top-customers.relix -a city=Leeds scripts/by-city.relix -o tsv +``` +``` +name total +Alice 200 +Carol 200 +name +Bob +``` + +[Binding inputs](../inputs.md), [Output formats](../output.md) and +[Pipelines](../pipelines.md) cover the options in use. diff --git a/docs/command-examples/trace.md b/docs/command-examples/trace.md new file mode 100644 index 0000000..f453d2c --- /dev/null +++ b/docs/command-examples/trace.md @@ -0,0 +1,24 @@ +## Examples + +Each event ends with how long it took, which varies from run to run, so here `cut` +leaves the timings off: + +```shell +relix trace -e 'SELECT amount > 100 (Orders)' | cut -d'(' -f1 +``` +``` +[EXECUTE ] orders SCAN scanned 4 rows +[EXECUTE ] ROWS query delivered 2 rows +``` + +`relix run --trace` writes the same feed to standard error while the rows go to standard +output, and `--trace=FILE` writes it to a file: + +```shell +relix --trace=trace.txt -e 'SELECT amount > 100 (Orders)' -o tsv +``` +``` +order_id customer_id amount status placed +1 1 120 completed 2026-09-02 +4 3 200 completed 2026-10-01 +``` diff --git a/docs/command-examples/version.md b/docs/command-examples/version.md new file mode 100644 index 0000000..e181891 --- /dev/null +++ b/docs/command-examples/version.md @@ -0,0 +1,11 @@ +## Examples + +`relix version`, like `relix --version`, prints one line: the version of the command, +then the version of the Relix engine it runs, as + +```text +relix 1.0.0 (engine 1.0.0) +``` + +The two are numbered independently: a release of the command names the engine release +it was built on. diff --git a/docs/design.md b/docs/design.md index 90fb8b3..1e84099 100644 --- a/docs/design.md +++ b/docs/design.md @@ -119,6 +119,9 @@ The schema of a bound input is inferred (header for CSV/TSV, first N rows for JSON/NDJSON — `--infer-rows=N`), or given explicitly with `--schema NAME='{ id: NUMBER, … }'`. Inference and stream reading are API request **R1** (§9). +An input that never ends, such as `tail -f`'s, is declared with `--unbounded=NAME`: its rows +stream as they arrive, and a query that would have to read all of it first is refused. + ### 3.3 Where data goes: output formats `-o FORMAT` / `--output=FORMAT`, or `RELIX_OUTPUT`: @@ -191,6 +194,7 @@ adds timings. | `-C DIR` | act as if started in DIR (catalog discovery, relative paths) — `git -C`, `make -C`, `tar -C` | — | | `-P NAME`, `--profile=NAME` | the named profile `${VAR}`s resolve from (`RELIX_PROFILE`) | `-e/--env` (renamed: `-e` is the expression, by convention) | | `-D NAME=VALUE` | set one `${VAR}` for this run (`awk -v`, `cc -D`); beats the profile. Not for secrets: it lands in shell history and `ps` | — | +| `-a NAME=VALUE`, `--arg` | bind the query parameter `$NAME` (`awk -v`, `psql -v`); repeatable. Bound, never pasted into the text, so it cannot change the query (R3) | — | | `-N`, `--no-catalog` | ignore `.relix/` discovery | — | | `--catalog=FILE` | add a catalog file (repeatable; `RELIX_CATALOG_PATH`, `:`-separated) | — | | `--now=INSTANT` | pin `NOW()` (`RELIX_NOW`) | same | @@ -324,7 +328,7 @@ Everything goes through `com.darkcollective.relix.embed`: | queries | `script(text)`, `relation(expression)` | | rows | `Relation.stream()` (closed on `EPIPE`/`SIGINT`) | | `explain`, `optimize`, `trace`, `bundle`, `ir`, `provenance` | `explain`/`explainJson`, `optimized().render()` + `rewrites()`, `stream(listener)`, `renderJson`, `model().ir()`, `provenance(semiring[, weight])` | -| `fmt` | the statement printer behind `Relix.definitions()` | +| `fmt` | `ScriptPrinter.print(Script, Spelling)`, which keeps comments | | `catalog ls/schema` | `relation("relix.relations")` etc. | | `doc` | the reference lookup in `relix-docs` (R5): language pages and the installed functions' pages, one index | @@ -383,7 +387,10 @@ sees. `brew tap DarkCollective/relix` resolves) and the Scoop bucket (`DarkCollective/scoop-relix`) stay separate repositories, updated by the release job; they stay private until the first release and must be public from then on, - as must the archives they point at. + as must the archives they point at. The formula and the manifest are written whole + from templates in this repository's `packaging/`, with each release's checksums, + and a dry run of the release workflow checks those against the archives before + anything is published. - Windows: `isatty` via `System.console()`; `SIGPIPE` does not exist and the `EPIPE` path covers it; paths in `-i` accept both separators. @@ -400,6 +407,13 @@ version of it. `.relix/` tree and fixture files, comparing stdout and the exit status — so a page cannot be published with an example the command does not honour. +The pages live in `docs/guide/command-line`, laid out as the engine's own manuals are +(a `README.md` index of chapters and pages), with `index.json` as the machine-readable +index. The site renders the section inside the engine's programming guide, at +`guide/command-line/`, so a link out of it is written as if it sat in the engine's +`docs/guide`. Until releases attach the archive, the site pins a commit of this +repository and reads the same tree from it. + ### 7.5 Tests Each command end-to-end through `Main` with captured streams; catalog discovery and @@ -457,7 +471,8 @@ keyword or name, across the language pages and the pages of the installed functi a language keyword keeping a shared name (`fix` is the operator; `Fix` the function is found as a function). Every program that shows the reference needs that same index, so it belongs beside the pages it indexes rather than in each front end. Filed as -DarkCollective/relix-core#95. +DarkCollective/relix-core#95, and done: `relix doc` looks pages up in relix-docs' +`ReferenceLookup`. ## 10. Decisions diff --git a/docs/guide/command-line/README.md b/docs/guide/command-line/README.md new file mode 100644 index 0000000..1aa3479 --- /dev/null +++ b/docs/guide/command-line/README.md @@ -0,0 +1,65 @@ +# The relix command line + +Using Relix from a shell: the `relix` command, which runs scripts as one stage of a Unix +pipeline, and the commands beside it that check, explain, optimise and format them. + +`relix` writes rows to standard output and everything else to standard error. A script +comes from `-e`, a file or standard input; its data from files, standard input, or the +tables a project declares in its `.relix/` directory; and its rows leave as a table on a +terminal, or tab-separated values, CSV, NDJSON, JSON or Markdown for the next program. + +Every example in this section is run on each build of the command, against a small +project, and the block under it is what it printed. The command reference is generated +from the same model as `relix --help` and the man pages. + +## The command line + +Installing it, and the shape every command shares; then data in, rows out. + +| Page | What it covers | +|---|---| +| [The relix command](getting-started.md) | Installing; a first query; where the script comes from; the commands; diagnostics and exit status | +| [Binding inputs](inputs.md) | `-i` and its formats; standard input; inferred and given headings; inputs that never end; shadowing the catalog; query parameters with `-a` | +| [Output formats](output.md) | `table`, `tsv`, `csv`, `ndjson`, `json` and `markdown`; NULLs and headers; scripts with several queries; `--fail-empty` and `--fail-rows`; messages; closed pipes and interrupts | + +## Projects on the command line + +What a directory gives the commands run in it. + +| Page | What it covers | +|---|---| +| [The catalog](catalog.md) | `.relix/` directories and how they are found; trusting a project; a nearer declaration wins; inspecting the catalog; `--catalog`; `relixrc` | +| [Secrets and profiles](secrets.md) | `${NAME}` placeholders; `profiles.json` and `-P`; the environment and `-D`; private profile files; what is never printed | + +## Command-line recipes + +| Page | What it covers | +|---|---| +| [Pipelines](pipelines.md) | With `jq`, `awk`, `sort` and `head`; with `find` and `xargs`; scripts in here-documents; a data check in CI; optimising a script and reviewing the change | + +## Command reference + +One page per command: what it does, its usage, arguments and options, its exit status, +and examples. + +| Page | What it covers | +|---|---| +| [`relix`](commands/relix.md) | The global options every command takes; the exit statuses; the environment variables and files | +| [`relix run`](commands/run.md) | Runs scripts and writes their queries' rows | +| [`relix check`](commands/check.md) | Analyses scripts without running them, and reports their problems | +| [`relix explain`](commands/explain.md) | Prints each query's physical plan | +| [`relix optimize`](commands/optimize.md) | Prints the script with each query rewritten by the optimiser | +| [`relix trace`](commands/trace.md) | Runs each query and prints what the engine rewrote, planned and ran | +| [`relix bundle`](commands/bundle.md) | Prints each script's JSON bundle of diagnostics, plans and events | +| [`relix ir`](commands/ir.md) | Prints the symbols, views and queries the analyser made of a script | +| [`relix provenance`](commands/provenance.md) | Runs each query and annotates each row with where it came from | +| [`relix fmt`](commands/fmt.md) | Prints scripts canonically, keeping their comments | +| [`relix catalog`](commands/catalog.md) | Lists the catalog's files, relations, columns and declarations; trusts a project | +| [`relix drivers`](commands/drivers.md) | Lists and installs JDBC drivers | +| [`relix connectors`](commands/connectors.md) | Lists and installs connector plugins | +| [`relix doc`](commands/doc.md) | Shows a page of the language reference | +| [`relix help`](commands/help.md) | Prints a command's usage | +| [`relix version`](commands/version.md) | Prints the versions of the command and of the engine | +| [`relix completion`](commands/completion.md) | Prints a completion script for bash, zsh, fish or PowerShell | + +A machine-readable index of these pages is `index.json`, beside this one. diff --git a/docs/guide/command-line/catalog.md b/docs/guide/command-line/catalog.md new file mode 100644 index 0000000..1078175 --- /dev/null +++ b/docs/guide/command-line/catalog.md @@ -0,0 +1,328 @@ +# The catalog + +A project's tables, connections and views are declared once, in its `.relix/` directory, +and every command run inside the project can name them. `relix` finds those directories by +walking up from where it starts, the way `git` finds `.git`, so the same command works +from any subdirectory. This page covers how the directories are laid out and found, how +a nearer declaration replaces a farther one, how to see what a run will load, and the +trust a project's directory needs before anything in it is loaded. + +## The layout + +A `.relix/` directory holds: + +| Path | What it holds | +|---|---| +| `catalog/*.relix` | the declarations: `source`, `connection`, views (`Name := { … };`), `def`, `import` | +| `relixrc` | defaults for the commands run below it: the output format and the profile | +| `profiles.json` | named sets of `${VAR}` values; see [Secrets and profiles](secrets.md) | +| `root` | a marker that stops the walk up at this directory | + +The shop the examples use has one: + +```shell +find .relix -type f | sort +``` +``` +.relix/catalog/00-sources.relix +.relix/catalog/10-views.relix +.relix/catalog/20-warehouse.relix +.relix/profiles.json +``` + +```shell +cat .relix/catalog/00-sources.relix +``` +``` +-- The shop's data. A relative path resolves against this file's directory. +source Customers from csv("../../data/customers.csv") { + header: true, + schema: { customer_id: NUMBER, name: STRING, city: STRING } +}; + +source Orders from csv("../../data/orders.csv") { + header: true, + schema: { order_id: NUMBER, customer_id: NUMBER, amount: NUMBER, status: STRING, placed: DATE } +}; +``` + +```shell +cat .relix/catalog/10-views.relix +``` +``` +Completed := { SELECT status = 'completed' (Orders) }; + +Spend := { GROUP customer_id, SUM(amount) -> total (Completed) }; + +BigSpenders := { PROJECT name, total (Customers JOIN (SELECT total >= 200 (Spend))) }; +``` + +A catalog file holds declarations only; a `query` statement in one is an error. A +relative path in a catalog file resolves against the file's own directory, not the +directory the command runs in, so `../../data/orders.csv` above means `~/shop/data` +from wherever in the project `relix` is started. + +Declaring costs nothing at run time: no file is read and no connection opened until a +query reaches the name. A large catalog costs a parse and an analysis, not a connection. + +## Finding the directories + +A run's catalog comes from these directories, loaded **outermost first**: + +1. `~/.relix`, your own, which every directory under your home sees. +2. Each `.relix/` from the outermost directory down to the one the command starts in, + walking up from it and stopping at your home directory. A `.relix/` that holds a file + named `root` ends the walk there, so the projects of a monorepo do not inherit each + other's declarations. Started outside your home directory, `relix` reads only that + directory's own `.relix/`. +3. The files named by `$RELIX_CATALOG_PATH`, separated as in `PATH`, then those named + with `--catalog FILE`, in order. + +Within a directory, `catalog/*.relix` load in the lexical order of their names, so +`00-sources.relix` comes before `10-views.relix`, as in `/etc/*.d`. + +`-N` (`--no-catalog`) leaves out every `.relix/` directory's catalog and `relixrc`; +`--catalog` files still load. `-C DIR` starts the walk from `DIR`. + +## A project must be trusted + +Loading code because of the directory you are standing in is a risk: a cloned +repository's `.relix/` could declare a source that sends a `${TOKEN}` from your +environment to a server of its author's choosing. So a project's `.relix/` is loaded only +once you have trusted it, as `direnv` does with an `.envrc`. Until then, it is skipped, +with a warning saying how to trust it, and a run that needs one of its names fails with +exit status 5: + +```shell +relix -e 'Customers' 2>&1; echo "exit $?" +``` +``` +relix: warning: skipping ~/shop/.relix, which is not trusted; to load it: relix catalog trust ~/shop +-e:1:1: error: Undefined relation: 'Customers' +relix: Customers is declared in ~/shop/.relix/catalog/00-sources.relix, which is not trusted; to load it: relix catalog trust ~/shop +exit 5 +``` + +`relix catalog trust` trusts the working directory's project, or the one it names. It +records a hash of every file under the `.relix/` directory in `~/.relix/trusted`: + +```shell +relix catalog trust +relix -e 'PROJECT name (Customers)' -o tsv +``` +``` +name +Alice +Bob +Carol +Dan +``` + +Changing, adding or removing any file under a trusted `.relix/` — a catalog file, the +`relixrc`, the `profiles.json` — makes it untrusted again until it is trusted again, so a +`git pull` cannot change what runs without you seeing it. `--list` shows each trusted +project and whether it has changed since: + +```shell +echo "Pending := { SELECT status = 'pending' (Orders) };" > .relix/catalog/30-pending.relix +relix catalog trust --list -o tsv +``` +``` +directory state +~/shop changed +``` + +```shell +relix catalog trust +relix catalog trust --list -o tsv +``` +``` +directory state +~/shop trusted +``` + +`--revoke DIR` removes a project from the record. + +Some things are trusted without asking, because you chose them: `~/.relix`, a script +named on the command line or given with `-e` or on standard input, and `--catalog` and +`$RELIX_CATALOG_PATH` files. `RELIX_TRUST_ALL=1` trusts every directory, for a CI +container whose checkout is the only thing in it. + +## A nearer declaration wins + +When two directories declare the same name, the **nearer** one's declaration replaces the +farther one's. That is an overlay, not an error, and it is what lets a sub-project change +one table while keeping everything else above it. `~/shop/reports` declares its own +`Orders`, a fixed sample to write reports against: + +```shell +cat reports/.relix/catalog/10-sample.relix +``` +``` +-- Reports are written against a fixed sample rather than the live orders. +source Orders from csv("../../sample-orders.csv") { + header: true, + schema: { order_id: NUMBER, customer_id: NUMBER, amount: NUMBER, status: STRING, placed: DATE } +}; +``` + +Every view the shop declares reads whichever `Orders` is nearest, so `BigSpenders` from +inside `reports` is computed over the sample: + +```shell +relix catalog trust reports +``` + +```shell +relix -e 'BigSpenders' +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 200 + Carol 200 +(2 rows) +``` + +```shell +relix -C reports -e 'BigSpenders' +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 500 + Dan 250 +(2 rows) +``` + +Each file is parsed first, the nearest declaration of every name kept, and only then are +the survivors defined together, so the order of the files decides which declaration +wins, never which can see which. A name is replaced only by a declaration of the same +kind: a relation by a relation, a function by a function. + +A file in an untrusted directory neither declares nor replaces anything. + +## Inspecting the catalog + +`relix catalog` answers what a run would load, without running anything: + +| Command | What it lists | +|---|---| +| `relix catalog files` | every catalog file, in load order, whether it is trusted, and the names it declares | +| `relix catalog ls` | every relation: its name, kind, number of columns, and the file whose declaration won | +| `relix catalog schema NAME` | the columns of a relation and their types | +| `relix catalog where NAME` | every declaration of a name, nearest first: the one that won, then those it replaced | + +Each writes rows like any query, as a table on a terminal and in any `-o` format: + +```shell +relix catalog files -o tsv +``` +``` +file trusted declares +~/.relix/catalog/00-regions.relix true Regions +~/shop/.relix/catalog/00-sources.relix true Customers Orders +~/shop/.relix/catalog/10-views.relix true Completed Spend BigSpenders +~/shop/.relix/catalog/20-warehouse.relix true warehouse Stock +~/shop/.relix/catalog/30-pending.relix true Pending +``` + +```shell +relix catalog ls -o tsv +``` +``` +name kind arity file +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/.relix/catalog/00-sources.relix +Pending QR 5 ~/shop/.relix/catalog/30-pending.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +The `kind` is the engine's: `SRC` a source, `INL` an inline table, `QR` a view, `DB` a +table reached through a connection. + +```shell +relix catalog schema Orders +``` +``` +── Orders ────────────────────────────────────────────────────────────────────── + column type + ─────────── ──── + order_id N + customer_id N + amount N + status S + placed D +(5 rows) +``` + +The type codes are the engine's: `N` a number, `S` a string, `B` a boolean, `D` a date, +`T` a time, `TS` a timestamp, `DUR` a duration, `?` any type. + +```shell +relix -C reports catalog where Orders -o tsv +``` +``` +file line kind state +~/shop/reports/.relix/catalog/10-sample.relix 2 source wins +~/shop/.relix/catalog/00-sources.relix 7 source shadowed +``` + +## Catalog files from elsewhere + +`--catalog FILE` adds a file to the catalog for one run, nearer than any `.relix/` +directory's. `$RELIX_CATALOG_PATH` names files the same way for every run. Both are +trusted, since you named them: + +```shell +cat > ~/rates.relix <<'EOF' +Rates := [ +| status | fee | +|-----------|-----| +| completed | 2 | +| pending | 0 | +]; +EOF +relix --catalog ~/rates.relix -e 'PROJECT order_id, fee (Orders JOIN Rates)' -o tsv +``` +``` +order_id fee +1 2 +2 2 +3 0 +4 2 +``` + +## Defaults: relixrc + +A `relixrc` file sets defaults for the commands run below it, as `key = value` lines; a +line starting `#` is a comment. The keys are: + +| Key | Default it sets | Beaten by | +|---|---|---| +| `output` | the output format | `-o`, `$RELIX_OUTPUT` | +| `profile` | the profile `${VAR}` placeholders resolve from | `-P`, `$RELIX_PROFILE` | + +The nearest file's value for a key wins. A key the command does not know, or a line that +is not `key = value`, is reported as a warning and ignored, so a file written for a later +version still works. A `relixrc` is part of its `.relix/` directory, so it is read only +once the directory is trusted: + +```shell +printf 'output = csv\n' > .relix/relixrc +relix catalog trust +relix -e 'PROJECT name (Customers)' +``` +``` +name +Alice +Bob +Carol +Dan +``` diff --git a/docs/guide/command-line/commands/bundle.md b/docs/guide/command-line/commands/bundle.md new file mode 100644 index 0000000..81ef74d --- /dev/null +++ b/docs/guide/command-line/commands/bundle.md @@ -0,0 +1,38 @@ +# relix bundle + +Prints each script's JSON bundle: diagnostics and, per query, the logical plan, the rewrites, the physical plan and the planner's events. A script that does not analyse still has one, holding its diagnostics. + +## Usage + +```text +relix bundle [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +A bundle is one JSON document per script, for a tool such as a query playground: + +```shell +relix bundle -e 'SELECT amount > 100 (Orders)' | jq -c 'keys' +``` +``` +["diagnostics","namespace","queries"] +``` diff --git a/docs/guide/command-line/commands/catalog.md b/docs/guide/command-line/commands/catalog.md new file mode 100644 index 0000000..e8ba414 --- /dev/null +++ b/docs/guide/command-line/commands/catalog.md @@ -0,0 +1,195 @@ +# relix catalog + +Inspects the catalog: the declarations a run takes from the .relix/ directories from ~/.relix down to the working directory, and from --catalog files. A project's .relix/ is loaded only once it is trusted (relix catalog trust). + +## Usage + +```text +relix catalog [OPTIONS] [COMMAND] +``` + +It also takes the [global options](relix.md#global-options). + +## relix catalog files + +Lists every catalog file, in the order they load, with the names each declares and whether it is trusted. An untrusted file is listed but not loaded. + +### Usage + +```text +relix catalog files [OPTIONS] +``` + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## relix catalog ls + +Lists the relations the catalog declares: name, kind, arity and the file whose declaration won. kind is the engine's: SRC a source, INL an inline table, QR a view, DB a connection's table. + +### Usage + +```text +relix catalog ls [OPTIONS] +``` + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## relix catalog schema + +Lists the columns of the relation NAME and their types, in order. + +### Usage + +```text +relix catalog schema [OPTIONS] NAME +``` + +### Arguments + +| Argument | Description | +|---|---| +| `NAME` | A relation the catalog declares. | + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +### Exit status + +| Status | Meaning | +|---|---| +| 0 | success | +| 3 | the catalog declares no relation NAME, or does not analyse | + +## relix catalog where + +Lists every declaration of NAME, nearest first: the one that won, then the ones it shadowed. A declaration in an untrusted directory is listed as untrusted. + +### Usage + +```text +relix catalog where [OPTIONS] NAME +``` + +### Arguments + +| Argument | Description | +|---|---| +| `NAME` | A name the catalog declares. | + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +### Exit status + +| Status | Meaning | +|---|---| +| 0 | success | +| 3 | the catalog declares no NAME | + +## relix catalog trust + +Trusts the .relix/ directory of DIR (default: the working directory), as its files are now: a project's catalog, relixrc and profiles.json are loaded only once it is trusted, and changing, adding or removing a file there means trusting it again. ~/.relix, --catalog files and the script itself are always trusted; RELIX_TRUST_ALL=1 trusts everything, for a CI container. + +### Usage + +```text +relix catalog trust [OPTIONS] [DIR] +``` + +### Arguments + +| Argument | Description | +|---|---| +| `DIR` | The project to trust, or its .relix/ directory. | + +### Options + +| Option | Description | +|---|---| +| `--list` | List the trusted directories and whether each has changed since. | +| `--revoke=DIR` | Stop trusting DIR. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +```shell +relix catalog ls -o tsv +``` +``` +name kind arity file +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/.relix/catalog/00-sources.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +```shell +relix catalog schema Customers -o tsv +``` +``` +column type +customer_id N +name S +city S +``` + +```shell +relix catalog where Orders -o tsv +``` +``` +file line kind state +~/shop/.relix/catalog/00-sources.relix 7 source wins +``` + +```shell +relix catalog trust --list -o tsv +``` +``` +directory state +~/shop trusted +~/shop/reports trusted +``` + +[The catalog](../catalog.md) covers discovery, the overlay and trust. diff --git a/docs/guide/command-line/commands/check.md b/docs/guide/command-line/commands/check.md new file mode 100644 index 0000000..253c6ca --- /dev/null +++ b/docs/guide/command-line/commands/check.md @@ -0,0 +1,70 @@ +# relix check + +Analyses scripts without running them, and reports what is wrong with them on standard error as FILE:LINE:COL: severity: message. + +```text +find . -name '*.relix' -print0 | xargs -0 relix check +``` + +## Usage + +```text +relix check [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--json` | Write the diagnostics to standard output as one JSON document instead. | + +It also takes the [global options](relix.md#global-options). + +## Exit status + +| Status | Meaning | +|---|---| +| 0 | every script analyses | +| 3 | a script does not parse or analyse | + +## Examples + +A script that analyses prints nothing, and exits 0: + +```shell +relix check scripts/top-customers.relix +``` + +A problem is printed on standard error, and the status is 3: + +```shell +relix check scripts/broken.relix 2>&1; echo "exit $?" +``` +``` +scripts/broken.relix:1:30: error: Undefined relation: 'Ordrs' +exit 3 +``` + +`--json` writes the diagnostics to standard output instead, as one document for a tool to +read: + +```shell +relix check --json scripts/broken.relix; echo "exit $?" +``` +``` +{"diagnostics":[{"file":"scripts/broken.relix","line":1,"column":30,"severity":"error","message":"Undefined relation: 'Ordrs'"}]} +exit 3 +``` + +[Pipelines](../pipelines.md#lint-every-script) checks every script in a tree. diff --git a/docs/guide/command-line/commands/completion.md b/docs/guide/command-line/commands/completion.md new file mode 100644 index 0000000..023a3f9 --- /dev/null +++ b/docs/guide/command-line/commands/completion.md @@ -0,0 +1,39 @@ +# relix completion + +Prints a completion script for SHELL: bash, zsh, fish or powershell. + +```text +bash: source <(relix completion bash) +zsh: source <(relix completion zsh) +fish: relix completion fish > ~/.config/fish/completions/relix.fish +powershell: relix completion powershell | Out-String | Invoke-Expression +``` + +## Usage + +```text +relix completion [OPTIONS] SHELL +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SHELL` | bash, zsh, fish or powershell. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +The release archives carry each shell's script in `completions/`, and Homebrew installs +them. To load one by hand, add the line for your shell to its startup file: + +```text +bash source <(relix completion bash) +zsh source <(relix completion zsh) +fish relix completion fish > ~/.config/fish/completions/relix.fish +PowerShell relix completion powershell | Out-String | Invoke-Expression +``` + +The zsh script is also a completion function, so it can be saved as `_relix` in a +directory on `$fpath` instead. diff --git a/docs/guide/command-line/commands/connectors.md b/docs/guide/command-line/commands/connectors.md new file mode 100644 index 0000000..a6b7fc7 --- /dev/null +++ b/docs/guide/command-line/commands/connectors.md @@ -0,0 +1,82 @@ +# relix connectors + +Lists and installs connector plugins, such as mongodb. A source of a type no installed connector reads needs one, installed here ahead of a run, or fetched during one with --allow-download. + +## Usage + +```text +relix connectors [OPTIONS] [COMMAND] +``` + +It also takes the [global options](relix.md#global-options). + +## relix connectors ls + +Lists the connectors installed: those shipped with relix and the plugins installed in ~/.relix/connectors. --available adds the ones the hosted catalog offers, which it fetches. + +### Usage + +```text +relix connectors ls [OPTIONS] +``` + +### Options + +| Option | Description | +|---|---| +| `--available` | Also list the connectors that can be installed, from the hosted catalog. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## relix connectors install + +Downloads the connector plugin TYPE, checks its checksums, and installs it. + +### Usage + +```text +relix connectors install [OPTIONS] TYPE +``` + +### Arguments + +| Argument | Description | +|---|---| +| `TYPE` | A connector type, such as mongodb. | + +It also takes the [global options](relix.md#global-options). + +### Exit status + +| Status | Meaning | +|---|---| +| 0 | installed, or already installed | +| 2 | the catalog has no connector of that type | +| 5 | the download failed | + +## Examples + +The connectors that ship with `relix`, and any plugins installed in `~/.relix/connectors`: + +```shell +relix connectors ls +``` +``` +── connectors ────────────────────────────────────────────────────────────────── + type installed + ────── ───────── + clf true + csv true + gedcom true + log true +(4 rows) +``` + +`relix connectors ls --available` adds the connectors that can be installed, which it +fetches from the hosted catalog, and `relix connectors install TYPE` installs one. A +source whose type no installed connector reads — `mongodb`, say — fails with exit status +5, unless `--allow-download` lets the run fetch the connector. diff --git a/docs/guide/command-line/commands/doc.md b/docs/guide/command-line/commands/doc.md new file mode 100644 index 0000000..5108066 --- /dev/null +++ b/docs/guide/command-line/commands/doc.md @@ -0,0 +1,75 @@ +# relix doc + +Shows the reference page for TOPIC, found by glyph, keyword or name (σ, select, selection), through $PAGER on a terminal. With no TOPIC, lists the pages. + +## Usage + +```text +relix doc [OPTIONS] [TOPIC] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `TOPIC` | A glyph, keyword or name, such as σ, select, join or Len. | + +## Options + +| Option | Description | +|---|---| +| `--function` | Look TOPIC up among the functions only, or list only them. | +| `--markdown` | Print the page's markdown as it is, not rendered. | +| `--no-pager` | Write the page to standard output even on a terminal. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## Exit status + +| Status | Meaning | +|---|---| +| 0 | success | +| 2 | no page has that topic | + +## Examples + +A page is found by its glyph, its keyword or its name: + +```shell +relix doc --markdown σ | head -6 +``` +``` +# Name: Selection (σ / SELECT) + +# Syntax: +σ (Relation) +SELECT (Relation) +``` + +With no topic, `relix doc` lists the pages, as rows: + +```shell +relix doc -o tsv | head -4 +``` +``` +topic category title summary +getting-started guide Getting started Your first script — declare data, name a result, run it +glossary guide Glossary The vocabulary — relation, tuple, bag vs set, materialisation, pushdown +σ operator Selection Filter rows by a condition +``` + +`--function` lists, or looks up, the functions only: + +```shell +relix doc --function -o tsv | grep -i '^round' +``` +``` +Round function Round Round to nearest +``` + +On a terminal a page is shown through `$PAGER` (`less` when it is not set); `--no-pager` +writes it straight to standard output. diff --git a/docs/guide/command-line/commands/drivers.md b/docs/guide/command-line/commands/drivers.md new file mode 100644 index 0000000..de2182d --- /dev/null +++ b/docs/guide/command-line/commands/drivers.md @@ -0,0 +1,79 @@ +# relix drivers + +Lists and installs JDBC drivers. A connection to a database needs its driver, installed here ahead of a run, or fetched during one with --allow-download. + +## Usage + +```text +relix drivers [OPTIONS] [COMMAND] +``` + +It also takes the [global options](relix.md#global-options). + +## relix drivers ls + +Lists the drivers relix can install: name, the JDBC scheme each serves, and whether one is installed. + +### Usage + +```text +relix drivers ls [OPTIONS] +``` + +### Options + +| Option | Description | +|---|---| +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | + +It also takes the [global options](relix.md#global-options). + +## relix drivers install + +Downloads the driver NAME, checks its checksum, and installs it. + +### Usage + +```text +relix drivers install [OPTIONS] NAME +``` + +### Arguments + +| Argument | Description | +|---|---| +| `NAME` | A driver relix drivers ls lists, such as postgresql. | + +It also takes the [global options](relix.md#global-options). + +### Exit status + +| Status | Meaning | +|---|---| +| 0 | installed, or already installed | +| 2 | no driver has that name | +| 5 | the download failed | + +## Examples + +```shell +relix drivers ls +``` +``` +── drivers ───────────────────────────────────────────────────────────────────── + name scheme installed + ────────── ──────────────── ───────── + postgresql jdbc:postgresql: false + mysql jdbc:mysql: false +(2 rows) +``` + +`relix drivers install NAME` downloads one of those drivers, checks its checksum, and +installs it in `~/.relix/drivers`, or in `$RELIX_DRIVERS` when that is set. A run that +needs a driver that is not installed fails with exit status 5, unless `--allow-download` +lets it fetch the driver itself. The H2 driver is part of +`relix`, and needs no installing. A driver `relix drivers ls` does not list is installed +by copying its jar into the driver directory. diff --git a/docs/guide/command-line/commands/explain.md b/docs/guide/command-line/commands/explain.md new file mode 100644 index 0000000..4520817 --- /dev/null +++ b/docs/guide/command-line/commands/explain.md @@ -0,0 +1,50 @@ +# relix explain + +Prints the physical plan of each query: how the engine would run it. + +## Usage + +```text +relix explain [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--json` | Write one JSON object per query, {"script", "query", "plan"}, one per line. | +| `--query=NAME` | Explain only the query NAME. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +```shell +relix explain -e 'GROUP city, COUNT(*) -> n (Customers JOIN Orders)' +``` +``` +Aggregate +└─ Join NATURAL/HASH build=RIGHT + ├─ Scan Customers + └─ Scan Orders +``` + +`--json` writes each query's plan as a JSON object instead: + +```shell +relix explain --json -e 'SELECT amount > 100 (Orders)' | jq -c 'keys' +``` +``` +["plan","query","script"] +``` diff --git a/docs/guide/command-line/commands/fmt.md b/docs/guide/command-line/commands/fmt.md new file mode 100644 index 0000000..fdfed97 --- /dev/null +++ b/docs/guide/command-line/commands/fmt.md @@ -0,0 +1,96 @@ +# relix fmt + +Prints scripts canonically, keeping their comments. With no SCRIPT, formats standard input. + +## Usage + +```text +relix fmt [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files; - is standard input, which is also read when none is named. | + +## Options + +| Option | Description | +|---|---| +| `-w` | Rewrite each file that changes in place, instead of printing it. | +| `--check` | Change nothing; print the name of each file that would change, and exit 1 if any would. | +| `--glyphs` | Write operators as glyphs: σ, ⋈, ∧ (the default). | +| `--keywords` | Write operators as ASCII keywords: SELECT, JOIN, AND. | + +It also takes the [global options](relix.md#global-options). + +## Exit status + +| Status | Meaning | +|---|---| +| 0 | success; with --check, no file would change | +| 1 | with --check, a file would change | +| 3 | a script does not parse | + +## Examples + +```shell +cat scripts/messy.relix +``` +``` +-- Orders worth a look +Large:={select amount>100(Orders)}; +query {project order_id,amount (Large)}; +``` + +`relix fmt` prints it canonically — one statement per line, spaced, each operator in one +spelling — keeping its comments: + +```shell +relix fmt scripts/messy.relix +``` +``` +-- Orders worth a look +Large := { σ amount > 100 (Orders) }; +query { π order_id, amount (Large) }; +``` + +`--keywords` writes the operators as ASCII keywords instead of glyphs: + +```shell +relix fmt --keywords scripts/messy.relix +``` +``` +-- Orders worth a look +Large := { SELECT amount > 100 (Orders) }; +query { PROJECT order_id, amount (Large) }; +``` + +`--check` changes nothing and names each file that would change, exiting 1 if there is +one; `-w` rewrites each such file in place: + +```shell +relix fmt --check --keywords scripts/messy.relix scripts/by-city.relix; echo "exit $?" +``` +``` +scripts/messy.relix +exit 1 +``` + +```shell +relix fmt -w --keywords scripts/messy.relix +relix fmt --check --keywords scripts/messy.relix; echo "exit $?" +``` +``` +exit 0 +``` + +With no file, `relix fmt` formats standard input to standard output: + +```shell +echo 'query {select amount>100(Orders)};' | relix fmt +``` +``` +query { σ amount > 100 (Orders) }; +``` diff --git a/docs/guide/command-line/commands/help.md b/docs/guide/command-line/commands/help.md new file mode 100644 index 0000000..9f4359d --- /dev/null +++ b/docs/guide/command-line/commands/help.md @@ -0,0 +1,30 @@ +# relix help + +When no COMMAND is given, the usage help for the main command is displayed. If a COMMAND is specified, the help for that command is shown. + +## Usage + +```text +relix help [OPTIONS] [COMMAND] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `COMMAND` | The COMMAND to display the usage help message for. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +`relix help COMMAND` prints what `relix COMMAND --help` prints: + +```shell +relix help version | head -3 +``` +``` +Usage: relix version [-hNqVv] [--allow-download] [--remote] [-C=DIR] + [--max-fixpoint-rounds=N] [--max-materialized-rows=N] + [--max-processed-rows=N] [--now=INSTANT] [-P=NAME] +``` diff --git a/docs/guide/command-line/commands/ir.md b/docs/guide/command-line/commands/ir.md new file mode 100644 index 0000000..968419d --- /dev/null +++ b/docs/guide/command-line/commands/ir.md @@ -0,0 +1,95 @@ +# relix ir + +Prints the IR report: the symbols, views and queries the analyser made of each script. + +## Usage + +```text +relix ir [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +```shell +relix ir -e 'Big := { SELECT amount > 100 (Orders) };' -e 'query Big;' +``` +``` +════════════════════════════════════════════════════════════════════════════════ +RELIX IR namespace=default +════════════════════════════════════════════════════════════════════════════════ + kind: SRC=source INL=inline SYS=system QR=view DB=database + FN=function TVF=table-fn LIT=truth-literal + type: N=number S=string B=boolean ?=any [x]=array {…}=struct + D=date T=time TS=timestamp DUR=duration + mat: [bag]=bag [set]=dedup-set [sort]=sorted (stream=no label) + +── SYMBOLS ───────────────────────────────────────────────────────────────────── + Big QR order_id:N customer_id:N amount:N status:S placed:D + BigSpenders QR name:S total:N + catalog SYS name:S kind:S arity:N schema_code:S + columns SYS relation:S column:S type:S ordinal:N distinct_count:N … + Completed QR order_id:N customer_id:N amount:N status:S placed:D + connections SYS name:S dialect:S + Customers SRC customer_id:N name:S city:S + cycles QR name:S + dependencies QR dependent:S depends_on:S + deps TVF (r:S)→relation + events SYS seq:N stage:S code:S description:S target:S rows:N el… + find TVF (col:S)→relation + funcs TVF (cat:S)→relation + functions SYS name:S category:S arity:N max_arity:N return_type:S pu… + impact TVF (r:S)→relation + keys SYS relation:S key:N ordinal:N column:S + Orders SRC order_id:N customer_id:N amount:N status:S placed:D + plan SYS query:S node_id:N parent_id:N ordinal:N op:S label:S … + Regions INL city:S region:S + relations SYS name:S kind:S namespace:S materialization:S row_count:N… + rules QR seq:N code:S description:S target:S + schema TVF (rel:S)→relation + Spend QR customer_id:N total:N + Stock SRC sku:S on_hand:N + unused QR name:S + version SYS component:S kind:S version:S + +── EXPRESSION TREES ──────────────────────────────────────────────────────────── +Big [QR] order_id:N customer_id:N amount:N status:S placed:D + σ amount > 100 + └─ Orders [SRC] + +BigSpenders [QR] name:S total:N + π name, total + └─ ⋈ + ├─ Customers [SRC] + └─ σ total ≥ 200 + └─ Spend [QR] + +Completed [QR] order_id:N customer_id:N amount:N status:S placed:D + σ status = "completed" + └─ Orders [SRC] + +Spend [QR] customer_id:N total:N + γ [customer_id] SUM(amount)→total [bag] + └─ Completed [QR] + +── ROOT QUERIES ──────────────────────────────────────────────────────────────── + query Big [QR] +════════════════════════════════════════════════════════════════════════════════ +``` diff --git a/docs/guide/command-line/commands/optimize.md b/docs/guide/command-line/commands/optimize.md new file mode 100644 index 0000000..8fbd381 --- /dev/null +++ b/docs/guide/command-line/commands/optimize.md @@ -0,0 +1,68 @@ +# relix optimize + +Prints the script with each query rewritten by the optimiser, so that it works as a filter: relix optimize slow.relix > fast.relix + +## Usage + +```text +relix optimize [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--report` | Print the rules that fired and each query before and after, instead. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +The rewritten script is printed in the same form the engine runs it in: + +```shell +relix optimize -e "query { SELECT city = 'London' (Customers JOIN Orders) };" +``` +``` +query { (σ city = "London" (Customers)) ⋈ (Orders) }; +``` + +`--report` prints the rules that fired, and each query before and after: + +```shell +relix optimize --report -e "query { SELECT city = 'London' (Customers JOIN Orders) };" +``` +``` +════════════════════════════════════════════════════════════════════════════════ +RELIX OPTIMIZER namespace=default transformations=1 +════════════════════════════════════════════════════════════════════════════════ + +── SUMMARY ───────────────────────────────────────────────────────────────────── + SEL-005 Selection pushed into join input ...................................×1 + +── TREES ─────────────────────────────────────────────────────────────────────── + +── + before: σ city = "London" ((Customers) ⋈ (Orders)) + after: (σ city = "London" (Customers)) ⋈ (Orders) + +── DETAIL ────────────────────────────────────────────────────────────────────── + +── (1 transformation) + [SEL-005] selection pushed into left input of natural join @ :1:9 +════════════════════════════════════════════════════════════════════════════════ +``` + +[Pipelines](../pipelines.md#optimise-a-script-and-review-the-change) compares a script +with its rewrite. diff --git a/docs/guide/command-line/commands/provenance.md b/docs/guide/command-line/commands/provenance.md new file mode 100644 index 0000000..2c4346d --- /dev/null +++ b/docs/guide/command-line/commands/provenance.md @@ -0,0 +1,86 @@ +# relix provenance + +Runs each query and prints its rows, each annotated with where it came from in a provenance semiring. + +## Usage + +```text +relix provenance [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--semiring=NAME` | The semiring: boolean, counting, tropical, security, lineage, cheapest-route, or one an installed library offers (default: counting). | +| `--weight=COLUMN` | The column a weighted closure takes each edge's weight from, such as --semiring tropical --weight cost for shortest paths. | +| `-o`, `--output=FORMAT` | table or json (default: table on a terminal, json in a pipe). | +| `--query=NAME` | Annotate only the query NAME. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +The default semiring, `counting`, says in how many ways each row is derived. Two London +customers each derive the row `London`: + +```shell +relix provenance -e 'PROJECT city (Customers)' +``` +``` + +------ ------ +city [prov] +------ ------ +London 2 +Leeds 1 +York 1 +------ ------ +(3 tuples; provenance semiring: counting) +``` + +The `lineage` semiring names the source rows each result row came from: + +```shell +relix provenance --semiring=lineage -e 'PROJECT city (SELECT amount > 100 (Customers JOIN Orders))' +``` +``` + +------ ------------------------------------------- +city [prov] +------ ------------------------------------------- +London Customers#1·Orders#1 + Customers#3·Orders#4 +------ ------------------------------------------- +(1 tuple; provenance semiring: lineage) +``` + +In a pipe, or with `-o json`, the rows and their annotations are one JSON document: + +```shell +relix provenance -e 'PROJECT city (Customers)' -o json +``` +``` +{ + "query": "", + "semiring": "counting", + "schema": ["city"], + "tuples": [ + {"row": {"city": "London"}, "provenance": "2"}, + {"row": {"city": "Leeds"}, "provenance": "1"}, + {"row": {"city": "York"}, "provenance": "1"} + ] +} +``` + +The engine's [provenance page](../../provenance.md) explains each semiring. diff --git a/docs/guide/command-line/commands/relix.md b/docs/guide/command-line/commands/relix.md new file mode 100644 index 0000000..948ac64 --- /dev/null +++ b/docs/guide/command-line/commands/relix.md @@ -0,0 +1,155 @@ +# relix + +Runs Relix relational-algebra scripts as a stage of a Unix pipeline. Rows go to standard output; everything else goes to standard error. + +## Usage + +```text +relix [OPTIONS] [SCRIPT...] [COMMAND] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Run options + +With no command, `relix` is `relix run`, and takes its options: + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | +| `--line-buffered` | Flush each row as it is written, for a reader that wants it at once. | +| `--query=NAME` | Print only the query NAME. A machine format (tsv, csv, ndjson, json) prints one query: the script's last, unless this names another. | +| `--all` | With ndjson, print every query, each row tagged "_query": NAME. | +| `--fail-empty` | Exit 1 when the queries printed return no rows. | +| `--fail-rows` | Exit 1 when the queries printed return any row. | +| `--trace=FILE` | Write the event feed (what was rewritten, planned and run) to standard error, or to FILE with --trace=FILE, while the rows go to standard output. | + +## Global options + +Every command takes these, before or after its name. + +| Option | Description | +|---|---| +| `-C DIR` | Act as if started in DIR: relative paths and .relix/ discovery start there. | +| `-P`, `--profile=NAME` | The profile ${VAR} placeholders resolve from (default: $RELIX_PROFILE). | +| `-D NAME=VALUE` | Set one ${VAR} for this run; it beats the profile and the environment. Not for secrets: it lands in shell history and ps. | +| `-a`, `--arg=NAME=VALUE` | Bind the query parameter $NAME to VALUE. Repeatable. The value is bound, never pasted into the script, so it cannot change the query. | +| `-N`, `--no-catalog` | Ignore the .relix/ directories' catalogs and relixrc; --catalog files still load. | +| `--catalog=FILE` | Add a catalog file, nearer than the .relix/ directories'. Repeatable. $RELIX_CATALOG_PATH adds files too, separated as in PATH. | +| `-q`, `--quiet` | Print errors only, not warnings. | +| `-v`, `--verbose` | Also print notices (drivers loaded); -vv adds timings. | +| `--now=INSTANT` | Pin NOW() to an ISO-8601 instant, such as 2026-07-22T12:00:00Z (default: $RELIX_NOW). | +| `--max-fixpoint-rounds=N` | Fail a recursive query that needs more than N rounds. | +| `--max-materialized-rows=N` | Fail a query in which one blocking operator buffers more than N rows. | +| `--max-processed-rows=N` | Fail a query that processes more than N rows in all. | +| `--timeout=DURATION` | Fail a query that runs longer than DURATION: 500ms, 30s, 5m, 1h, or ISO-8601 (PT30S). | +| `--sandbox=FILE` | Run under the sandbox FILE describes. | +| `--remote` | Permit http(s) file locations. | +| `--allow-download` | Permit fetching a missing JDBC driver or connector during the run. | +| `-h`, `--help` | Show this help message and exit. | +| `-V`, `--version` | Print version information and exit. | + +## Commands + +| Command | What it does | +|---|---| +| [`relix run`](run.md) | Runs scripts and writes their queries' rows to standard output. | +| [`relix check`](check.md) | Analyses scripts without running them, and reports what is wrong with them on standard error as FILE:LINE:COL: severity: message. | +| [`relix explain`](explain.md) | Prints the physical plan of each query: how the engine would run it. | +| [`relix optimize`](optimize.md) | Prints the script with each query rewritten by the optimiser, so that it works as a filter: relix optimize slow.relix > fast.relix. | +| [`relix trace`](trace.md) | Runs each query and prints its event feed, what the engine rewrote, planned and ran, discarding the rows. | +| [`relix bundle`](bundle.md) | Prints each script's JSON bundle: diagnostics and, per query, the logical plan, the rewrites, the physical plan and the planner's events. | +| [`relix ir`](ir.md) | Prints the IR report: the symbols, views and queries the analyser made of each script. | +| [`relix provenance`](provenance.md) | Runs each query and prints its rows, each annotated with where it came from in a provenance semiring. | +| [`relix fmt`](fmt.md) | Prints scripts canonically, keeping their comments. | +| [`relix catalog`](catalog.md) | Inspects the catalog: the declarations a run takes from the .relix/ directories from ~/.relix down to the working directory, and from --catalog files. | +| [`relix drivers`](drivers.md) | Lists and installs JDBC drivers. | +| [`relix connectors`](connectors.md) | Lists and installs connector plugins, such as mongodb. | +| [`relix doc`](doc.md) | Shows the reference page for TOPIC, found by glyph, keyword or name (σ, select, selection), through $PAGER on a terminal. | +| [`relix help`](help.md) | When no COMMAND is given, the usage help for the main command is displayed. | +| [`relix version`](version.md) | Prints the versions of relix and of the Relix engine it runs. | +| [`relix completion`](completion.md) | Prints a completion script for SHELL: bash, zsh, fish or powershell. | + +## Exit status + +| Status | Meaning | +|---|---| +| 0 | success | +| 1 | an assertion failed (--fail-empty, --fail-rows) | +| 2 | usage error (bad option, nothing to run) | +| 3 | the script did not parse or analyse | +| 4 | execution failed (a data error, a limit hit) | +| 5 | environment (a missing driver, an untrusted catalog, a connection refused) | +| 70 | internal error in relix | +| 130 | interrupted (SIGINT) | +| 141 | standard output closed (SIGPIPE) | + +## Examples + +With no command, `relix` runs scripts, as [`relix run`](run.md) does: + +```shell +relix -e 'PROJECT name, city (Customers)' -o tsv +``` +``` +name city +Alice London +Bob Leeds +Carol London +Dan York +``` + +Global options go before or after the command's name: + +```shell +relix -C reports catalog ls -o tsv --no-header +``` +``` +BigSpenders QR 2 ~/shop/.relix/catalog/10-views.relix +Completed QR 5 ~/shop/.relix/catalog/10-views.relix +Customers SRC 3 ~/shop/.relix/catalog/00-sources.relix +Orders SRC 5 ~/shop/reports/.relix/catalog/10-sample.relix +Regions INL 2 ~/.relix/catalog/00-regions.relix +Spend QR 2 ~/shop/.relix/catalog/10-views.relix +Stock SRC 2 ~/shop/.relix/catalog/20-warehouse.relix +``` + +## Environment + +| Variable | Effect | +|---|---| +| `RELIX_OUTPUT` | the output format when `-o` is not given | +| `RELIX_PROFILE` | the profile when `-P` is not given | +| `RELIX_NOW` | the instant `NOW()` returns when `--now` is not given | +| `RELIX_CATALOG_PATH` | catalog files to load, separated as in `PATH`, before the `--catalog` files | +| `RELIX_TRUST_ALL` | `1` or `true` trusts every project's `.relix/` directory | +| `RELIX_DRIVERS` | the directory JDBC drivers are installed in and loaded from, instead of `~/.relix/drivers` | +| `NO_COLOR` | when set, a table on a terminal is not coloured unless `--color=always` | +| `PAGER` | the program [`relix doc`](doc.md) shows a page through on a terminal; `cat` or empty shows it directly | + +A `${NAME}` placeholder in a declaration may also be resolved from the environment; see +[Secrets and profiles](../secrets.md). + +## Files + +| Path | What it holds | +|---|---| +| `~/.relix/catalog/*.relix` | your own catalog, which every directory under your home sees | +| `~/.relix/profiles.json` | your own profiles | +| `~/.relix/trusted` | the projects you have trusted, with a hash of each one's files | +| `~/.relix/drivers/` | installed JDBC drivers | +| `~/.relix/connectors/` | installed connector plugins | +| `~/.relix/cache/files/` | files fetched over `https` by a file connection | +| `.relix/catalog/*.relix`, `.relix/relixrc`, `.relix/profiles.json`, `.relix/root` | a project's catalog, defaults, profiles, and the marker that ends the walk up; see [The catalog](../catalog.md) | diff --git a/docs/guide/command-line/commands/run.md b/docs/guide/command-line/commands/run.md new file mode 100644 index 0000000..5a62bc1 --- /dev/null +++ b/docs/guide/command-line/commands/run.md @@ -0,0 +1,78 @@ +# relix run + +Runs scripts and writes their queries' rows to standard output. + +## Usage + +```text +relix run [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `-o`, `--output=FORMAT` | The rows' format: table, tsv, csv, ndjson, json or markdown (default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe). | +| `--no-header` | Leave out the header row of csv and tsv. | +| `--null=STRING` | How NULL is written (default: empty in csv, tsv and markdown; NULL in a table). | +| `--color=WHEN` | Colour a table: auto, always or never (default: auto). auto colours on a terminal unless $NO_COLOR is set. | +| `--line-buffered` | Flush each row as it is written, for a reader that wants it at once. | +| `--query=NAME` | Print only the query NAME. A machine format (tsv, csv, ndjson, json) prints one query: the script's last, unless this names another. | +| `--all` | With ndjson, print every query, each row tagged "_query": NAME. | +| `--fail-empty` | Exit 1 when the queries printed return no rows. | +| `--fail-rows` | Exit 1 when the queries printed return any row. | +| `--trace=FILE` | Write the event feed (what was rewritten, planned and run) to standard error, or to FILE with --trace=FILE, while the rows go to standard output. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +`relix run` is what `relix` does with no command, so these two are the same: + +```shell +relix run -e 'GROUP status, SUM(amount) -> total (Orders)' +``` +``` +── ────────────────────────────────────────────────────────────── + status total + ───────── ───── + completed 400 + pending 50 +(2 rows) +``` + +```shell +relix -e 'GROUP status, SUM(amount) -> total (Orders)' -o csv +``` +``` +status,total +completed,400 +pending,50 +``` + +Each script file is a run of its own, with its own declarations: + +```shell +relix run scripts/top-customers.relix -a city=Leeds scripts/by-city.relix -o tsv +``` +``` +name total +Alice 200 +Carol 200 +name +Bob +``` + +[Binding inputs](../inputs.md), [Output formats](../output.md) and +[Pipelines](../pipelines.md) cover the options in use. diff --git a/docs/guide/command-line/commands/trace.md b/docs/guide/command-line/commands/trace.md new file mode 100644 index 0000000..ab01e12 --- /dev/null +++ b/docs/guide/command-line/commands/trace.md @@ -0,0 +1,53 @@ +# relix trace + +Runs each query and prints its event feed, what the engine rewrote, planned and ran, discarding the rows. relix run --trace keeps the rows too. + +## Usage + +```text +relix trace [OPTIONS] [SCRIPT...] +``` + +## Arguments + +| Argument | Description | +|---|---| +| `SCRIPT...` | Script files, each run in a session of its own; - is standard input. With neither -e nor SCRIPT, standard input is read when it is not a terminal. | + +## Options + +| Option | Description | +|---|---| +| `-e`, `--expr=TEXT` | Relix text to run. Repeatable: the texts are one script, in order. A bare relational expression runs as a query. | +| `-i`, `--input=NAME=[FORMAT:]LOCATION` | Bind the relation NAME to a file, an http(s) URL (with --remote), or - for standard input. FORMAT is csv, tsv, json or ndjson; without it the extension decides, and standard input must name one. Repeatable. | +| `--schema=NAME={...}` | The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one. | +| `--infer-rows=N` | Infer an input's heading from its first N records (default: 1000). | +| `--unbounded=NAME` | The input NAME never ends, as tail -f's does: its rows stream as they arrive, and a query that would have to read all of it first is refused. Repeatable. | +| `--query=NAME` | Trace only the query NAME. | + +It also takes the [global options](relix.md#global-options). + +## Examples + +Each event ends with how long it took, which varies from run to run, so here `cut` +leaves the timings off: + +```shell +relix trace -e 'SELECT amount > 100 (Orders)' | cut -d'(' -f1 +``` +``` +[EXECUTE ] orders SCAN scanned 4 rows +[EXECUTE ] ROWS query delivered 2 rows +``` + +`relix run --trace` writes the same feed to standard error while the rows go to standard +output, and `--trace=FILE` writes it to a file: + +```shell +relix --trace=trace.txt -e 'SELECT amount > 100 (Orders)' -o tsv +``` +``` +order_id customer_id amount status placed +1 1 120 completed 2026-09-02 +4 3 200 completed 2026-10-01 +``` diff --git a/docs/guide/command-line/commands/version.md b/docs/guide/command-line/commands/version.md new file mode 100644 index 0000000..0419b2e --- /dev/null +++ b/docs/guide/command-line/commands/version.md @@ -0,0 +1,23 @@ +# relix version + +Prints the versions of relix and of the Relix engine it runs. + +## Usage + +```text +relix version [OPTIONS] +``` + +It also takes the [global options](relix.md#global-options). + +## Examples + +`relix version`, like `relix --version`, prints one line: the version of the command, +then the version of the Relix engine it runs, as + +```text +relix 1.0.0 (engine 1.0.0) +``` + +The two are numbered independently: a release of the command names the engine release +it was built on. diff --git a/docs/guide/command-line/getting-started.md b/docs/guide/command-line/getting-started.md new file mode 100644 index 0000000..f5b036d --- /dev/null +++ b/docs/guide/command-line/getting-started.md @@ -0,0 +1,218 @@ +# The relix command + +`relix` runs Relix scripts from a shell. It is built to be one stage of a Unix pipeline: +the script comes from the command line, a file or standard input; data comes from files, +standard input or the tables a project declares; and the rows go to standard output in a +format the next program reads. Everything else — errors, warnings, progress — goes to +standard error, so a pipeline never mistakes a message for a row. + +This page installs the command, runs a first query, and shows the shape every command +shares. The pages after it take one subject each, and the +[command reference](commands/relix.md) lists every command and option. + +## Installing + +`relix` ships as a self-contained archive for macOS, Linux and Windows: a launcher, its +own Java runtime, the man pages and completion scripts for bash, zsh, fish and +PowerShell. Nothing else needs to be installed, and no Java is needed on the machine. + +On macOS and Linux, with Homebrew: + +```text +brew install DarkCollective/relix/relix +``` + +On Windows, with Scoop: + +```text +scoop bucket add relix https://github.com/DarkCollective/scoop-relix +scoop install relix +``` + +Or download the archive for your platform from the +[releases page](https://github.com/DarkCollective/relix-cli/releases), unpack it, and put +its `bin` directory on your `PATH`. Each release lists the archives' SHA-256 checksums +in `checksums.txt`. + +`relix version` prints the version of the command and the version of the Relix engine it +runs, as `relix VERSION (engine VERSION)`. The two are numbered independently. + +## The examples on these pages + +Every example in this section is run on each build of the command, and the block under +it is what it printed. The examples run in `~/shop`, a small project with a customer +file, an order file and a stock database: + +```shell +cat data/customers.csv data/orders.csv +``` +``` +customer_id,name,city +1,Alice,London +2,Bob,Leeds +3,Carol,London +4,Dan,York +order_id,customer_id,amount,status,placed +1,1,120,completed,2026-09-02 +2,1,80,completed,2026-09-14 +3,2,50,pending,2026-09-20 +4,3,200,completed,2026-10-01 +``` + +The project declares its tables once, in `~/shop/.relix/catalog/`, so the examples can +name `Customers` and `Orders` without declaring them. [The catalog](catalog.md) explains +how. A path printed under your home directory is shown as `~`. + +## A first query + +`-e` takes Relix text. A bare expression is run as a query: + +```shell +relix -e 'SELECT amount > 100 (Orders)' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id customer_id amount status placed + ──────── ─────────── ────── ───────── ────────── + 1 1 120 completed 2026-09-02 + 4 3 200 completed 2026-10-01 +(2 rows) +``` + +On a terminal the rows are a table. In a pipe they are tab-separated values, one row per +line under a header, which is what `cut`, `sort`, `awk` and `column` expect: + +```shell +relix -e 'SELECT amount > 100 (Orders)' | cut -f1,3 +``` +``` +order_id amount +1 120 +4 200 +``` + +The language itself — every operator, predicate and function — is the subject of the +[language reference](../../reference/README.md). Operators have a Unicode glyph and an ASCII +keyword, and the command reads both: `σ amount > 100 (Orders)` is the same query. + +## Where the script comes from + +A command reads its script from the first of these that is given: + +1. **`-e TEXT`**, repeatable. The texts are joined, in order, into one script, so one + `-e` can declare a view and the next use it. A text that is a bare relational + expression rather than statements is run as `query { … };`. +2. **Script files**, named as arguments. Each runs in a session of its own, so two files on + one command line never see each other's declarations. `-` names standard input. +3. **Standard input**, when nothing is named and standard input is not a terminal. + +With none of them, the command prints its usage and exits 2. + +```shell +relix -e 'Big := { SELECT amount > 100 (Orders) };' -e 'query { PROJECT order_id (Big) };' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id + ──────── + 1 + 4 +(2 rows) +``` + +A script file holds statements, as a script always does: + +```shell +cat scripts/top-customers.relix +``` +``` +-- The customers who have spent the most, best first. +query { SORT total DESC (PROJECT name, total (Customers JOIN Spend)) }; +``` + +```shell +relix scripts/top-customers.relix +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 200 + Carol 200 +(2 rows) +``` + +and a script can arrive on standard input: + +```shell +echo 'query { PROJECT name (Customers) };' | relix -o csv +``` +``` +name +Alice +Bob +Carol +Dan +``` + +A relative path in a script — a `csv("…")` source, an `import` — resolves against the +script file's own directory, and in `-e` text or standard input against the working +directory. `-C DIR` makes the command act as if it were started in `DIR`. + +## Commands + +`relix` with no command name runs scripts; it is `relix run`. Every other job is a command +of its own, and each writes one thing to standard output: + +| Command | Standard output | +|---|---| +| [`relix run`](commands/run.md) | the rows of the queries (the default) | +| [`relix check`](commands/check.md) | nothing; problems go to standard error, or `--json` to standard output | +| [`relix explain`](commands/explain.md) | the physical plan of each query | +| [`relix optimize`](commands/optimize.md) | the script, rewritten by the optimiser | +| [`relix trace`](commands/trace.md) | what the engine rewrote, planned and ran | +| [`relix bundle`](commands/bundle.md) | one JSON document per script: diagnostics, plans and events | +| [`relix ir`](commands/ir.md) | the symbols, views and queries the analyser made of the script | +| [`relix provenance`](commands/provenance.md) | the rows, each with where it came from | +| [`relix fmt`](commands/fmt.md) | the script, printed canonically | +| [`relix catalog`](commands/catalog.md) | the catalog's files, relations, columns and trust | +| [`relix drivers`](commands/drivers.md), [`relix connectors`](commands/connectors.md) | the JDBC drivers and connector plugins, and installing them | +| [`relix doc`](commands/doc.md) | a page of the language reference | +| [`relix completion`](commands/completion.md) | a completion script for a shell | +| [`relix help`](commands/help.md), [`relix version`](commands/version.md) | usage, and the versions | + +Options follow the usual conventions: short options cluster (`-qN`), a long option takes +its value as `--output=csv` or `--output csv`, `--` ends the options, and `-` is standard +input. The [global options](commands/relix.md#global-options) — the profile, the limits, +`-C` and the rest — are accepted by every command, before or after its name. + +## When something goes wrong + +A problem in a script is reported on standard error as `FILE:LINE:COLUMN: severity: +message`, the shape compilers use, so an editor or a CI annotator can jump to it: + +```shell +relix check scripts/broken.relix 2>&1; echo "exit $?" +``` +``` +scripts/broken.relix:1:30: error: Undefined relation: 'Ordrs' +exit 3 +``` + +The exit status says what kind of failure it was, so a script can tell a typo from a +database that is down: + +| Status | Meaning | +|---|---| +| 0 | success | +| 1 | an assertion asked for failed: `--fail-empty` or `--fail-rows` | +| 2 | a usage error: a bad option, nothing to run | +| 3 | the script did not parse or analyse | +| 4 | the query failed while it ran: a data error, a limit hit | +| 5 | the environment: a missing driver, an untrusted catalog, a connection refused | +| 70 | an internal error in `relix`; the stack trace goes to standard error | +| 130 | interrupted with Ctrl-C (`SIGINT`) | +| 141 | standard output was closed by the reader (`SIGPIPE`) | + +`-q` keeps only the errors, `-v` adds notices such as the catalog files and drivers +loaded, and `-vv` adds timings. diff --git a/docs/guide/command-line/index.json b/docs/guide/command-line/index.json new file mode 100644 index 0000000..7830f5e --- /dev/null +++ b/docs/guide/command-line/index.json @@ -0,0 +1,28 @@ +{ + "title": "The relix command line", + "pages": [ + {"path": "getting-started.md", "chapter": "The command line", "title": "The relix command", "summary": "Installing; a first query; where the script comes from; the commands; diagnostics and exit status"}, + {"path": "inputs.md", "chapter": "The command line", "title": "Binding inputs", "summary": "`-i` and its formats; standard input; inferred and given headings; inputs that never end; shadowing the catalog; query parameters with `-a`"}, + {"path": "output.md", "chapter": "The command line", "title": "Output formats", "summary": "`table`, `tsv`, `csv`, `ndjson`, `json` and `markdown`; NULLs and headers; scripts with several queries; `--fail-empty` and `--fail-rows`; messages; closed pipes and interrupts"}, + {"path": "catalog.md", "chapter": "Projects on the command line", "title": "The catalog", "summary": "`.relix/` directories and how they are found; trusting a project; a nearer declaration wins; inspecting the catalog; `--catalog`; `relixrc`"}, + {"path": "secrets.md", "chapter": "Projects on the command line", "title": "Secrets and profiles", "summary": "`${NAME}` placeholders; `profiles.json` and `-P`; the environment and `-D`; private profile files; what is never printed"}, + {"path": "pipelines.md", "chapter": "Command-line recipes", "title": "Pipelines", "summary": "With `jq`, `awk`, `sort` and `head`; with `find` and `xargs`; scripts in here-documents; a data check in CI; optimising a script and reviewing the change"}, + {"path": "commands/relix.md", "chapter": "Command reference", "title": "relix", "summary": "The global options every command takes; the exit statuses; the environment variables and files", "command": "relix"}, + {"path": "commands/run.md", "chapter": "Command reference", "title": "relix run", "summary": "Runs scripts and writes their queries' rows", "command": "relix run"}, + {"path": "commands/check.md", "chapter": "Command reference", "title": "relix check", "summary": "Analyses scripts without running them, and reports their problems", "command": "relix check"}, + {"path": "commands/explain.md", "chapter": "Command reference", "title": "relix explain", "summary": "Prints each query's physical plan", "command": "relix explain"}, + {"path": "commands/optimize.md", "chapter": "Command reference", "title": "relix optimize", "summary": "Prints the script with each query rewritten by the optimiser", "command": "relix optimize"}, + {"path": "commands/trace.md", "chapter": "Command reference", "title": "relix trace", "summary": "Runs each query and prints what the engine rewrote, planned and ran", "command": "relix trace"}, + {"path": "commands/bundle.md", "chapter": "Command reference", "title": "relix bundle", "summary": "Prints each script's JSON bundle of diagnostics, plans and events", "command": "relix bundle"}, + {"path": "commands/ir.md", "chapter": "Command reference", "title": "relix ir", "summary": "Prints the symbols, views and queries the analyser made of a script", "command": "relix ir"}, + {"path": "commands/provenance.md", "chapter": "Command reference", "title": "relix provenance", "summary": "Runs each query and annotates each row with where it came from", "command": "relix provenance"}, + {"path": "commands/fmt.md", "chapter": "Command reference", "title": "relix fmt", "summary": "Prints scripts canonically, keeping their comments", "command": "relix fmt"}, + {"path": "commands/catalog.md", "chapter": "Command reference", "title": "relix catalog", "summary": "Lists the catalog's files, relations, columns and declarations; trusts a project", "command": "relix catalog", "subcommands": [{"command": "relix catalog files", "anchor": "relix-catalog-files"}, {"command": "relix catalog ls", "anchor": "relix-catalog-ls"}, {"command": "relix catalog schema", "anchor": "relix-catalog-schema"}, {"command": "relix catalog where", "anchor": "relix-catalog-where"}, {"command": "relix catalog trust", "anchor": "relix-catalog-trust"}]}, + {"path": "commands/drivers.md", "chapter": "Command reference", "title": "relix drivers", "summary": "Lists and installs JDBC drivers", "command": "relix drivers", "subcommands": [{"command": "relix drivers ls", "anchor": "relix-drivers-ls"}, {"command": "relix drivers install", "anchor": "relix-drivers-install"}]}, + {"path": "commands/connectors.md", "chapter": "Command reference", "title": "relix connectors", "summary": "Lists and installs connector plugins", "command": "relix connectors", "subcommands": [{"command": "relix connectors ls", "anchor": "relix-connectors-ls"}, {"command": "relix connectors install", "anchor": "relix-connectors-install"}]}, + {"path": "commands/doc.md", "chapter": "Command reference", "title": "relix doc", "summary": "Shows a page of the language reference", "command": "relix doc"}, + {"path": "commands/help.md", "chapter": "Command reference", "title": "relix help", "summary": "Prints a command's usage", "command": "relix help"}, + {"path": "commands/version.md", "chapter": "Command reference", "title": "relix version", "summary": "Prints the versions of the command and of the engine", "command": "relix version"}, + {"path": "commands/completion.md", "chapter": "Command reference", "title": "relix completion", "summary": "Prints a completion script for bash, zsh, fish or PowerShell", "command": "relix completion"} + ] +} diff --git a/docs/guide/command-line/inputs.md b/docs/guide/command-line/inputs.md new file mode 100644 index 0000000..6f0dc12 --- /dev/null +++ b/docs/guide/command-line/inputs.md @@ -0,0 +1,291 @@ +# Binding inputs + +A script names the relations it reads. Most of the time those are declared somewhere — in +the script, or in the project's [catalog](catalog.md) — but a pipeline often has data that +exists only for this run: a file another program wrote, the output of `jq`, a log being +followed. `-i` binds a relation name to such data for one run, without a declaration. +`-a` does the same for a single value, a query parameter. + +## `-i NAME=[FORMAT:]LOCATION` + +`-i` binds the relation `NAME` to the data at `LOCATION`, which is a file, `-` for +standard input, or an `http(s)` URL. It is repeatable, one relation per `-i`: + +```shell +cat data/returns.ndjson +``` +``` +{"order_id": 2, "reason": "damaged", "refund": {"amount": 80, "currency": "GBP"}} +{"order_id": 4, "reason": "late", "refund": {"amount": 20, "currency": "GBP"}} +``` + +```shell +relix -i Returns=data/returns.ndjson -e 'PROJECT order_id, reason (Returns)' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id reason + ──────── ─────── + 2 damaged + 4 late +(2 rows) +``` + +A bound relation is used like any other, and joins with the catalog's tables: + +```shell +relix -i Returns=data/returns.ndjson -e 'PROJECT order_id, amount, reason (Orders JOIN Returns)' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id amount reason + ──────── ────── ─────── + 2 80 damaged + 4 200 late +(2 rows) +``` + +A relative path resolves against the working directory, or against `-C DIR`. The same name +may not be bound twice. + +### Formats + +`FORMAT` is one of: + +| Format | What it reads | +|---|---| +| `csv` | comma-separated values, RFC 4180, with a header row | +| `tsv` | tab-separated values, with a header row | +| `json` | one JSON array of objects | +| `ndjson` | one JSON object per line, the format `jq -c` writes | + +Without a format, the file's extension decides: `.csv`, `.tsv` or `.tab`, `.json`, and +`.ndjson` or `.jsonl`. A file with any other extension needs its format named: + +```shell +cp data/orders.csv orders.txt +relix -i O=orders.txt -e 'O' 2>&1; echo "exit $?" +``` +``` +relix: -i O=orders.txt: cannot tell the format from the extension .txt; name it, as NAME=csv:LOCATION (csv, tsv, json or ndjson) +exit 2 +``` + +```shell +relix -i O=csv:orders.txt -e 'GROUP status, COUNT(*) -> n (O)' +``` +``` +── ────────────────────────────────────────────────────────────── + status n + ───────── ─ + completed 3 + pending 1 +(2 rows) +``` + +A nested JSON object becomes a nested value, which the language's dotted paths and +`UNNEST` take apart; [Nested data](../nested.md) covers them. + +### Standard input + +`-` binds standard input. Standard input has no extension, so it always names its format: + +```shell +jq -c 'select(.refund.amount > 50)' data/returns.ndjson \ + | relix -i Returns=ndjson:- -e 'PROJECT name, order_id, reason (Customers JOIN Orders JOIN Returns)' +``` +``` +── ────────────────────────────────────────────────────────────── + name order_id reason + ───── ──────── ─────── + Alice 2 damaged +(1 row) +``` + +Only one input can be standard input, and when one is, the script cannot be read from it +too: it comes from `-e` or a file. + +```shell +relix -i A=csv:- -i B=csv:- -e 'A' 2>&1 < data/orders.csv; echo "exit $?" +``` +``` +relix: standard input (-) can be bound to one input only +exit 2 +``` + +### The heading of an input + +The columns of a bound input and their types are inferred. For `csv` and `tsv` the names +are the header row's; for `json` and `ndjson` they are the keys of the objects. The types +are inferred from the values in the first 1000 records, or in as many as `--infer-rows=N` +says, so `order_id` above is a number and `placed` a date. + +`--schema NAME='{ … }'` gives the heading instead, in the syntax a `source` declaration's +`schema` uses. Here the `amount` column is read as a string rather than a number: + +```shell +relix -i O=data/orders.csv --schema O='{ order_id: NUMBER, amount: STRING }' -e 'PROJECT order_id, amount (O)' -o json +``` +``` +[ + {"order_id":1,"amount":"120"}, + {"order_id":2,"amount":"80"}, + {"order_id":3,"amount":"50"}, + {"order_id":4,"amount":"200"} +] +``` + +`--schema` and `--unbounded` name an input bound with `-i`; naming any other is a usage +error. + +### Inputs that never end + +An input such as `tail -f`'s never ends. `--unbounded=NAME` declares that, and two things +follow. The rows stream: each is filtered, joined against bounded tables and written as +it arrives, rather than after the input ends. And a query that would have to read all of +the input before writing anything — a `GROUP`, a `SORT`, a `DISTINCT` — is refused before +it starts, rather than waiting forever: + +```shell +printf '{"level":"error","msg":"disk"}\n{"level":"info","msg":"ok"}\n' | relix -i Log=ndjson:- --unbounded=Log -e "SELECT level = 'error' (Log)" -o ndjson +``` +``` +{"level":"error","msg":"disk"} +``` + +```shell +printf '{"level":"error"}\n' | relix -i Log=ndjson:- --unbounded=Log -e 'GROUP level, COUNT(*) -> n (Log)' -o ndjson 2>&1; echo "exit $?" +``` +``` +relix: -e: cannot materialise unbounded relation for blocking operator γ (GROUP); add a bound (e.g. λ n) below it +exit 3 +``` + +A `table` needs every row before it can align the columns, so an unbounded input is +written in a format that streams — `tsv`, `csv`, `ndjson` or `json` — and on a terminal +that means naming one with `-o`. A `json` input is one array, read whole, so it cannot be +unbounded. Write one object per +line and bind it as `ndjson`. With `--line-buffered` each row is written out as soon as it +is produced, for a reader that wants it at once; see [Output formats](output.md). + +### Files on the web + +A location that is an `http://` or `https://` URL is fetched, and only with `--remote`, +which a run must give for anything to be fetched. Without it, binding a URL is a usage +error. The format comes from the URL's path, or is named. + +### Shadowing the catalog + +A bound input replaces a relation of the same name that the catalog declares, for this +run. Every view that reads the name reads the input instead, so a view written against the +real `Orders` can be run over a sample file without changing it: + +```shell +relix -e 'BigSpenders' +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 200 + Carol 200 +(2 rows) +``` + +```shell +relix -i Orders=reports/sample-orders.csv -e 'BigSpenders' +``` +``` +── ────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 500 + Dan 250 +(2 rows) +``` + +## `-a NAME=VALUE`: query parameters + +A script can leave a value to be supplied when it runs, as a parameter: `$city` in + +```shell +cat scripts/by-city.relix +``` +``` +-- The customers of one city, named when the script runs: relix -a city=London +query { PROJECT name (SELECT city = $city (Customers)) }; +``` + +`-a city=London` binds it (the long form is `--arg`): + +```shell +relix -a city=London scripts/by-city.relix +``` +``` +── ────────────────────────────────────────────────────────────── + name + ───── + Alice + Carol +(2 rows) +``` + +```shell +relix --arg city=Leeds scripts/by-city.relix -o tsv +``` +``` +name +Bob +``` + +The value is bound to the parameter, never pasted into the script's text, so whatever it +holds — quotes, a semicolon, a fragment of Relix — it is only ever a value, and cannot +change what the query does: + +```shell +relix -a "city=London' OR 1 = 1 --" scripts/by-city.relix -o tsv +``` +``` +name +``` + +A parameter takes the type of what it is compared with, and the text `-a` gives is read +as that type: a number, a date, `true` or `false`. Text that is not of the type is an +error naming the parameter: + +```shell +relix -a since=2026-09-15 -e 'PROJECT order_id, placed (SELECT placed >= $since (Orders))' +``` +``` +── ────────────────────────────────────────────────────────────── + order_id placed + ──────── ────────── + 3 2026-09-20 + 4 2026-10-01 +(2 rows) +``` + +```shell +relix -a since=yesterday -e 'SELECT placed >= $since (Orders)' 2>&1; echo "exit $?" +``` +``` +relix: -e: parameter $since is compared with a DATE, so it cannot be bound to the text 'yesterday' +exit 3 +``` + +A query cannot run until every parameter it reaches has a value: + +```shell +relix scripts/by-city.relix 2>&1; echo "exit $?" +``` +``` +relix: scripts/by-city.relix: parameter $city is not bound; give it a value with -a city=VALUE +exit 3 +``` + +The [parameter reference page](../../reference/language/parameter.md) has the rules in +full. + +`-a` binds a value a query compares with. A value a *declaration* needs — a database's +address, a password, a token — is a placeholder, `${NAME}`, given by a profile, the +environment or `-D`, and the subject of [Secrets and profiles](secrets.md). diff --git a/docs/guide/command-line/output.md b/docs/guide/command-line/output.md new file mode 100644 index 0000000..580ac4e --- /dev/null +++ b/docs/guide/command-line/output.md @@ -0,0 +1,269 @@ +# Output formats + +Rows go to standard output and nothing else does. Errors, warnings, notices and +progress go to standard error, so the rows can be piped, redirected or captured without +anything to strip out of them. This page covers the formats rows are written in, which +queries of a script are written, how a run can assert something about its rows, and how +a run ends when its reader stops reading. + +## Choosing a format + +`-o FORMAT` (`--output=FORMAT`) chooses one of: + +| Format | Shape | Streams | +|---|---|---| +| `table` | aligned columns under a rule, with a row count | no | +| `tsv` | tab-separated values under a header row | yes | +| `csv` | comma-separated values under a header row, RFC 4180 | yes | +| `ndjson` | one JSON object per row, one per line | yes | +| `json` | one JSON array of objects | yes | +| `markdown` | a Markdown table | no | + +Without `-o`, the format is `$RELIX_OUTPUT`, else the `output` a project's +[`relixrc`](catalog.md#defaults-relixrc) sets, else `table` when standard output is a +terminal and `tsv` when it is not. So the same command prints a table to a person and +tab-separated values to a pipe. + +```shell +relix -e 'Customers' -o csv +``` +``` +customer_id,name,city +1,Alice,London +2,Bob,Leeds +3,Carol,London +4,Dan,York +``` + +```shell +relix -e 'Customers' -o ndjson +``` +``` +{"customer_id":1,"name":"Alice","city":"London"} +{"customer_id":2,"name":"Bob","city":"Leeds"} +{"customer_id":3,"name":"Carol","city":"London"} +{"customer_id":4,"name":"Dan","city":"York"} +``` + +```shell +relix -e 'PROJECT name, city (Customers)' -o json +``` +``` +[ + {"name":"Alice","city":"London"}, + {"name":"Bob","city":"Leeds"}, + {"name":"Carol","city":"London"}, + {"name":"Dan","city":"York"} +] +``` + +```shell +relix -e 'PROJECT name, city (Customers)' -o markdown +``` +``` +### + +| name | city | +| --- | --- | +| Alice | London | +| Bob | Leeds | +| Carol | London | +| Dan | York | +``` + +```shell +RELIX_OUTPUT=tsv relix -e 'PROJECT name (Customers)' +``` +``` +name +Alice +Bob +Carol +Dan +``` + +`tsv` writes a tab, a newline, a carriage return or a backslash inside a value as `\t`, +`\n`, `\r` or `\\`, so that every line is one row and every tab separates two columns; +this is the form `cut`, `awk` and `sort -t` read. `csv` quotes a value holding a comma, a +quote or a line break instead. `json` and `ndjson` write numbers as JSON numbers, booleans +as `true` and `false`, NULL as `null`, and dates, times and durations as ISO-8601 +strings. + +A format that streams writes each row as it is produced, so a result larger than memory +goes straight through and the first rows arrive before the last are computed. `table` and +`markdown` align their columns, which needs every row first. In a table a value longer +than 30 characters is cut short, ending `…`; the machine formats write every value whole. + +### Headers, NULLs and colour + +| Option | Effect | +|---|---| +| `--no-header` | leave out the header row of `csv` and `tsv` | +| `--null=STRING` | write NULL as `STRING`; by default it is empty in `csv`, `tsv` and `markdown`, and `NULL` in a table | +| `--color=WHEN` | colour a table `auto`, `always` or `never`; `auto` colours on a terminal unless `$NO_COLOR` is set | + +```shell +relix -e 'PROJECT name, order_id (Customers LJOIN Customers.customer_id = Orders.customer_id Orders)' -o csv --null=- +``` +``` +name,order_id +Alice,1 +Alice,2 +Bob,3 +Carol,4 +Dan,- +``` + +```shell +relix -e 'PROJECT name (Customers)' -o tsv --no-header +``` +``` +Alice +Bob +Carol +Dan +``` + +## Scripts with several queries + +A script may hold several `query` statements. A table prints each of them, under its +name: the view's name for `query Name;`, and ``, its place among the +script's queries, for `query { … };`. + +```shell +relix -e 'query Completed;' -e 'query BigSpenders;' +``` +``` +── Completed ─────────────────────────────────────────────────────────────────── + order_id customer_id amount status placed + ──────── ─────────── ────── ───────── ────────── + 1 1 120 completed 2026-09-02 + 2 1 80 completed 2026-09-14 + 4 3 200 completed 2026-10-01 +(3 rows) +── BigSpenders ───────────────────────────────────────────────────────────────── + name total + ───── ───── + Alice 200 + Carol 200 +(2 rows) +``` + +A machine format holds one result, since a CSV file with two headers is not a CSV file. It +prints the script's **last** query, as a shell function returns its last command's status, +or the one `--query=NAME` names: + +```shell +relix -e 'query Completed;' -e 'query BigSpenders;' -o tsv +``` +``` +name total +Alice 200 +Carol 200 +``` + +```shell +relix -e 'query Completed;' -e 'query BigSpenders;' -o tsv --query=Completed +``` +``` +order_id customer_id amount status placed +1 1 120 completed 2026-09-02 +2 1 80 completed 2026-09-14 +4 3 200 completed 2026-10-01 +``` + +`--all` prints every query in `ndjson`, each row tagged with the query it belongs to in a +`"_query"` field, so one stream can carry several results and `jq` can take them apart: + +```shell +relix -e 'query Spend;' -e 'query BigSpenders;' -o ndjson --all +``` +``` +{"_query":"Spend","customer_id":1,"total":200} +{"_query":"Spend","customer_id":3,"total":200} +{"_query":"BigSpenders","name":"Alice","total":200} +{"_query":"BigSpenders","name":"Carol","total":200} +``` + +With any other machine format `--all` is a usage error. + +Several script files on one command line are independent runs, and their output is +written one after another in the order the files were named. + +## Asserting something about the rows + +`--fail-empty` makes a run exit 1 when the queries it printed returned no rows, and +`--fail-rows` when they returned any. The rows are printed either way. This is the +`grep -q` idiom, for a data check in CI or in a shell `if`: + +```shell +relix --fail-rows -e 'SELECT amount < 0 (Orders)' -o tsv; echo "exit $?" +``` +``` +order_id customer_id amount status placed +exit 0 +``` + +```shell +relix --fail-rows -e "SELECT status = 'pending' (Orders)" -o tsv > /dev/null || echo "orders are still pending" +``` +``` +orders are still pending +``` + +[Pipelines](pipelines.md#a-data-check-in-ci) shows one in a CI job. + +## Where messages go + +A problem in a script is reported on standard error as `FILE:LINE:COLUMN: severity: +message`. `FILE` is the script's path, `-e` (or `-e#2` for the second of several `-e` +texts), or ``; `LINE` and `COLUMN` count within that text, so a message about an +`-e` points into what was typed. A problem with the run itself — an option, a file that +is not there, a database that refuses the connection — is one line beginning `relix:`. + +```shell +relix -e 'SELECT amount > 100 (Ordrs)' 2>&1; echo "exit $?" +``` +``` +-e:1:22: error: Undefined relation: 'Ordrs' +exit 3 +``` + +Warnings are printed too, unless `-q` is given. `-v` adds notices — each catalog file +read, each JDBC driver loaded — and `-vv` adds how long each stage took. `relix check +--json` writes the diagnostics to standard output as one JSON document instead, for a +tool to read; see [`relix check`](commands/check.md). + +The [exit status](getting-started.md#when-something-goes-wrong) says which kind of +failure it was. + +## When the reader stops reading + +A program at the other end of a pipe can stop reading at any time: `head` does after its +last line. When it does, `relix` stops: the query in flight is cancelled — a plan stops +pulling rows, a database cursor is closed — and the command exits 141, the status a shell +gives a program ended by `SIGPIPE`, without printing anything. So a query over a large or +endless relation can be cut short with `head`: + +```shell +relix -e 'Orders' -o tsv | head -3 +``` +``` +order_id customer_id amount status placed +1 1 120 completed 2026-09-02 +2 1 80 completed 2026-09-14 +``` + +Ctrl-C (`SIGINT`) likewise cancels the query and closes the session, and the command exits +130. + +Standard output is written in blocks and flushed at the end of each result. With +`--line-buffered` each row is flushed as it is written, for a reader that wants each row +at once — an input following a log, read by something that reacts to each line. + +## Following what the engine did + +`--trace` writes the run's event feed — what the optimiser rewrote, the plan it chose, +how many rows each step produced — to standard error while the rows go to standard +output; `--trace=FILE` writes it to `FILE` instead. [`relix trace`](commands/trace.md) +prints the same feed on standard output, without the rows. diff --git a/docs/guide/command-line/pipelines.md b/docs/guide/command-line/pipelines.md new file mode 100644 index 0000000..8f4fe90 --- /dev/null +++ b/docs/guide/command-line/pipelines.md @@ -0,0 +1,192 @@ +# Pipelines + +`relix` reads text and writes text, so it works with the programs a shell already has. +This page is a set of recipes: `relix` with `jq` for JSON, with `find` and `xargs` for +many files, with `awk`, `sort` and `head` for its rows, and as a check in CI. + +The rule that makes all of them work: **standard output carries rows and nothing else.** +Messages go to standard error, the format in a pipe is tab-separated values unless `-o` +says otherwise, and the [exit status](getting-started.md#when-something-goes-wrong) says +what happened. + +## With jq + +`jq` is the tool for shaping JSON, and `relix` the one for joining and aggregating it. A +pipeline passes JSON between them as NDJSON, one object per line, which `jq -c` writes, +`-i NAME=ndjson:-` reads and `-o ndjson` writes. + +Here `jq` flattens the returns file, `relix` joins it to the customers and orders, and +`jq` formats the result: + +```shell +jq -c '{order_id, reason, refund: .refund.amount}' data/returns.ndjson \ + | relix -i Returns=ndjson:- -e 'PROJECT name, order_id, refund (Customers JOIN Orders JOIN Returns)' -o ndjson \ + | jq -r '"\(.name) was refunded \(.refund) on order \(.order_id)"' +``` +``` +Alice was refunded 80 on order 2 +Carol was refunded 20 on order 4 +``` + +`relix` can also hand `jq` a single JSON document, an array, with `-o json`: + +```shell +relix -e 'GROUP city, COUNT(*) -> customers (Customers)' -o json | jq -c 'map({(.city): .customers}) | add' +``` +``` +{"London":2,"Leeds":1,"York":1} +``` + +## With awk, sort and cut + +Tab-separated values are what the classic text tools split on. The header is the first +line; `--no-header` leaves it out. + +```shell +relix -e 'Orders' --no-header | awk -F'\t' '{ total[$4] += $3 } END { for (s in total) print s, total[s] }' | sort +``` +``` +completed 400 +pending 50 +``` + +## Stopping early + +When the reader of a pipe stops, `relix` stops too: it cancels the query rather than +computing rows nobody will read. `head` can cut a large result short: + +```shell +relix -e 'SORT amount DESC (Orders)' | head -3 +``` +``` +order_id customer_id amount status placed +4 3 200 completed 2026-10-01 +1 1 120 completed 2026-09-02 +``` + +## With find and xargs + +### Lint every script + +`relix check` analyses scripts without running them, reporting each problem as +`FILE:LINE:COLUMN: severity: message` and exiting 3 if any script has one. Over every +script in a tree: + +```shell +find scripts -name '*.relix' | sort | xargs relix check 2>&1 || echo "a script has a problem" +``` +``` +scripts/broken.relix:1:30: error: Undefined relation: 'Ordrs' +a script has a problem +``` + +`xargs` fails when any run of the command it starts fails, so the pipeline's status says +whether every script passed. Use `find … -print0 | xargs -0` when a file name may hold a +space. + +### Run a query over each of many files + +`-i` binds one file per run, so `xargs -I{}` runs a query over each file in turn. Here +each CSV file's row count, with its name: + +```shell +find data reports -name '*.csv' | sort | xargs -I{} sh -c 'printf "%s\t" {}; relix -i T={} -e "GROUP COUNT(*) -> n (T)" -o tsv --no-header' +``` +``` +data/customers.csv 4 +data/orders.csv 4 +reports/sample-orders.csv 2 +``` + +### Check every script's formatting + +`relix fmt --check` prints the name of each file that is not formatted canonically, and +exits 1 if there is one; `relix fmt -w` rewrites them in place. The shop's scripts spell +operators as ASCII keywords, so they are checked against `--keywords`: + +```shell +find scripts -name '*.relix' | sort | xargs relix fmt --check --keywords || echo "a script needs formatting" +``` +``` +scripts/messy.relix +scripts/top-customers.relix +a script needs formatting +``` + +## Scripts written in the shell + +A script can be written inline with a here-document, which keeps a longer query readable +and still needs no file: + +```shell +relix -o csv <<'EOF' +Late := { SELECT placed >= DATE '2026-09-15' (Orders) }; +query { PROJECT name, order_id (Customers JOIN Late) }; +EOF +``` +``` +name,order_id +Bob,3 +Carol,4 +``` + +A value the shell knows goes in as a [parameter](inputs.md#a-name-value-query-parameters), +never pasted into the text: + +```shell +city=London +relix -a city="$city" scripts/by-city.relix -o tsv +``` +``` +name +Alice +Carol +``` + +## A data check in CI + +`--fail-rows` exits 1 when the query returns any row, and `--fail-empty` when it returns +none, so a data assertion is one line of a CI job: + +```shell +relix --fail-rows -e 'SELECT amount <= 0 (Orders)' -o tsv > /dev/null && echo "every order has a positive amount" +``` +``` +every order has a positive amount +``` + +```shell +relix --fail-rows -e "SELECT status = 'pending' (Orders)" -o tsv || echo "pending orders found" +``` +``` +order_id customer_id amount status placed +3 2 50 pending 2026-09-20 +pending orders found +``` + +A CI job runs in a fresh container where the project's `.relix/` cannot have been +trusted by hand; `RELIX_TRUST_ALL=1` trusts it, and a profile or the job's secret +environment variables supply the [placeholders](secrets.md). + +## Optimise a script, and review the change + +`relix optimize` prints a script with each query rewritten by the optimiser, so it works +as a filter, and `diff` shows what it changed: + +```shell +cat > slow.relix <<'EOF' +query { SELECT city = 'London' (Customers JOIN Orders) }; +EOF +relix optimize slow.relix | diff slow.relix -; echo "exit $?" +``` +``` +1c1 +< query { SELECT city = 'London' (Customers JOIN Orders) }; +--- +> query { (σ city = "London" (Customers)) ⋈ (Orders) }; +exit 1 +``` + +`diff` exits 1 when the files differ. To keep the rewritten script, redirect it: +`relix optimize slow.relix > fast.relix`. [`relix optimize --report`](commands/optimize.md) +lists the rules that fired instead. diff --git a/docs/guide/command-line/secrets.md b/docs/guide/command-line/secrets.md new file mode 100644 index 0000000..2e2d57c --- /dev/null +++ b/docs/guide/command-line/secrets.md @@ -0,0 +1,184 @@ +# Secrets and profiles + +A connection to a database needs an address and a password; a source that calls an API +needs a token. None of them belongs in a script or a catalog file that is committed and +shared. They are written as **placeholders**, `${NAME}`, and the command supplies the +values when a query runs: from `-D`, from a named **profile**, or from the environment. + +## Placeholders + +The shop's stock lives in a database, declared in the catalog with placeholders where +its address and credentials go: + +```shell +cat .relix/catalog/20-warehouse.relix +``` +``` +-- The stock database. Its address and credentials come from placeholders. +connection warehouse from database { + url: "${WAREHOUSE_URL}", + user: "${WAREHOUSE_USER}", + password: "${WAREHOUSE_PASSWORD}" +}; + +source Stock from warehouse { + table: "STOCK", + schema: { sku: STRING, on_hand: NUMBER } +}; +``` + +A placeholder may stand in any string value of a declaration: a URL, a file path, a +table name, a header, a credential. The engine's +[Secrets in declarations](../secrets.md) page has the rules. + +A placeholder is resolved each time a query runs, for the declarations that query +reaches, and only then. A query that never reaches the warehouse needs none of its +values, and one that does fails before anything is sent when a value is missing, naming +the placeholder; [below](#the-environment-and-d) is an example. + +Each `${NAME}` is looked up in three places, and the first that has it wins: + +1. **`-D NAME=VALUE`** on the command line. +2. **The selected profile**, from the `profiles.json` files. +3. **The environment** of the process. + +## Profiles + +A profile is a named set of values, kept in a `profiles.json` file in a `.relix/` +directory: one JSON object whose members are profiles, each an object of string values. +The shop's project keeps the parts of its `dev` profile that are not secret: + +```shell +cat .relix/profiles.json +``` +``` +{ + "dev": { + "WAREHOUSE_URL": "jdbc:h2:~/shop/data/warehouse", + "WAREHOUSE_USER": "shop" + } +} +``` + +and the password is in your own `~/.relix/profiles.json`, outside the project and its +repository: + +```shell +cat ~/.relix/profiles.json +``` +``` +{ + "dev": { "WAREHOUSE_PASSWORD": "s3cret" } +} +``` + +`-P NAME` (`--profile=NAME`) selects a profile. Its values are merged across every +`profiles.json` the run reads, outermost first, so a nearer file's value for a name +replaces a farther one's, and a profile may be split between your own file and the +project's, as `dev` is here: + +```shell +relix -P dev -e 'Stock' +``` +``` +── ────────────────────────────────────────────────────────────── + sku on_hand + ──── ─────── + mug 12 + tee 0 + tote 31 +(3 rows) +``` + +Without `-P`, the profile is `$RELIX_PROFILE`, else the `profile` a +[`relixrc`](catalog.md#defaults-relixrc) names. With none of them, no profile is +selected and placeholders come from `-D` and the environment alone. + +```shell +RELIX_PROFILE=dev relix -e 'PROJECT sku (SELECT on_hand = 0 (Stock))' -o tsv +``` +``` +sku +tee +``` + +Naming a profile no file defines is a usage error, which lists the ones that are defined: + +```shell +relix -P production -e 'Stock' 2>&1; echo "exit $?" +``` +``` +relix: no profile 'production' (defined: dev) +exit 2 +``` + +A project's `profiles.json` is part of its `.relix/` directory, so it is read only once +the project is [trusted](catalog.md#a-project-must-be-trusted); `~/.relix/profiles.json` +is always read. + +### A profile file must be private + +A profile may hold a password, so a `profiles.json` that its group or others can read is +refused, as `ssh` refuses a private key anyone can read. The error names the `chmod` that +fixes it: + +```shell +chmod 644 ~/.relix/profiles.json +relix -P dev -e 'Stock' 2>&1; echo "exit $?" +chmod 600 ~/.relix/profiles.json +``` +``` +relix: ~/.relix/profiles.json is readable by group or others, and a profile may hold credentials; make it private with: chmod 600 ~/.relix/profiles.json +exit 5 +``` + +Where the file system has no POSIX permissions, as on Windows, the check is not made. + +## The environment, and `-D` + +A value not in the selected profile comes from the environment, which is what a CI job's +secrets and a twelve-factor deployment use. Here no profile is selected: `-D` gives the +address and the user, and the environment the password: + +```shell +WAREHOUSE_PASSWORD=s3cret relix -D WAREHOUSE_URL=jdbc:h2:$HOME/shop/data/warehouse -D WAREHOUSE_USER=shop -e 'Stock' -o tsv +``` +``` +sku on_hand +mug 12 +tee 0 +tote 31 +``` + +Without the password, the query fails before the connection is opened, and the error +names what is missing. The status is 3, as for any script that cannot be analysed: + +```shell +relix -D WAREHOUSE_URL=jdbc:h2:$HOME/shop/data/warehouse -D WAREHOUSE_USER=shop -e 'Stock' 2>&1; echo "exit $?" +``` +``` +relix: -e: placeholder ${WAREHOUSE_PASSWORD} in connection 'warehouse' has no value: the session's placeholder resolver returned none +exit 3 +``` + +`-D NAME=VALUE` sets one value for one run, and beats the profile and the environment. +It is for values that are not secret: a command line is kept in your shell's history and +shown by `ps` to everyone on the machine. A secret belongs in a private `profiles.json`, +or in the environment. + +## What is never printed + +The placeholder stays in the declaration, and the value goes only to the connector that +needs it, for the run that needs it. Commands that print a script or a plan — +`relix fmt`, `relix ir`, `relix bundle`, `relix explain`, `relix catalog` — print +`${WAREHOUSE_PASSWORD}`, never the password, and so does an error message: + +```shell +relix fmt .relix/catalog/20-warehouse.relix +``` +``` +-- The stock database. Its address and credentials come from placeholders. +connection warehouse from jdbc { password: "${WAREHOUSE_PASSWORD}", url: "${WAREHOUSE_URL}", user: "${WAREHOUSE_USER}" }; + +source Stock from warehouse { table: "STOCK", schema: { sku: string, on_hand: number } }; +``` diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 1b33c55..5097068 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index aaaabb3..299848b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.8.0-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 23d15a9..249efbb 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ #!/bin/sh # -# Copyright © 2015-2021 the original authors. +# Copyright © 2015 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ ############################################################################## # -# Gradle start up script for POSIX generated by Gradle. +# gradlew start up script for POSIX generated by Gradle. # # Important for running: # @@ -29,7 +29,7 @@ # bash, then to run this script, type that shell name before the whole # command line, like: # -# ksh Gradle +# ksh gradlew # # Busybox and similar reduced shells will NOT work, because this script # requires all of these POSIX shell features: @@ -57,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -114,7 +114,6 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac -CLASSPATH="\\\"\\\"" # Determine the Java command to use to start the JVM. @@ -172,7 +171,6 @@ fi # For Cygwin or MSYS, switch paths to Windows format before running java if "$cygwin" || "$msys" ; then APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) JAVACMD=$( cygpath --unix "$JAVACMD" ) @@ -212,7 +210,6 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" diff --git a/gradlew.bat b/gradlew.bat index db3a6ac..3185a43 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -19,12 +19,39 @@ @if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem -@rem Gradle startup script for Windows +@rem gradlew startup script for Windows @rem @rem ########################################################################## -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal +@rem Set local scope for the variables, and ensure extensions are enabled +setlocal EnableExtensions + +@rem Catch executions from older scripts and ensure they exit cleanly. +@rem This can be removed once we can be reasonably confident that few people +@rem will be migrating directly to this new wrapper. +goto afterSafetyNet +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +goto exitWithErrorLevel +:afterSafetyNet set DIRNAME=%~dp0 if "%DIRNAME%"=="" set DIRNAME=. @@ -45,13 +72,14 @@ set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if %ERRORLEVEL% equ 0 goto execute -echo. 1>&2 -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 -echo. 1>&2 -echo Please set the JAVA_HOME variable in your environment to match the 1>&2 -echo location of your Java installation. 1>&2 +1>&2 echo. +1>&2 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +1>&2 echo. +1>&2 echo Please set the JAVA_HOME variable in your environment to match the +1>&2 echo location of your Java installation. -goto fail +"%COMSPEC%" /c exit 1 +goto exitWithErrorLevel :findJavaFromJavaHome set JAVA_HOME=%JAVA_HOME:"=% @@ -59,36 +87,26 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. 1>&2 -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 -echo. 1>&2 -echo Please set the JAVA_HOME variable in your environment to match the 1>&2 -echo location of your Java installation. 1>&2 +1>&2 echo. +1>&2 echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +1>&2 echo. +1>&2 echo Please set the JAVA_HOME variable in your environment to match the +1>&2 echo location of your Java installation. -goto fail +"%COMSPEC%" /c exit 1 +goto exitWithErrorLevel :execute @rem Setup the command line -set CLASSPATH= -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* +@rem Execute gradlew +@rem endlocal doesn't take effect until after the line is parsed and variables are expanded +@rem which allows us to clear the local environment before executing the java command +endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel & goto exitWithErrorLevel -:end -@rem End local scope for the variables with windows NT shell -if %ERRORLEVEL% equ 0 goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -set EXIT_CODE=%ERRORLEVEL% -if %EXIT_CODE% equ 0 set EXIT_CODE=1 -if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% -exit /b %EXIT_CODE% - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega +@rem This label must not be changed. We rely on old scripts being able to jump to this point. +:exitWithErrorLevel +@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts +"%COMSPEC%" /c exit %ERRORLEVEL% diff --git a/packaging/homebrew/relix.rb b/packaging/homebrew/relix.rb new file mode 100644 index 0000000..3718a8e --- /dev/null +++ b/packaging/homebrew/relix.rb @@ -0,0 +1,46 @@ +# The relix formula. The release workflow of DarkCollective/relix-cli writes it from +# packaging/homebrew/relix.rb there, with the release's version and checksums: change it +# there, since a change made in the tap is overwritten by the next release. +class Relix < Formula + desc "Run Relix relational-algebra scripts as a stage of a Unix pipeline" + homepage "https://github.com/DarkCollective/relix-cli" + version "@VERSION@" + license "Apache-2.0" + + on_macos do + on_arm do + url "https://github.com/DarkCollective/relix-cli/releases/download/v#{version}/relix-#{version}-macos-aarch64.tar.gz" + sha256 "@SHA256:macos-aarch64@" + end + on_intel do + url "https://github.com/DarkCollective/relix-cli/releases/download/v#{version}/relix-#{version}-macos-x86_64.tar.gz" + sha256 "@SHA256:macos-x86_64@" + end + end + + on_linux do + on_arm do + url "https://github.com/DarkCollective/relix-cli/releases/download/v#{version}/relix-#{version}-linux-aarch64.tar.gz" + sha256 "@SHA256:linux-aarch64@" + end + on_intel do + url "https://github.com/DarkCollective/relix-cli/releases/download/v#{version}/relix-#{version}-linux-x86_64.tar.gz" + sha256 "@SHA256:linux-x86_64@" + end + end + + def install + libexec.install Dir["*"] + bin.install_symlink libexec/"bin/relix" + man1.install Dir[libexec/"man/man1/*.1"] + bash_completion.install libexec/"completions/relix.bash" => "relix" + zsh_completion.install libexec/"completions/_relix" + fish_completion.install libexec/"completions/relix.fish" + end + + test do + (testpath/"people.csv").write "id,name\n1,ada\n2,bob\n" + output = pipe_output("#{bin}/relix -i People=csv:people.csv", "query { SELECT id > 1 (People) };\n") + assert_equal "id\tname\n2\tbob\n", output + end +end diff --git a/packaging/release.py b/packaging/release.py new file mode 100644 index 0000000..5fc5bf5 --- /dev/null +++ b/packaging/release.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Write a release's checksums, Homebrew formula and Scoop manifest, and check them. + +Usage: + release.py VERSION DIST OUT + +DIST holds the five platform archives the release workflow built. OUT receives: + + checksums.txt the archives' SHA-256 sums, as `sha256sum` prints them + relix.rb packaging/homebrew/relix.rb with the version and checksums filled in + relix.json packaging/scoop/relix.json, likewise + +Then it reads the formula and the manifest back and fails unless every archive each +one names is in DIST under that name with that checksum, which is what a dry-run +release proves before anything is published. +""" +import hashlib +import json +import pathlib +import re +import sys + +PLATFORMS = ["macos-aarch64", "macos-x86_64", "linux-aarch64", "linux-x86_64", "windows-x86_64"] +PACKAGING = pathlib.Path(__file__).resolve().parent + + +def archive_name(version: str, platform: str) -> str: + extension = "zip" if platform.startswith("windows") else "tar.gz" + return f"relix-{version}-{platform}.{extension}" + + +def sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as f: + for block in iter(lambda: f.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def fill(template: str, version: str, sums: dict) -> str: + text = template.replace("@VERSION@", version) + for platform, digest in sums.items(): + text = text.replace(f"@SHA256:{platform}@", digest) + left = re.findall(r"@[A-Z0-9:_-]+@", text) + if left: + sys.exit(f"placeholders left unfilled: {', '.join(sorted(set(left)))}") + return text + + +def formula_archives(formula: str, version: str) -> dict: + """The archive name and checksum of each url/sha256 pair in the formula.""" + pairs = re.findall(r'url "([^"]+)"\s*\n\s*sha256 "([0-9a-f]{64})"', formula) + return {url.replace("#{version}", version).rsplit("/", 1)[1]: digest for url, digest in pairs} + + +def manifest_archives(manifest: dict) -> dict: + """The archive name and checksum of each architecture in the manifest.""" + return {entry["url"].rsplit("/", 1)[1]: entry["hash"] + for entry in manifest["architecture"].values()} + + +def check(name: str, named: dict, expected: set, dist: pathlib.Path) -> list: + problems = [] + if set(named) != expected: + problems.append(f"{name} names {sorted(named)}, not {sorted(expected)}") + for archive, digest in named.items(): + path = dist / archive + if not path.is_file(): + problems.append(f"{name} names {archive}, which is not in {dist}") + elif sha256(path) != digest: + problems.append(f"{name} gives {archive} the checksum {digest}, not {sha256(path)}") + return problems + + +def main() -> None: + if len(sys.argv) != 4: + sys.exit(__doc__) + version, dist, out = sys.argv[1], pathlib.Path(sys.argv[2]), pathlib.Path(sys.argv[3]) + if not re.fullmatch(r"\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?", version): + sys.exit(f"'{version}' is not a version such as 1.2.0 or 1.2.0-rc1") + + archives = {platform: dist / archive_name(version, platform) for platform in PLATFORMS} + missing = [str(path) for path in archives.values() if not path.is_file()] + if missing: + sys.exit("missing archives:\n " + "\n ".join(missing)) + sums = {platform: sha256(path) for platform, path in archives.items()} + + out.mkdir(parents=True, exist_ok=True) + (out / "checksums.txt").write_text( + "".join(f"{sums[p]} {archives[p].name}\n" for p in PLATFORMS), encoding="utf-8") + formula = fill((PACKAGING / "homebrew" / "relix.rb").read_text(encoding="utf-8"), version, sums) + (out / "relix.rb").write_text(formula, encoding="utf-8") + manifest = fill((PACKAGING / "scoop" / "relix.json").read_text(encoding="utf-8"), version, sums) + (out / "relix.json").write_text(manifest, encoding="utf-8") + + unix = {archive_name(version, p) for p in PLATFORMS if not p.startswith("windows")} + windows = {archive_name(version, p) for p in PLATFORMS if p.startswith("windows")} + problems = (check("relix.rb", formula_archives(formula, version), unix, dist) + + check("relix.json", manifest_archives(json.loads(manifest)), windows, dist)) + if problems: + sys.exit("\n".join(problems)) + print(f"relix {version}: checksums.txt, relix.rb and relix.json in {out}; " + f"every archive they name matches its checksum") + + +if __name__ == "__main__": + main() diff --git a/packaging/scoop/relix.json b/packaging/scoop/relix.json new file mode 100644 index 0000000..dfd2fcd --- /dev/null +++ b/packaging/scoop/relix.json @@ -0,0 +1,44 @@ +{ + "##": "The relix manifest. The release workflow of DarkCollective/relix-cli writes it from packaging/scoop/relix.json there, with the release's version and checksum: change it there, since a change made in the bucket is overwritten by the next release.", + "version": "@VERSION@", + "description": "Run Relix relational-algebra scripts as a stage of a Unix pipeline", + "homepage": "https://github.com/DarkCollective/relix-cli", + "license": "Apache-2.0", + "architecture": { + "64bit": { + "url": "https://github.com/DarkCollective/relix-cli/releases/download/v@VERSION@/relix-@VERSION@-windows-x86_64.zip", + "hash": "@SHA256:windows-x86_64@", + "extract_dir": "relix-@VERSION@" + } + }, + "bin": "bin\\relix.bat", + "post_install": [ + "$completion = Join-Path (appdir $app $global) 'current\\completions\\relix.ps1'", + "$marker = '# relix completion, added by scoop'", + "$line = \"if (Test-Path '$completion') { . '$completion' } $marker\"", + "$profileFile = $PROFILE.CurrentUserAllHosts", + "if (!(Test-Path $profileFile)) { New-Item -ItemType File -Force -Path $profileFile | Out-Null }", + "if (!(Select-String -Path $profileFile -SimpleMatch $marker -Quiet)) { Add-Content -Path $profileFile -Value $line }" + ], + "uninstaller": { + "script": [ + "$profileFile = $PROFILE.CurrentUserAllHosts", + "if (Test-Path $profileFile) {", + " $kept = Get-Content $profileFile | Where-Object { $_ -notlike '*# relix completion, added by scoop' }", + " Set-Content -Path $profileFile -Value $kept", + "}" + ] + }, + "notes": "Tab completion for relix is loaded by your PowerShell profile ($PROFILE.CurrentUserAllHosts); open a new session to use it.", + "checkver": { + "github": "https://github.com/DarkCollective/relix-cli" + }, + "autoupdate": { + "architecture": { + "64bit": { + "url": "https://github.com/DarkCollective/relix-cli/releases/download/v$version/relix-$version-windows-x86_64.zip", + "extract_dir": "relix-$version" + } + } + } +} diff --git a/settings.gradle b/settings.gradle index d2a1d66..888c47c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,6 +1,6 @@ plugins { // Lets the Java 21 toolchain be provisioned on a machine that has no Java 21. - id 'org.gradle.toolchains.foojay-resolver-convention' version '0.9.0' + id 'org.gradle.toolchains.foojay-resolver-convention' version '1.0.0' } rootProject.name = 'relix-cli' diff --git a/src/main/java/com/darkcollective/relix/cli/EngineMessages.java b/src/main/java/com/darkcollective/relix/cli/EngineMessages.java new file mode 100644 index 0000000..2c68961 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/EngineMessages.java @@ -0,0 +1,54 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli; + +import com.darkcollective.relix.embed.RelixException; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * The message of a failure the engine raised, as the command prints it. + * + *

The engine writes for a program that embeds it, so the remedy it names may be a + * Java call. Where the command has its own way to give what is missing, the message names + * that instead; every other message is the engine's own. + */ +public final class EngineMessages { + + /** The engine's unbound-parameter message, whose remedy is {@code Relix.Builder.parameters}. */ + private static final Pattern UNBOUND = + Pattern.compile("parameter \\$(\\S+) is not bound; give a value with .*"); + + private EngineMessages() { + } + + /** + * The message to print for a failure the engine raised. + * + * @param failure what the engine threw + * @return its message, its remedy put in the command's terms + */ + public static String of(RelixException failure) { + String message = failure.getMessage(); + Matcher unbound = UNBOUND.matcher(message); + if (unbound.matches()) { + String name = unbound.group(1); + return "parameter $" + name + " is not bound; give it a value with -a " + name + "=VALUE"; + } + return message; + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/Main.java b/src/main/java/com/darkcollective/relix/cli/Main.java index c495a9f..e77a137 100644 --- a/src/main/java/com/darkcollective/relix/cli/Main.java +++ b/src/main/java/com/darkcollective/relix/cli/Main.java @@ -17,6 +17,7 @@ import com.darkcollective.relix.cli.command.RelixCommand; import com.darkcollective.relix.cli.io.Host; +import com.darkcollective.relix.cli.io.Interruption; import com.darkcollective.relix.embed.Relix; import com.darkcollective.relix.embed.RelixException; import picocli.CommandLine; @@ -31,16 +32,24 @@ */ public final class Main { + /** {@code run}'s option whose file is attached with {@code =} only. */ + private static final String TRACE = "--trace"; + private Main() { } /** * Runs the command and exits with its status. * + *

An interrupt ({@code SIGINT}) runs the shutdown hook, which stops the query in + * flight and closes its session; the JVM then exits 130, the shell's value for it. + * * @param args the command line */ public static void main(String[] args) { - System.exit(run(Host.system(), args)); + Interruption interruption = new Interruption(); + Runtime.getRuntime().addShutdownHook(new Thread(interruption::interrupt, "relix-interrupt")); + System.exit(run(Host.system(), interruption, args)); } /** @@ -51,9 +60,22 @@ public static void main(String[] args) { * @return the exit status */ public static int run(Host host, String... args) { + return run(host, new Interruption(), args); + } + + /** + * Runs the command over a host and returns its exit status, without exiting. + * + * @param host the process to run in + * @param interruption what an interrupt stops; {@link Interruption#interrupt()} ends + * the run with 130 + * @param args the command line + * @return the exit status + */ + public static int run(Host host, Interruption interruption, String... args) { PrintWriter out = new PrintWriter(host.out(), true, StandardCharsets.UTF_8); PrintWriter err = new PrintWriter(host.err(), true, StandardCharsets.UTF_8); - CommandLine command = new CommandLine(new RelixCommand(host)) + CommandLine command = new CommandLine(new RelixCommand(host, interruption)) .setOut(out) .setErr(err) .setPosixClusteredShortOptionsAllowed(true) @@ -69,8 +91,8 @@ public static int run(Host host, String... args) { err.print("relix: " + failure.getMessage() + "\n"); } else if (code == ExitCode.USAGE) { failed.usage(err); - } else if (e instanceof RelixException) { - err.print("relix: " + e.getMessage() + "\n"); + } else if (e instanceof RelixException failure) { + err.print("relix: " + EngineMessages.of(failure) + "\n"); } else if (code == ExitCode.INTERNAL) { err.print("relix: internal error; please report it with what follows\n"); e.printStackTrace(err); @@ -78,12 +100,34 @@ public static int run(Host host, String... args) { err.flush(); return code.status(); }); - int status = command.execute(args); + int status = command.execute(attachOptionalValues(args)); out.flush(); err.flush(); return status; } + /** + * The command line with a bare {@code --trace} given an empty value, so that the word + * after it is never taken for its file: {@code relix --trace q.relix} traces + * {@code q.relix}, and a file is named only as {@code --trace=FILE}. Nothing after + * {@code --} is touched. + * + * @param args the command line + * @return the command line to parse + */ + static String[] attachOptionalValues(String[] args) { + String[] attached = args.clone(); + for (int i = 0; i < attached.length; i++) { + if (attached[i].equals("--")) { + break; + } + if (attached[i].equals(TRACE)) { + attached[i] = TRACE + "="; + } + } + return attached; + } + /** * The exit code for a failure that escaped a command. * diff --git a/src/main/java/com/darkcollective/relix/cli/catalog/Catalog.java b/src/main/java/com/darkcollective/relix/cli/catalog/Catalog.java new file mode 100644 index 0000000..6f19519 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/catalog/Catalog.java @@ -0,0 +1,237 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.catalog; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.embed.Relix; +import com.darkcollective.relix.lang.ast.Statement; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +/** + * The declarations a run starts with, overlaid so that the nearest declaration of each + * name wins (design §5.2). + * + *

Files load outermost first: each {@code .relix/} directory's {@code catalog/*.relix}, + * in lexical order within a directory, from {@code ~/.relix} to the working directory's, + * and then the {@code --catalog} files. A later file is nearer, and its declaration of a + * name replaces a farther one's — an overlay, not a duplicate-name error. Every file is + * parsed first, the nearest declaration of each name kept, and the survivors defined into + * the session together, so a view declared at an outer level reads whichever + * {@code Orders} is nearest. + * + *

A file in a project directory the user has not trusted is parsed but never defined + * (design §5.4): its declarations neither win nor shadow anything. + * + *

Defining declares; it opens nothing. No connection is made and no file read until a + * query references the name. + */ +public final class Catalog { + + /** The directory inside a {@code .relix/} that holds its catalog files. */ + public static final String DIRECTORY = "catalog"; + + /** A catalog file's extension. */ + public static final String EXTENSION = ".relix"; + + private static final Catalog EMPTY = new Catalog(List.of()); + + private final List files; + private final Map nearest = new HashMap<>(); + + private Catalog(List files) { + this.files = List.copyOf(files); + for (CatalogFile file : this.files) { + if (file.trusted()) { + for (Declaration declaration : file.declarations()) { + nearest.put(declaration.key(), declaration); + } + } + } + } + + /** + * A catalog of nothing, as {@code -N} gives when no {@code --catalog} is named. + * + * @return the empty catalog + */ + public static Catalog empty() { + return EMPTY; + } + + /** + * Reads the catalog of the given directories and files. + * + * @param levels the {@code .relix/} directories, outermost first + * @param named the {@code --catalog} and {@code $RELIX_CATALOG_PATH} files, farthest + * first, absolute; always trusted, since the user named them + * @param trust which directories may be loaded + * @return the catalog + * @throws CommandFailure with {@link ExitCode#ANALYSIS} when a file does not parse, and + * with {@link ExitCode#ENVIRONMENT} when one cannot be read + */ + public static Catalog load(List levels, List named, Trust trust) { + List files = new ArrayList<>(); + for (Path level : levels) { + boolean trusted = trust.trusts(level); + for (Path file : filesOf(level)) { + files.add(CatalogFile.read(file, level, trusted)); + } + } + for (Path file : named) { + files.add(CatalogFile.read(file, null, true)); + } + return new Catalog(files); + } + + /** + * The catalog files of one {@code .relix/} directory, in lexical order. + * + * @param level a {@code .relix/} directory + * @return its {@code catalog/*.relix} files + */ + public static List filesOf(Path level) { + Path dir = level.resolve(DIRECTORY); + if (!Files.isDirectory(dir)) { + return List.of(); + } + try (Stream entries = Files.list(dir)) { + return entries + .filter(p -> p.getFileName().toString().endsWith(EXTENSION) && Files.isRegularFile(p)) + .sorted(Comparator.comparing(p -> p.getFileName().toString())) + .toList(); + } catch (IOException e) { + throw new CommandFailure(ExitCode.ENVIRONMENT, dir + ": cannot list: " + e.getMessage()); + } + } + + /** + * Every file loaded, in load order. + * + * @return the files, outermost first + */ + public List files() { + return files; + } + + /** + * The declarations an untrusted file would have supplied: those whose name no loaded + * declaration of the same kind answers, nearest first. + * + * @return the declarations left out for want of trust + */ + public List withheld() { + List withheld = new ArrayList<>(); + for (CatalogFile file : files) { + if (!file.trusted()) { + for (Declaration declaration : file.declarations()) { + if (!nearest.containsKey(declaration.key())) { + withheld.add(declaration); + } + } + } + } + return withheld.reversed(); + } + + /** + * Every declaration of a name, nearest first: the one that won, then those it shadowed. + * + * @param name a name + * @return its declarations, of any kind; empty when the catalog does not declare it + */ + public List declarations(String name) { + List found = new ArrayList<>(); + for (CatalogFile file : files) { + for (Declaration declaration : file.declarations()) { + if (declaration.name().equals(name)) { + found.add(declaration); + } + } + } + return found.reversed(); + } + + /** + * Whether a declaration is the one its name resolves to. + * + * @param declaration a declaration of this catalog + * @return {@code true} when no nearer declaration replaces it + */ + public boolean wins(Declaration declaration) { + Declaration winner = nearest.get(declaration.key()); + return winner != null && winner.statement() == declaration.statement(); + } + + /** + * The relations the catalog declares, each with the declaration that won. + * + * @return the winning relation declarations, by name + */ + public Map relations() { + Map relations = new HashMap<>(); + nearest.forEach((key, declaration) -> { + if (key.space() == Declaration.Space.RELATION) { + relations.put(key.name(), declaration); + } + }); + return relations; + } + + /** + * The statements that survive the overlay, in load order: every statement no nearer + * one replaces, and every statement that names nothing, such as an {@code import}. + * + * @return the statements to define + */ + public List survivors() { + List kept = new ArrayList<>(); + for (CatalogFile file : files) { + if (!file.trusted()) { + continue; + } + for (Statement statement : file.statements()) { + List declared = Declaration.of(statement, file); + if (declared.stream().allMatch(this::wins)) { + kept.add(statement); + } + } + } + return kept; + } + + /** + * Defines the survivors into a session. + * + * @param session the session, before its script + * @throws com.darkcollective.relix.embed.RelixException when they do not analyse + */ + public void define(Relix session) { + List kept = survivors(); + if (!kept.isEmpty()) { + session.define(kept.toArray(Statement[]::new)); + } + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/catalog/CatalogFile.java b/src/main/java/com/darkcollective/relix/cli/catalog/CatalogFile.java new file mode 100644 index 0000000..1fd8da7 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/catalog/CatalogFile.java @@ -0,0 +1,107 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.catalog; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.embed.Relix; +import com.darkcollective.relix.lang.ast.QueryStatement; +import com.darkcollective.relix.lang.ast.ScriptParseException; +import com.darkcollective.relix.lang.ast.Statement; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * One file of the catalog, parsed but not yet defined. + * + *

Parsing opens nothing and resolves nothing, so a file is parsed whether or not it + * will be loaded. It is parsed under its absolute path, which is what makes a relative + * path in one of its declarations resolve against the file's own directory. + * + * @param path the file, absolute + * @param level the {@code .relix/} directory it belongs to, or {@code null} for a + * {@code --catalog} or {@code $RELIX_CATALOG_PATH} file + * @param statements its statements, in order + * @param trusted whether it is loaded: it is in {@code ~/.relix} or a trusted directory, + * or was named on the command line. An untrusted file is parsed, so that + * {@code relix catalog} can say what it declares, but never defined. + */ +public record CatalogFile(Path path, Path level, List statements, boolean trusted) { + + /** + * A catalog file, checked. + */ + public CatalogFile { + Objects.requireNonNull(path, "path"); + statements = List.copyOf(statements); + } + + /** + * Reads and parses a catalog file. + * + * @param path the file, absolute + * @param level its {@code .relix/} directory, or {@code null} + * @param trusted whether it is to be loaded + * @return the parsed file; an untrusted file that does not parse is read as empty, + * since nothing of it will be loaded + * @throws CommandFailure with {@link ExitCode#ANALYSIS} when a trusted file does not + * parse or holds a query, and with {@link ExitCode#ENVIRONMENT} when it cannot + * be read + */ + static CatalogFile read(Path path, Path level, boolean trusted) { + String text; + try { + text = Files.readString(path, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new CommandFailure(ExitCode.ENVIRONMENT, path + ": cannot read: " + e.getMessage()); + } + List statements; + try { + statements = Relix.parse(text, path.toString()).statements(); + } catch (ScriptParseException e) { + if (!trusted) { + return new CatalogFile(path, level, List.of(), false); + } + throw new CommandFailure(ExitCode.ANALYSIS, path + ": " + e.getMessage()); + } + for (Statement statement : trusted ? statements : List.of()) { + if (statement instanceof QueryStatement query) { + throw new CommandFailure(ExitCode.ANALYSIS, path + ":" + query.location().line() + + ": a catalog file declares; a query belongs in a script"); + } + } + return new CatalogFile(path, level, statements, trusted); + } + + /** + * The named declarations this file makes, in order. + * + * @return its declarations + */ + public List declarations() { + List declared = new ArrayList<>(); + for (Statement statement : statements) { + declared.addAll(Declaration.of(statement, this)); + } + return declared; + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/catalog/Declaration.java b/src/main/java/com/darkcollective/relix/cli/catalog/Declaration.java new file mode 100644 index 0000000..c3ade0f --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/catalog/Declaration.java @@ -0,0 +1,133 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.catalog; + +import com.darkcollective.relix.lang.ast.AssignmentStatement; +import com.darkcollective.relix.lang.ast.ConnectionDeclaration; +import com.darkcollective.relix.lang.ast.DefRelationStatement; +import com.darkcollective.relix.lang.ast.DefStatement; +import com.darkcollective.relix.lang.ast.InlineTableBody; +import com.darkcollective.relix.lang.ast.RelateStatement; +import com.darkcollective.relix.lang.ast.SourceDeclaration; +import com.darkcollective.relix.lang.ast.Statement; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +/** + * One named declaration in a catalog file. + * + * @param name the name it declares + * @param kind what it declares + * @param statement the statement itself + * @param file the file it is in + */ +public record Declaration(String name, Kind kind, Statement statement, CatalogFile file) { + + /** + * A declaration, checked. + */ + public Declaration { + Objects.requireNonNull(name, "name"); + Objects.requireNonNull(kind, "kind"); + Objects.requireNonNull(statement, "statement"); + Objects.requireNonNull(file, "file"); + } + + /** What a declaration declares, and so which names it competes with. */ + public enum Kind { + /** {@code source NAME from …}: a relation. */ + SOURCE(Space.RELATION), + /** {@code NAME := { … }}: a relation. */ + VIEW(Space.RELATION), + /** {@code NAME := [ … ]}: a relation. */ + TABLE(Space.RELATION), + /** {@code def NAME(…) := RELATION}: a relation. */ + RELATION_FUNCTION(Space.RELATION), + /** {@code connection NAME …}. */ + CONNECTION(Space.CONNECTION), + /** {@code def NAME(…) -> TYPE := …}: a scalar function. */ + FUNCTION(Space.FUNCTION), + /** {@code relate NAME …}: a relationship between relations. */ + RELATIONSHIP(Space.RELATIONSHIP); + + private final Space space; + + Kind(Space space) { + this.space = space; + } + + /** + * The kind as {@code relix catalog} prints it. + * + * @return its lower-case name, such as {@code source} or {@code relation-function} + */ + public String label() { + return name().toLowerCase(Locale.ROOT).replace('_', '-'); + } + } + + /** + * The names that compete: a nearer declaration replaces a farther one with the same + * name in the same space. A view and a source share one, so a sub-project can point a + * view's {@code Orders} at a sample file. + */ + enum Space { RELATION, CONNECTION, FUNCTION, RELATIONSHIP } + + /** + * What this declaration competes for. + * + * @return its space and name + */ + Key key() { + return new Key(kind.space, name); + } + + /** A name in a space. */ + record Key(Space space, String name) { + } + + /** + * The named declarations one statement makes: none for an {@code import} or + * {@code env}, two for a {@code relate} with an inverse, otherwise one. + * + * @param statement a statement of a catalog file + * @param file the file + * @return its declarations + */ + static List of(Statement statement, CatalogFile file) { + List declared = new ArrayList<>(2); + switch (statement) { + case SourceDeclaration s -> declared.add(new Declaration(s.name(), Kind.SOURCE, s, file)); + case AssignmentStatement s -> declared.add(new Declaration(s.name(), + s.body() instanceof InlineTableBody ? Kind.TABLE : Kind.VIEW, s, file)); + case DefRelationStatement s -> declared.add(new Declaration(s.name(), Kind.RELATION_FUNCTION, s, file)); + case ConnectionDeclaration s -> declared.add(new Declaration(s.name(), Kind.CONNECTION, s, file)); + case DefStatement s -> declared.add(new Declaration(s.name(), Kind.FUNCTION, s, file)); + case RelateStatement s -> { + declared.add(new Declaration(s.name(), Kind.RELATIONSHIP, s, file)); + s.inverseName().ifPresent(inverse -> + declared.add(new Declaration(inverse, Kind.RELATIONSHIP, s, file))); + } + default -> { + // An import or an env statement names nothing of its own. + } + } + return declared; + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/catalog/Trust.java b/src/main/java/com/darkcollective/relix/cli/catalog/Trust.java new file mode 100644 index 0000000..4f168c0 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/catalog/Trust.java @@ -0,0 +1,239 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.catalog; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.config.RelixDirectories; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Stream; + +/** + * Which project {@code .relix/} directories the user has trusted (design §5.4), as + * {@code direnv} trusts an {@code .envrc}. + * + *

A project catalog can declare a connection or an HTTP source that sends a + * {@code ${SECRET}} to a host of its author's choosing, so loading one because of the + * directory you are standing in is the attack {@code direnv} answers. Here too, a project + * {@code .relix/} is loaded only once {@code relix catalog trust} has recorded a hash of + * its files in {@code ~/.relix/trusted}. The hash covers every file under the directory, + * its catalog, {@code relixrc} and {@code profiles.json} alike, so changing, adding or + * removing any of them means trusting it again. + * + *

{@code ~/.relix} is the user's own and always trusted, as is everything named on the + * command line. {@code RELIX_TRUST_ALL=1} trusts every directory, for a CI container whose + * checkout is the only thing in it. + * + *

The record is one line per directory, its hash and then its path, as + * {@code sha256sum} writes them. + */ +public final class Trust { + + /** The record's name inside {@code ~/.relix}. */ + public static final String FILE_NAME = "trusted"; + + /** The variable that trusts everything. */ + public static final String TRUST_ALL_VARIABLE = "RELIX_TRUST_ALL"; + + private final Path user; + private final Path record; + private final boolean all; + private final Map trusted; + + private Trust(Path user, boolean all, Map trusted) { + this.user = user; + this.record = user.resolve(FILE_NAME); + this.all = all; + this.trusted = trusted; + } + + /** + * The user's record of trust. + * + * @param home the user's home directory, absolute + * @param all whether {@code RELIX_TRUST_ALL} trusts everything + * @return what the user trusts + * @throws CommandFailure with {@link ExitCode#ENVIRONMENT} when the record cannot be read + */ + public static Trust load(Path home, boolean all) { + Path user = home.resolve(RelixDirectories.NAME); + Path record = user.resolve(FILE_NAME); + Map trusted = new TreeMap<>(); + if (Files.isRegularFile(record)) { + try { + for (String line : Files.readAllLines(record, StandardCharsets.UTF_8)) { + int space = line.indexOf(" "); + if (space > 0) { + trusted.put(Path.of(line.substring(space + 2)), line.substring(0, space)); + } + } + } catch (IOException e) { + throw new CommandFailure(ExitCode.ENVIRONMENT, record + ": cannot read: " + e.getMessage()); + } + } + return new Trust(user, all, trusted); + } + + /** + * Whether a {@code .relix/} directory may be loaded: it is the user's own, everything + * is trusted, or its files are as they were when it was trusted. + * + * @param level a {@code .relix/} directory, absolute + * @return {@code true} when it may be loaded + */ + public boolean trusts(Path level) { + return all || level.equals(user) || state(level) == State.TRUSTED; + } + + /** What the record says of a directory. */ + public enum State { + /** Trusted, and unchanged since. */ + TRUSTED, + /** Trusted, but a file has changed, been added or been removed since. */ + CHANGED, + /** Trusted, but no longer there. */ + GONE, + /** Never trusted. */ + UNTRUSTED; + + /** + * The state as {@code relix catalog trust --list} prints it. + * + * @return its lower-case name + */ + public String label() { + return name().toLowerCase(java.util.Locale.ROOT); + } + } + + /** + * What the record says of a {@code .relix/} directory now. + * + * @param level a {@code .relix/} directory, absolute + * @return its state + */ + public State state(Path level) { + String recorded = trusted.get(level); + if (recorded == null) { + return State.UNTRUSTED; + } + if (!Files.isDirectory(level)) { + return State.GONE; + } + return recorded.equals(hash(level)) ? State.TRUSTED : State.CHANGED; + } + + /** + * The directories the record holds, in order. + * + * @return the trusted {@code .relix/} directories + */ + public List directories() { + return List.copyOf(trusted.keySet()); + } + + /** + * Records a directory as trusted, as its files are now. + * + * @param level a {@code .relix/} directory, absolute + * @throws CommandFailure with {@link ExitCode#ENVIRONMENT} when the record cannot be written + */ + public void trust(Path level) { + trusted.put(level, hash(level)); + save(); + } + + /** + * Removes a directory from the record. + * + * @param level a {@code .relix/} directory, absolute + * @return whether it was there + * @throws CommandFailure with {@link ExitCode#ENVIRONMENT} when the record cannot be written + */ + public boolean revoke(Path level) { + if (trusted.remove(level) == null) { + return false; + } + save(); + return true; + } + + private void save() { + StringBuilder text = new StringBuilder(); + trusted.forEach((dir, hash) -> text.append(hash).append(" ").append(dir).append('\n')); + try { + Files.createDirectories(user); + Path temporary = Files.createTempFile(user, FILE_NAME, ".tmp"); + Files.writeString(temporary, text, StandardCharsets.UTF_8); + Files.move(temporary, record, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + throw new CommandFailure(ExitCode.ENVIRONMENT, record + ": cannot write: " + e.getMessage()); + } + } + + /** + * A hash of every file under a directory: each one's path relative to it and its + * bytes, in path order. + * + * @param level a directory + * @return the hash, in hexadecimal + * @throws CommandFailure with {@link ExitCode#ENVIRONMENT} when a file cannot be read + */ + static String hash(Path level) { + MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("every Java has SHA-256", e); + } + List files; + try (Stream walk = Files.walk(level)) { + files = new ArrayList<>(walk.filter(Files::isRegularFile).toList()); + } catch (IOException e) { + throw new CommandFailure(ExitCode.ENVIRONMENT, level + ": cannot read: " + e.getMessage()); + } + // The relative path with / separators, so that the order and the hash are the + // same on every platform. + files.sort((a, b) -> relative(level, a).compareTo(relative(level, b))); + for (Path file : files) { + digest.update(relative(level, file).getBytes(StandardCharsets.UTF_8)); + digest.update((byte) 0); + try { + digest.update(Files.readAllBytes(file)); + } catch (IOException e) { + throw new CommandFailure(ExitCode.ENVIRONMENT, file + ": cannot read: " + e.getMessage()); + } + digest.update((byte) 0); + } + return HexFormat.of().formatHex(digest.digest()); + } + + private static String relative(Path level, Path file) { + return level.relativize(file).toString().replace('\\', '/'); + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/catalog/package-info.java b/src/main/java/com/darkcollective/relix/cli/catalog/package-info.java new file mode 100644 index 0000000..01ab9c4 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/catalog/package-info.java @@ -0,0 +1,21 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * The catalog: the declarations a run takes from the {@code .relix/} directories above it + * and from {@code --catalog} files, overlaid so that the nearest declaration of a name wins + * (design §5). + */ +package com.darkcollective.relix.cli.catalog; diff --git a/src/main/java/com/darkcollective/relix/cli/command/BundleCommand.java b/src/main/java/com/darkcollective/relix/cli/command/BundleCommand.java new file mode 100644 index 0000000..e62cfa4 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/BundleCommand.java @@ -0,0 +1,107 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.io.Reporter; +import com.darkcollective.relix.cli.io.RowSink; +import com.darkcollective.relix.cli.report.QueryBundleJson; +import com.darkcollective.relix.embed.Diagnostic; +import com.darkcollective.relix.embed.Relation; +import com.darkcollective.relix.embed.RelixException; +import com.darkcollective.relix.semantic.SemanticError; +import picocli.CommandLine.Command; + +import java.util.ArrayList; +import java.util.List; + +/** + * {@code relix bundle}: the playground's JSON bundle of each script, its diagnostics and, + * per query, the logical plan, the rewrites, the physical plan and the planner's events + * (design §3). + * + *

A script that does not analyse still has a bundle, holding its diagnostics: a tool + * reading it wants to know what is wrong, and the exit status is still 3. A query whose + * physical plan cannot be made is bundled without one, and the reason goes to standard + * error. + */ +@Command(name = "bundle", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Prints each script's JSON bundle: diagnostics and, per query, the logical plan,", + "the rewrites, the physical plan and the planner's events. A script that does", + "not analyse still has one, holding its diagnostics."}) +final class BundleCommand extends ScriptCommand { + + @Override + ScriptRunner.View view(Invocation invocation) { + RowSink out = stdout(invocation); + Reporter reporter = invocation.reporter(); + return new ScriptRunner.View() { + @Override + public ExitCode show(ScriptRunner.Script script) { + List queries = queries(script); + List bundled = new ArrayList<>(); + for (Relation relation : queries) { + Relation optimized = relation.optimized(); + String label = relation.label().orElse(""); + String physical; + try { + physical = optimized.explainJson(); + } catch (RelixException e) { + reporter.error(script.source().name() + ": " + label + ": no physical plan: " + + e.getMessage()); + physical = null; + } + bundled.add(new QueryBundleJson.Query(label, relation.renderJson(), + optimized.rewrites(), physical, optimized.events())); + } + List diagnostics = script.diagnostics().stream() + .map(d -> error(script, d)) + .toList(); + out.text(QueryBundleJson.generate(queries.isEmpty() + ? script.session().model() : queries.getFirst().model(), diagnostics, bundled) + "\n"); + return ExitCode.SUCCESS; + } + + @Override + public boolean showsBroken() { + return true; + } + }; + } + + /** A broken script's queries cannot all be built; its bundle then holds none. */ + private static List queries(ScriptRunner.Script script) { + if (!script.broken()) { + return script.queries(); + } + try { + return script.queries(); + } catch (RelixException e) { + return List.of(); + } + } + + /** A diagnostic back in the analyser's own shape, placed where the user wrote it. */ + private static SemanticError error(ScriptRunner.Script script, Diagnostic diagnostic) { + Reporter.Place place = script.source().place(diagnostic.location()); + return new SemanticError(place.file(), place.line(), place.column(), diagnostic.message(), + diagnostic.severity()); + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/CatalogCommand.java b/src/main/java/com/darkcollective/relix/cli/command/CatalogCommand.java new file mode 100644 index 0000000..7ce0488 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/CatalogCommand.java @@ -0,0 +1,241 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.catalog.Catalog; +import com.darkcollective.relix.cli.catalog.CatalogFile; +import com.darkcollective.relix.cli.catalog.Declaration; +import com.darkcollective.relix.embed.Relix; +import com.darkcollective.relix.embed.RelixException; +import com.darkcollective.relix.embed.Tuple; +import com.darkcollective.relix.symbol.ScalarType; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.ParentCommand; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * {@code relix catalog}: the declarations the {@code .relix/} directories supply, and which + * of those directories are trusted (design §5.3, §5.4). + * + *

Every listing is rows, written as a run writes its results: a table on a terminal, + * tsv in a pipe, or whatever {@code -o} names. {@code ls} and {@code schema} are queries + * over the engine's own introspection relations, {@code relix.relations} and + * {@code relix.columns}, run in a session holding the catalog and nothing else. + */ +@Command(name = "catalog", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Inspects the catalog: the declarations a run takes from the .relix/ directories", + "from ~/.relix down to the working directory, and from --catalog files.", + "A project's .relix/ is loaded only once it is trusted (relix catalog trust)."}, + subcommands = { + CatalogCommand.Files.class, CatalogCommand.Ls.class, CatalogCommand.Schema.class, + CatalogCommand.Where.class, TrustCommand.class}) +final class CatalogCommand implements Callable { + + @ParentCommand + RelixCommand root; + + @Override + public Integer call() { + throw new CommandFailure(ExitCode.USAGE, null); + } + + /** + * Runs {@code use} over a session holding the catalog and nothing else. + * + * @throws CommandFailure with {@link ExitCode#ANALYSIS} when the catalog does not analyse + */ + private static T inSession(Invocation invocation, Function use) { + try (Relix session = invocation.session(invocation.directory()).build()) { + invocation.catalog().define(session); + return use.apply(session); + } catch (RelixException e) { + throw new CommandFailure(ExitCode.of(e), "the catalog does not analyse: " + e.getMessage()); + } + } + + private static List rows(Relix session, String expression) { + try (Stream rows = session.relation(expression).stream()) { + return rows.toList(); + } + } + + private static CommandFailure unknown(String name) { + return new CommandFailure(ExitCode.ANALYSIS, "the catalog declares no '" + name + "'"); + } + + /** {@code relix catalog files}. */ + @Command(name = "files", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Lists every catalog file, in the order they load, with the names each declares", + "and whether it is trusted. An untrusted file is listed but not loaded."}) + static final class Files implements Callable { + + @ParentCommand + CatalogCommand catalog; + + @Mixin + FormatOptions formatting; + + @Override + public Integer call() { + Invocation invocation = catalog.root.invocation(true); + Listing listing = new Listing("file", ScalarType.STRING, "trusted", ScalarType.BOOLEAN, + "declares", ScalarType.STRING); + for (CatalogFile file : invocation.catalog().files()) { + listing.add(file.path().toString(), file.trusted(), file.declarations().stream() + .map(Declaration::name).distinct().collect(Collectors.joining(" "))); + } + listing.print(invocation, invocation.format(formatting), "files"); + return ExitCode.SUCCESS.status(); + } + } + + /** {@code relix catalog ls}. */ + @Command(name = "ls", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Lists the relations the catalog declares: name, kind, arity and the file whose", + "declaration won. kind is the engine's: SRC a source, INL an inline table,", + "QR a view, DB a connection's table."}) + static final class Ls implements Callable { + + @ParentCommand + CatalogCommand catalog; + + @Mixin + FormatOptions formatting; + + @Override + public Integer call() { + Invocation invocation = catalog.root.invocation(true); + Map declared = invocation.catalog().relations(); + List relations = inSession(invocation, session -> rows(session, + "τ name (π name, kind, arity (relix.relations ⟕ name = relation" + + " (γ relation, COUNT(*) → arity (relix.columns))))")); + Listing listing = new Listing("name", ScalarType.STRING, "kind", ScalarType.STRING, + "arity", ScalarType.NUMBER, "file", ScalarType.STRING); + for (Tuple relation : relations) { + Declaration declaration = declared.get(relation.string("name")); + listing.add(relation.string("name"), relation.string("kind"), relation.longValue("arity"), + declaration == null ? null : declaration.file().path().toString()); + } + listing.print(invocation, invocation.format(formatting), "relations"); + return ExitCode.SUCCESS.status(); + } + } + + /** {@code relix catalog schema NAME}. */ + @Command(name = "schema", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = "Lists the columns of the relation NAME and their types, in order.", + exitCodeListHeading = "%nExit status:%n", + exitCodeList = {" 0:success", " 3:the catalog declares no relation NAME, or does not analyse"}) + static final class Schema implements Callable { + + @ParentCommand + CatalogCommand catalog; + + @Parameters(paramLabel = "NAME", description = "A relation the catalog declares.") + String name; + + @Mixin + FormatOptions formatting; + + @Override + public Integer call() { + Invocation invocation = catalog.root.invocation(true); + List columns = inSession(invocation, session -> { + boolean known = rows(session, "π name (relix.relations)").stream() + .anyMatch(r -> name.equals(r.string("name"))); + if (!known) { + throw unknown(name); + } + return rows(session, "τ ordinal (relix.columns)").stream() + .filter(c -> name.equals(c.string("relation"))) + .toList(); + }); + Listing listing = new Listing("column", ScalarType.STRING, "type", ScalarType.STRING); + for (Tuple column : columns) { + listing.add(column.string("column"), column.string("type")); + } + listing.print(invocation, invocation.format(formatting), name); + return ExitCode.SUCCESS.status(); + } + } + + /** {@code relix catalog where NAME}. */ + @Command(name = "where", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Lists every declaration of NAME, nearest first: the one that won, then the ones", + "it shadowed. A declaration in an untrusted directory is listed as untrusted."}, + exitCodeListHeading = "%nExit status:%n", + exitCodeList = {" 0:success", " 3:the catalog declares no NAME"}) + static final class Where implements Callable { + + @ParentCommand + CatalogCommand catalog; + + @Parameters(paramLabel = "NAME", description = "A name the catalog declares.") + String name; + + @Mixin + FormatOptions formatting; + + @Override + public Integer call() { + Invocation invocation = catalog.root.invocation(true); + Catalog declared = invocation.catalog(); + List declarations = declared.declarations(name); + if (declarations.isEmpty()) { + throw unknown(name); + } + Listing listing = new Listing("file", ScalarType.STRING, "line", ScalarType.NUMBER, + "kind", ScalarType.STRING, "state", ScalarType.STRING); + for (Declaration declaration : declarations) { + String state = !declaration.file().trusted() ? "untrusted" + : declared.wins(declaration) ? "wins" : "shadowed"; + listing.add(declaration.file().path().toString(), declaration.statement().location().line(), + declaration.kind().label(), state); + } + listing.print(invocation, invocation.format(formatting), name); + return ExitCode.SUCCESS.status(); + } + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/CheckCommand.java b/src/main/java/com/darkcollective/relix/cli/command/CheckCommand.java new file mode 100644 index 0000000..1126bed --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/CheckCommand.java @@ -0,0 +1,117 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.io.Reporter; +import com.darkcollective.relix.embed.Diagnostic; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import tools.jackson.core.JsonGenerator; +import tools.jackson.core.ObjectWriteContext; +import tools.jackson.core.json.JsonFactory; + +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/** + * {@code relix check}: analyses scripts without running them, as a lint pass (design + * §3.4, §3.5). Nothing goes to standard output: the diagnostics go to standard error, or, + * with {@code --json}, to standard output as one document. + */ +@Command(name = "check", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Analyses scripts without running them, and reports what is wrong with them", + "on standard error as FILE:LINE:COL: severity: message.", + " find . -name '*.relix' -print0 | xargs -0 relix check"}, + exitCodeListHeading = "%nExit status:%n", + exitCodeList = {" 0:every script analyses", " 3:a script does not parse or analyse"}) +final class CheckCommand extends ScriptCommand { + + private static final JsonFactory JSON = new JsonFactory(); + + @Option(names = "--json", + description = "Write the diagnostics to standard output as one JSON document instead.") + boolean json; + + @Override + ScriptRunner.View view(Invocation invocation) { + return new ScriptRunner.View() { + + private final List found = new ArrayList<>(); + + @Override + public ExitCode show(ScriptRunner.Script script) { + for (Diagnostic diagnostic : script.diagnostics()) { + found.add(new Placed(script.source().place(diagnostic.location()), diagnostic)); + } + return ExitCode.SUCCESS; + } + + @Override + public boolean showsBroken() { + return true; + } + + @Override + public boolean reportsDiagnostics() { + return !json; + } + + @Override + public ExitCode finish(ExitCode result) { + if (json) { + stdout(invocation).text(document(found)); + } + return result; + } + }; + } + + /** A diagnostic and where it falls, in the terms the user wrote. */ + private record Placed(Reporter.Place place, Diagnostic diagnostic) { + } + + /** + * {@code {"diagnostics": [{"file", "line", "column", "severity", "message"}, …]}}, the + * line and column 0 when unknown. + */ + private static String document(List found) { + StringWriter out = new StringWriter(); + try (JsonGenerator w = JSON.createGenerator(ObjectWriteContext.empty(), out)) { + w.writeStartObject(); + w.writeName("diagnostics").writeStartArray(); + for (Placed placed : found) { + w.writeStartObject() + .writeName("file").writeString(placed.place().file()) + .writeName("line").writeNumber(placed.place().line()) + .writeName("column").writeNumber(placed.place().column()) + .writeName("severity").writeString( + placed.diagnostic().severity().name().toLowerCase(Locale.ROOT)) + .writeName("message").writeString(placed.diagnostic().message()) + .writeEndObject(); + } + w.writeEndArray(); + w.writeEndObject(); + } + return out + "\n"; + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/Completion.java b/src/main/java/com/darkcollective/relix/cli/command/Completion.java new file mode 100644 index 0000000..c47cf2b --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/Completion.java @@ -0,0 +1,241 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import picocli.AutoComplete; +import picocli.CommandLine; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Model.OptionSpec; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Completion scripts for the command, generated from its picocli model, so that they + * cannot drift from the options it parses (design §7.3). + * + *

bash's is picocli's own, which also runs under zsh through {@code bashcompinit}; zsh's + * wraps it so that it works both sourced and as {@code _relix} on {@code $fpath}, which is + * where Homebrew installs it. fish and PowerShell have none from picocli, so theirs are + * written here: each completes the commands, then each command's subcommands and options. + */ +final class Completion { + + /** The shells there is a script for. */ + enum Shell { + BASH, ZSH, FISH, POWERSHELL; + + static Shell of(String name) { + try { + return valueOf(name.strip().toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + return null; + } + } + } + + private Completion() { + } + + /** + * The completion script for a shell. + * + * @param shell the shell + * @param root the root command + * @return the script + */ + static String script(Shell shell, CommandLine root) { + String name = root.getCommandName(); + return switch (shell) { + case BASH -> AutoComplete.bash(name, root); + case ZSH -> zsh(name, root); + case FISH -> fish(name, root.getCommandSpec()); + case POWERSHELL -> powershell(name, root.getCommandSpec()); + }; + } + + // ------------------------------------------------------------------------- + // zsh + // ------------------------------------------------------------------------- + + /** + * bash's script, made a zsh completion function too. On {@code $fpath} the file is the + * body of {@code _relix}, which compinit binds to the command by its first line. Its + * first call defines the bash completion and rebinds the command to it, so it then + * completes the word it was called for; sourced, outside a completion, it does not. + */ + private static String zsh(String name, CommandLine root) { + return "#compdef " + name + "\n" + + "# zsh completion for " + name + ", generated from its command model: bash's, run\n" + + "# through bashcompinit.\n" + + "# Install: " + name + " completion zsh > \"${fpath[1]}/_" + name + "\"\n" + + "# or: source <(" + name + " completion zsh)\n\n" + + AutoComplete.bash(name, root) + + "\nif (( ${+compstate} )); then\n" + + " _bash_complete -F _complete_" + name + " -o default\n" + + "fi\n"; + } + + // ------------------------------------------------------------------------- + // fish + // ------------------------------------------------------------------------- + + private static String fish(String name, CommandSpec root) { + StringBuilder out = new StringBuilder(); + out.append("# fish completion for ").append(name).append(", generated from its command model.\n"); + out.append("# Install: ").append(name).append(" completion fish > ~/.config/fish/completions/") + .append(name).append(".fish\n\n"); + out.append("complete -c ").append(name).append(" -f\n"); + fish(out, name, root, List.of()); + return out.toString(); + } + + private static void fish(StringBuilder out, String name, CommandSpec command, List path) { + String condition = fishCondition(command, path); + String prefix = "complete -c " + name + " -n " + fishQuote(condition); + for (Map.Entry sub : subcommands(command).entrySet()) { + out.append(prefix).append(" -a ").append(fishQuote(sub.getKey())) + .append(" -d ").append(fishQuote(summary(sub.getValue().usageMessage().description()))) + .append('\n'); + } + for (OptionSpec option : command.options()) { + if (option.hidden()) { + continue; + } + StringBuilder line = new StringBuilder(prefix); + for (String optionName : option.names()) { + if (optionName.startsWith("--")) { + line.append(" -l ").append(fishQuote(optionName.substring(2))); + } else if (optionName.length() == 2) { + line.append(" -s ").append(fishQuote(optionName.substring(1))); + } + } + if (option.arity().max() > 0) { + line.append(" -r"); + } + line.append(" -d ").append(fishQuote(summary(option.description()))); + out.append(line).append('\n'); + } + if (!command.positionalParameters().isEmpty()) { + out.append(prefix).append(" -F\n"); + } + for (Map.Entry sub : subcommands(command).entrySet()) { + List deeper = new ArrayList<>(path); + deeper.add(sub.getKey()); + fish(out, name, sub.getValue(), deeper); + } + } + + /** Where in the command line a command's completions apply. */ + private static String fishCondition(CommandSpec command, List path) { + List parts = new ArrayList<>(); + if (path.isEmpty()) { + parts.add("__fish_use_subcommand"); + } else { + for (String step : path) { + parts.add("__fish_seen_subcommand_from " + step); + } + if (!command.subcommands().isEmpty()) { + parts.add("not __fish_seen_subcommand_from " + String.join(" ", subcommands(command).keySet())); + } + } + return String.join("; and ", parts); + } + + private static String fishQuote(String text) { + return "'" + text.replace("\\", "\\\\").replace("'", "\\'") + "'"; + } + + // ------------------------------------------------------------------------- + // PowerShell + // ------------------------------------------------------------------------- + + private static String powershell(String name, CommandSpec root) { + Map> words = new LinkedHashMap<>(); + powershell(words, root, ""); + StringBuilder out = new StringBuilder(); + out.append("# PowerShell completion for ").append(name).append(", generated from its command model.\n"); + out.append("# Install: ").append(name).append(" completion powershell >> $PROFILE\n\n"); + out.append("Register-ArgumentCompleter -Native -CommandName ").append(psQuote(name)) + .append(" -ScriptBlock {\n"); + out.append(" param($wordToComplete, $commandAst, $cursorPosition)\n"); + out.append(" $words = @{\n"); + for (Map.Entry> entry : words.entrySet()) { + out.append(" ").append(psQuote(entry.getKey())).append(" = @("); + List quoted = entry.getValue().stream().map(Completion::psQuote).toList(); + out.append(String.join(", ", quoted)).append(")\n"); + } + out.append(" }\n"); + out.append(" $path = ''\n"); + out.append(" foreach ($element in $commandAst.CommandElements | Select-Object -Skip 1) {\n"); + out.append(" if ($element.Extent.StartOffset -ge $cursorPosition) { break }\n"); + out.append(" $candidate = ($path + ' ' + $element.ToString()).Trim()\n"); + out.append(" if ($words.ContainsKey($candidate)) { $path = $candidate }\n"); + out.append(" }\n"); + out.append(" $words[$path] | Where-Object { $_ -like \"$wordToComplete*\" } | ForEach-Object {\n"); + out.append(" [System.Management.Automation.CompletionResult]::new($_, $_, 'ParameterValue', $_)\n"); + out.append(" }\n"); + out.append("}\n"); + return out.toString(); + } + + private static void powershell(Map> words, CommandSpec command, String path) { + List here = new ArrayList<>(subcommands(command).keySet()); + for (OptionSpec option : command.options()) { + if (!option.hidden()) { + here.addAll(List.of(option.names())); + } + } + words.put(path, here); + for (Map.Entry sub : subcommands(command).entrySet()) { + powershell(words, sub.getValue(), (path + " " + sub.getKey()).strip()); + } + } + + private static String psQuote(String text) { + return "'" + text.replace("'", "''") + "'"; + } + + // ------------------------------------------------------------------------- + + /** A command's subcommands by their names, aliases left out. */ + private static Map subcommands(CommandSpec command) { + Map named = new LinkedHashMap<>(); + command.subcommands().forEach((key, sub) -> { + if (key.equals(sub.getCommandName())) { + named.put(key, sub.getCommandSpec()); + } + }); + return named; + } + + /** + * The first line of a description that has any text, without picocli's line breaks + * ({@code %n}) and with its variables removed. + */ + private static String summary(String[] description) { + for (String line : String.join("%n", description).split("%n")) { + String text = line.replaceAll("(? ~/.config/fish/completions/relix.fish", + " powershell: relix completion powershell | Out-String | Invoke-Expression"}) +final class CompletionCommand implements Callable { + + @ParentCommand + RelixCommand root; + + @Spec + CommandSpec spec; + + @Parameters(paramLabel = "SHELL", description = "bash, zsh, fish or powershell.") + String shell; + + @Override + public Integer call() { + Completion.Shell chosen = Completion.Shell.of(shell); + if (chosen == null) { + throw new CommandFailure(ExitCode.USAGE, "'" + shell + "' is not one of bash, zsh, fish, powershell"); + } + new RowSink(root.host().out(), false).text(Completion.script(chosen, spec.root().commandLine())); + return ExitCode.SUCCESS.status(); + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/ConnectorsCommand.java b/src/main/java/com/darkcollective/relix/cli/command/ConnectorsCommand.java new file mode 100644 index 0000000..76d7deb --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/ConnectorsCommand.java @@ -0,0 +1,140 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.drivers.DownloadProgress; +import com.darkcollective.relix.processor.connector.ConnectorProvisioner; +import com.darkcollective.relix.symbol.ScalarType; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.ParentCommand; + +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.Callable; + +/** + * {@code relix connectors}: the connectors installed, those that can be downloaded, and + * installing one (design §3). Installing is how a connector arrives ahead of a run; + * during a run, {@code --allow-download} permits fetching one the run needs. + */ +@Command(name = "connectors", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Lists and installs connector plugins, such as mongodb. A source of a type no", + "installed connector reads needs one, installed here ahead of a run, or fetched", + "during one with --allow-download."}, + subcommands = {ConnectorsCommand.Ls.class, ConnectorsCommand.Install.class}) +final class ConnectorsCommand implements Callable { + + @ParentCommand + RelixCommand root; + + @Override + public Integer call() { + throw new CommandFailure(ExitCode.USAGE, null); + } + + private static ConnectorProvisioner provisioner(Invocation invocation, boolean download) { + PrintWriter progress = new PrintWriter(invocation.host().err(), true, StandardCharsets.UTF_8); + return ConnectorProvisioner.create(download, DownloadProgress.reporting(progress)); + } + + /** {@code relix connectors ls}. */ + @Command(name = "ls", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Lists the connectors installed: those shipped with relix and the plugins", + "installed in ~/.relix/connectors. --available adds the ones the hosted", + "catalog offers, which it fetches."}) + static final class Ls implements Callable { + + @ParentCommand + ConnectorsCommand connectors; + + @Option(names = "--available", + description = "Also list the connectors that can be installed, from the hosted catalog.") + boolean available; + + @Mixin + FormatOptions formatting; + + @Override + public Integer call() { + Invocation invocation = connectors.root.invocation(false); + Set installed = ConnectorProvisioner.installedTypes(); + Set types = new TreeSet<>(installed); + if (available) { + types.addAll(provisioner(invocation, false).catalogTypes()); + } + Listing listing = new Listing("type", ScalarType.STRING, "installed", ScalarType.BOOLEAN); + for (String type : types) { + listing.add(type, installed.contains(type)); + } + listing.print(invocation, invocation.format(formatting), "connectors"); + return ExitCode.SUCCESS.status(); + } + } + + /** {@code relix connectors install TYPE}. */ + @Command(name = "install", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = "Downloads the connector plugin TYPE, checks its checksums, and installs it.", + exitCodeListHeading = "%nExit status:%n", + exitCodeList = { + " 0:installed, or already installed", + " 2:the catalog has no connector of that type", + " 5:the download failed"}) + static final class Install implements Callable { + + @ParentCommand + ConnectorsCommand connectors; + + @Parameters(paramLabel = "TYPE", description = "A connector type, such as mongodb.") + String type; + + @Override + public Integer call() { + Invocation invocation = connectors.root.invocation(false); + if (ConnectorProvisioner.installedTypes().contains(type)) { + invocation.reporter().info("the " + type + " connector is already installed"); + return ExitCode.SUCCESS.status(); + } + ConnectorProvisioner.Result result = provisioner(invocation, true).provision(type); + return switch (result.outcome()) { + case PROVISIONED -> { + invocation.reporter().info("installed the " + type + " connector into " + + ConnectorProvisioner.defaultDirectory() + ": " + result.message()); + yield ExitCode.SUCCESS.status(); + } + case UNKNOWN -> throw new CommandFailure(ExitCode.USAGE, result.message()); + case DISABLED, FAILED -> throw new CommandFailure(ExitCode.ENVIRONMENT, result.message()); + }; + } + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/DocCommand.java b/src/main/java/com/darkcollective/relix/cli/command/DocCommand.java new file mode 100644 index 0000000..f1dc6cc --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/DocCommand.java @@ -0,0 +1,122 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.io.Pager; +import com.darkcollective.relix.cli.io.RowSink; +import com.darkcollective.relix.cli.render.DocRenderer; +import com.darkcollective.relix.docs.ReferenceLookup; +import com.darkcollective.relix.docs.ReferencePage; +import com.darkcollective.relix.docs.RelixDocs; +import com.darkcollective.relix.function.FunctionCatalog; +import com.darkcollective.relix.symbol.ScalarType; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.ParentCommand; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Callable; + +/** + * {@code relix doc [TOPIC]}: a page of the language reference (design §3). + * + *

A topic is found by glyph, keyword or name: {@code σ}, {@code select} and + * {@code selection} are one page. A language keyword keeps a name it shares with a + * function, so {@code relix doc fix} is the recursion operator and + * {@code relix doc --function fix} is the function. The page is rendered for a terminal + * and, on one, shown through {@code $PAGER}. With no topic, the pages are listed, as rows. + * + *

Pages are found by {@code relix-docs}' {@link ReferenceLookup}, over the language pages + * and the installed functions' pages (DarkCollective/relix-core#95). + */ +@Command(name = "doc", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Shows the reference page for TOPIC, found by glyph, keyword or name (σ, select,", + "selection), through $PAGER on a terminal. With no TOPIC, lists the pages."}, + exitCodeListHeading = "%nExit status:%n", + exitCodeList = {" 0:success", " 2:no page has that topic"}) +final class DocCommand implements Callable { + + @ParentCommand + RelixCommand root; + + @Parameters(paramLabel = "TOPIC", arity = "0..1", + description = "A glyph, keyword or name, such as σ, select, join or Len.") + String topic; + + @Option(names = "--function", + description = "Look TOPIC up among the functions only, or list only them.") + boolean function; + + @Option(names = "--markdown", description = "Print the page's markdown as it is, not rendered.") + boolean markdown; + + @Option(names = "--no-pager", description = "Write the page to standard output even on a terminal.") + boolean noPager; + + @Mixin + FormatOptions formatting; + + @Override + public Integer call() { + Invocation invocation = root.invocation(false); + ReferenceLookup index = ReferenceLookup.of(FunctionCatalog.discover()); + if (topic == null) { + list(invocation, index); + return ExitCode.SUCCESS.status(); + } + Optional found = function ? index.function(topic) : index.entry(topic); + ReferencePage page = found.orElseThrow(() -> new CommandFailure(ExitCode.USAGE, + "no reference page for '" + topic + "'" + (function ? " among the functions" : "") + + "; relix doc" + (function ? " --function" : "") + " lists them")); + String text = index.page(page).orElseThrow(() -> new CommandFailure(ExitCode.INTERNAL, + "the reference lists " + page.path() + " but has no page there")); + RowSink out = new RowSink(invocation.host().out(), false); + if (!markdown) { + text = DocRenderer.render(text, page.path()); + } + if (noPager) { + out.text(text); + } else { + Pager.show(invocation.host(), text, out); + } + return ExitCode.SUCCESS.status(); + } + + /** The language pages, in the reference's order, unless --function; then the functions'. */ + private void list(Invocation invocation, ReferenceLookup index) { + Listing listing = new Listing("topic", ScalarType.STRING, "category", ScalarType.STRING, + "title", ScalarType.STRING, "summary", ScalarType.STRING); + List pages = new ArrayList<>(); + if (!function) { + pages.addAll(RelixDocs.referencePages()); + } + index.functionsByCategory().values().forEach(pages::addAll); + for (ReferencePage page : pages) { + listing.add(page.symbol(), page.category(), page.title(), page.summary()); + } + listing.print(invocation, invocation.format(formatting), "reference"); + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/DriversCommand.java b/src/main/java/com/darkcollective/relix/cli/command/DriversCommand.java new file mode 100644 index 0000000..b333693 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/DriversCommand.java @@ -0,0 +1,155 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.drivers.DownloadProgress; +import com.darkcollective.relix.connectors.std.DriverCatalog; +import com.darkcollective.relix.connectors.std.DriverProvisioner; +import com.darkcollective.relix.symbol.ScalarType; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.ParentCommand; + +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Locale; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; + +/** + * {@code relix drivers}: the JDBC drivers relix can download, and installing one + * (design §3). Installing is how a driver arrives ahead of a run; during a run, + * {@code --allow-download} permits fetching one the run needs. + */ +@Command(name = "drivers", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Lists and installs JDBC drivers. A connection to a database needs its driver,", + "installed here ahead of a run, or fetched during one with --allow-download."}, + subcommands = {DriversCommand.Ls.class, DriversCommand.Install.class}) +final class DriversCommand implements Callable { + + /** A URL of a scheme's, for asking whether a driver accepts that scheme. */ + private static final String PROBE = "//localhost/"; + + @ParentCommand + RelixCommand root; + + @Override + public Integer call() { + throw new CommandFailure(ExitCode.USAGE, null); + } + + /** Whether a driver that accepts a JDBC scheme is registered. */ + static boolean registered(String scheme) { + try { + return DriverManager.getDriver(scheme + PROBE) != null; + } catch (SQLException none) { + return false; + } + } + + /** The short name of a scheme: {@code postgresql} for {@code jdbc:postgresql:}. */ + static String shortName(String scheme) { + String name = scheme.toLowerCase(Locale.ROOT); + name = name.startsWith("jdbc:") ? name.substring(5) : name; + return name.endsWith(":") ? name.substring(0, name.length() - 1) : name; + } + + /** {@code relix drivers ls}. */ + @Command(name = "ls", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Lists the drivers relix can install: name, the JDBC scheme each serves,", + "and whether one is installed."}) + static final class Ls implements Callable { + + @ParentCommand + DriversCommand drivers; + + @Mixin + FormatOptions formatting; + + @Override + public Integer call() { + Invocation invocation = drivers.root.invocation(false); + invocation.loadInstalledDrivers(); + Listing listing = new Listing("name", ScalarType.STRING, "scheme", ScalarType.STRING, + "installed", ScalarType.BOOLEAN); + for (DriverCatalog.Entry entry : DriverCatalog.load().entries()) { + listing.add(shortName(entry.scheme()), entry.scheme(), registered(entry.scheme())); + } + listing.print(invocation, invocation.format(formatting), "drivers"); + return ExitCode.SUCCESS.status(); + } + } + + /** {@code relix drivers install NAME}. */ + @Command(name = "install", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = "Downloads the driver NAME, checks its checksum, and installs it.", + exitCodeListHeading = "%nExit status:%n", + exitCodeList = { + " 0:installed, or already installed", + " 2:no driver has that name", + " 5:the download failed"}) + static final class Install implements Callable { + + @ParentCommand + DriversCommand drivers; + + @Parameters(paramLabel = "NAME", description = "A driver relix drivers ls lists, such as postgresql.") + String name; + + @Override + public Integer call() { + Invocation invocation = drivers.root.invocation(false); + DriverCatalog catalog = DriverCatalog.load(); + DriverCatalog.Entry entry = catalog.entries().stream() + .filter(e -> shortName(e.scheme()).equalsIgnoreCase(name) || e.name().equalsIgnoreCase(name)) + .findFirst() + .orElseThrow(() -> new CommandFailure(ExitCode.USAGE, "no driver named '" + name + "'; " + + "relix can install " + catalog.entries().stream() + .map(e -> shortName(e.scheme())).collect(Collectors.joining(", ")))); + invocation.loadInstalledDrivers(); + if (registered(entry.scheme())) { + invocation.reporter().info(entry.name() + " is already installed"); + return ExitCode.SUCCESS.status(); + } + PrintWriter progress = new PrintWriter(invocation.host().err(), true, StandardCharsets.UTF_8); + DriverProvisioner.Result result = DriverProvisioner.create(true, DownloadProgress.reporting(progress)) + .provision(entry.scheme() + PROBE); + return switch (result.outcome()) { + case PROVISIONED, ALREADY_AVAILABLE -> { + invocation.reporter().info("installed " + entry.name() + ": " + result.message()); + yield ExitCode.SUCCESS.status(); + } + case UNKNOWN, DISABLED, FAILED -> throw new CommandFailure(ExitCode.ENVIRONMENT, result.message()); + }; + } + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/ExplainCommand.java b/src/main/java/com/darkcollective/relix/cli/command/ExplainCommand.java new file mode 100644 index 0000000..75fb584 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/ExplainCommand.java @@ -0,0 +1,91 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.io.RowSink; +import com.darkcollective.relix.embed.Relation; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import tools.jackson.core.JsonGenerator; +import tools.jackson.core.ObjectWriteContext; +import tools.jackson.core.json.JsonFactory; + +import java.io.StringWriter; +import java.util.List; + +/** + * {@code relix explain}: each query's physical plan, the tree the engine would run + * (design §3). + * + *

As text, a script with several queries prints each plan under a {@code -- query NAME} + * line. With {@code --json}, each query is one line of its own, {@code {"script", "query", + * "plan"}}, so a stream of plans reads as NDJSON. + */ +@Command(name = "explain", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = "Prints the physical plan of each query: how the engine would run it.") +final class ExplainCommand extends ScriptCommand { + + private static final JsonFactory JSON = new JsonFactory(); + + @Option(names = "--json", + description = "Write one JSON object per query, {\"script\", \"query\", \"plan\"}, one per line.") + boolean json; + + @Option(names = "--query", paramLabel = "NAME", + description = "Explain only the query NAME.") + String query; + + @Override + ScriptRunner.View view(Invocation invocation) { + RowSink out = stdout(invocation); + return script -> { + List queries = script.queries(query); + StringBuilder text = new StringBuilder(); + for (Relation relation : queries) { + if (json) { + text.append(line(script.source().name(), relation)).append('\n'); + } else { + if (queries.size() > 1) { + text.append("-- query ").append(relation.label().orElse("")).append('\n'); + } + text.append(relation.explain().stripTrailing()).append('\n'); + if (queries.size() > 1 && relation != queries.getLast()) { + text.append('\n'); + } + } + } + out.text(text.toString()); + return ExitCode.SUCCESS; + }; + } + + private static String line(String script, Relation relation) { + StringWriter out = new StringWriter(); + try (JsonGenerator w = JSON.createGenerator(ObjectWriteContext.empty(), out)) { + w.writeStartObject(); + w.writeName("script").writeString(script); + w.writeName("query"); + relation.label().ifPresentOrElse(w::writeString, w::writeNull); + w.writeName("plan").writeRawValue(relation.explainJson()); + w.writeEndObject(); + } + return out.toString(); + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/FmtCommand.java b/src/main/java/com/darkcollective/relix/cli/command/FmtCommand.java new file mode 100644 index 0000000..a5af49a --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/FmtCommand.java @@ -0,0 +1,158 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.ast.Spelling; +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.io.Reporter; +import com.darkcollective.relix.cli.io.RowSink; +import com.darkcollective.relix.cli.io.ScriptSource; +import com.darkcollective.relix.cli.render.ScriptFormatter; +import com.darkcollective.relix.lang.ast.ScriptParseException; +import picocli.CommandLine.ArgGroup; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.ParentCommand; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; + +/** + * {@code relix fmt}: prints scripts canonically (design §3), as {@code gofmt} does Go. + * + *

Each script is printed by the engine's script printer, with its comments kept + * (see {@link ScriptFormatter}). With no option the formatted scripts go to standard + * output; {@code -w} rewrites each file that changes in place, and {@code --check} changes + * nothing, names each file that would change, and exits 1 if any would, for CI. + * + *

A script that does not parse is reported as {@code FILE:LINE:COL: error: message}, + * left alone, and makes the run exit 3. Nothing is analysed or run: a script naming a + * relation nothing declares is formatted all the same. + */ +@Command(name = "fmt", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Prints scripts canonically, keeping their comments. With no SCRIPT, formats", + "standard input."}, + exitCodeListHeading = "%nExit status:%n", + exitCodeList = { + " 0:success; with --check, no file would change", + " 1:with --check, a file would change", + " 3:a script does not parse"}) +final class FmtCommand implements Callable { + + @ParentCommand + RelixCommand root; + + @Option(names = "-w", description = "Rewrite each file that changes in place, instead of printing it.") + boolean write; + + @Option(names = "--check", + description = "Change nothing; print the name of each file that would change, and exit 1 if any would.") + boolean check; + + @ArgGroup(exclusive = true) + SpellingChoice spelling = new SpellingChoice(); + + @Parameters(paramLabel = "SCRIPT", arity = "0..*", + description = "Script files; - is standard input, which is also read when none is named.") + List files = new ArrayList<>(); + + /** {@code --glyphs} or {@code --keywords}. */ + static final class SpellingChoice { + + @Option(names = "--glyphs", description = "Write operators as glyphs: σ, ⋈, ∧ (the default).") + boolean glyphs; + + @Option(names = "--keywords", description = "Write operators as ASCII keywords: SELECT, JOIN, AND.") + boolean keywords; + } + + @Override + public Integer call() { + if (write && check) { + throw new CommandFailure(ExitCode.USAGE, "-w and --check cannot be given together"); + } + Invocation invocation = root.invocation(false); + Spelling chosen = spelling.keywords ? Spelling.KEYWORDS : Spelling.GLYPHS; + List sources = sources(invocation); + if (write && sources.stream().anyMatch(s -> s.name().equals(ScriptSource.STDIN))) { + throw new CommandFailure(ExitCode.USAGE, "-w rewrites files; standard input has none to rewrite"); + } + Reporter reporter = invocation.reporter(); + RowSink out = new RowSink(invocation.host().out(), false); + ExitCode result = ExitCode.SUCCESS; + boolean changes = false; + for (ScriptSource source : sources) { + String formatted; + try { + formatted = ScriptFormatter.format(source.text(), source.name(), chosen); + } catch (ScriptParseException e) { + reporter.error(new Reporter.Place(source.name(), e.line(), e.column()) + ": error: " + + e.description()); + result = ExitCode.ANALYSIS; + continue; + } + boolean changed = !formatted.equals(source.text()); + changes |= changed; + if (check) { + if (changed) { + out.text(source.name() + "\n"); + } + } else if (write) { + if (changed) { + rewrite(invocation.directory().resolve(source.name()), formatted); + } + } else { + out.text(formatted); + } + } + if (result == ExitCode.SUCCESS && check && changes) { + result = ExitCode.ASSERTION; + } + return result.status(); + } + + /** The scripts named, or standard input. */ + private List sources(Invocation invocation) { + if (files.isEmpty() && invocation.host().stdinIsTerminal()) { + throw new CommandFailure(ExitCode.USAGE, null); + } + return ScriptSource.resolve(List.of(), files.isEmpty() ? List.of("-") : files, + invocation.host(), invocation.directory(), false); + } + + private static void rewrite(Path file, String text) { + try { + Files.writeString(file, text, StandardCharsets.UTF_8); + } catch (NoSuchFileException e) { + throw new CommandFailure(ExitCode.USAGE, file + ": no such file"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/FormatOptions.java b/src/main/java/com/darkcollective/relix/cli/command/FormatOptions.java new file mode 100644 index 0000000..13b1bf4 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/FormatOptions.java @@ -0,0 +1,103 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.config.Relixrc; +import com.darkcollective.relix.cli.io.Host; +import com.darkcollective.relix.cli.render.OutputFormat; +import picocli.CommandLine.Option; + +import java.util.Locale; +import java.util.Optional; + +/** + * The shape rows are written in (design §3.3): the format and its modifiers. Every command + * that prints rows takes these, the catalog's listings as well as a run's results. + */ +final class FormatOptions { + + static final String OUTPUT_VARIABLE = "RELIX_OUTPUT"; + static final String NO_COLOR_VARIABLE = "NO_COLOR"; + + @Option(names = {"-o", "--output"}, paramLabel = "FORMAT", + description = { + "The rows' format: table, tsv, csv, ndjson, json or markdown", + "(default: $RELIX_OUTPUT, else relixrc's, else table on a terminal, tsv in a pipe)."}) + String format; + + @Option(names = "--no-header", + description = "Leave out the header row of csv and tsv.") + boolean noHeader; + + @Option(names = "--null", paramLabel = "STRING", + description = "How NULL is written (default: empty in csv, tsv and markdown; NULL in a table).") + String nullText; + + @Option(names = "--color", paramLabel = "WHEN", defaultValue = "auto", + description = { + "Colour a table: auto, always or never (default: ${DEFAULT-VALUE}).", + "auto colours on a terminal unless $NO_COLOR is set."}) + String color; + + /** + * The format these options and the host settle on. + * + * @param host the process, for its environment and whether standard output is a terminal + * @param fallback the format a {@code relixrc} names, when one does + * @return the format and its modifiers + * @throws CommandFailure with {@link ExitCode#USAGE} for an unknown format or colour + */ + Format settle(Host host, Optional fallback) { + OutputFormat chosen = format(host, fallback); + boolean colored = switch (color.toLowerCase(Locale.ROOT)) { + case "always" -> true; + case "never" -> false; + case "auto" -> host.stdoutIsTerminal() && host.variable(NO_COLOR_VARIABLE).isEmpty(); + default -> throw new CommandFailure(ExitCode.USAGE, + "--color: '" + color + "' is not one of auto, always, never"); + }; + return new Format(chosen, new OutputFormat.Options(!noHeader, nullText, colored, false)); + } + + private OutputFormat format(Host host, Optional fallback) { + if (format != null) { + return format("-o", format); + } + return host.variable(OUTPUT_VARIABLE) + .map(name -> format("$" + OUTPUT_VARIABLE, name)) + .or(() -> fallback.map(name -> format(Relixrc.FILE_NAME + " " + Relixrc.OUTPUT, name))) + .orElse(host.stdoutIsTerminal() ? OutputFormat.TABLE : OutputFormat.TSV); + } + + private static OutputFormat format(String from, String name) { + try { + return OutputFormat.fromString(name.strip()); + } catch (IllegalArgumentException e) { + throw new CommandFailure(ExitCode.USAGE, from + ": " + e.getMessage()); + } + } + + /** + * Rows' format, settled. + * + * @param format the format + * @param options its modifiers + */ + record Format(OutputFormat format, OutputFormat.Options options) { + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/GlobalOptions.java b/src/main/java/com/darkcollective/relix/cli/command/GlobalOptions.java index 7acbeba..1974246 100644 --- a/src/main/java/com/darkcollective/relix/cli/command/GlobalOptions.java +++ b/src/main/java/com/darkcollective/relix/cli/command/GlobalOptions.java @@ -20,7 +20,9 @@ import java.nio.file.Path; import java.time.Duration; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; /** @@ -45,6 +47,22 @@ final class GlobalOptions { "Not for secrets: it lands in shell history and ps."}) Map defines = new LinkedHashMap<>(); + @Option(names = {"-a", "--arg"}, paramLabel = "NAME=VALUE", scope = ScopeType.INHERIT, + description = { + "Bind the query parameter $$NAME to VALUE. Repeatable.", + "The value is bound, never pasted into the script, so it cannot change the query."}) + Map arguments = new LinkedHashMap<>(); + + @Option(names = {"-N", "--no-catalog"}, scope = ScopeType.INHERIT, + description = "Ignore the .relix/ directories' catalogs and relixrc; --catalog files still load.") + boolean noCatalog; + + @Option(names = "--catalog", paramLabel = "FILE", scope = ScopeType.INHERIT, + description = { + "Add a catalog file, nearer than the .relix/ directories'. Repeatable.", + "$RELIX_CATALOG_PATH adds files too, separated as in PATH."}) + List catalogs = new ArrayList<>(); + @Option(names = {"-q", "--quiet"}, scope = ScopeType.INHERIT, description = "Print errors only, not warnings.") boolean quiet; diff --git a/src/main/java/com/darkcollective/relix/cli/command/InputOptions.java b/src/main/java/com/darkcollective/relix/cli/command/InputOptions.java new file mode 100644 index 0000000..0947223 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/InputOptions.java @@ -0,0 +1,90 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.io.InputBinding; +import com.darkcollective.relix.embed.Input; +import picocli.CommandLine.Option; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The data a run's scripts read besides their own declarations (design §3.2): relations + * bound to files, URLs or standard input. + */ +final class InputOptions { + + @Option(names = {"-i", "--input"}, paramLabel = "NAME=[FORMAT:]LOCATION", + description = { + "Bind the relation NAME to a file, an http(s) URL (with --remote), or - for", + "standard input. FORMAT is csv, tsv, json or ndjson; without it the", + "extension decides, and standard input must name one. Repeatable."}) + List inputs = new ArrayList<>(); + + @Option(names = "--schema", paramLabel = "NAME={...}", + description = "The heading of the input NAME, such as NAME='{ id: NUMBER, name: STRING }', instead of inferring one.") + Map schemas = new LinkedHashMap<>(); + + @Option(names = "--infer-rows", paramLabel = "N", + description = "Infer an input's heading from its first N records (default: ${DEFAULT-VALUE}).") + int inferRows = Input.DEFAULT_SAMPLE; + + @Option(names = "--unbounded", paramLabel = "NAME", + description = { + "The input NAME never ends, as tail -f's does: its rows stream as they arrive,", + "and a query that would have to read all of it first is refused. Repeatable."}) + List unbounded = new ArrayList<>(); + + /** + * The inputs these options bind. + * + * @param directory where relative paths start from + * @param remote whether {@code --remote} permits a URL + * @return the bindings + */ + Inputs settle(Path directory, boolean remote) { + if (inferRows < 1) { + throw new CommandFailure(ExitCode.USAGE, "--infer-rows must be a positive number"); + } + return new Inputs(InputBinding.resolve(inputs, schemas, unbounded, directory, remote), + inferRows, List.copyOf(unbounded)); + } + + /** + * A run's inputs, settled. + * + * @param bindings one per {@code -i}, in order + * @param sample how many records a heading is inferred from + * @param unbounded the names of the inputs that never end + */ + record Inputs(List bindings, int sample, List unbounded) { + + /** + * Whether an input reads standard input, which the script then cannot. + * + * @return {@code true} when one binding is {@code -} + */ + boolean readStdin() { + return bindings.stream().anyMatch(InputBinding::isStdin); + } + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/Invocation.java b/src/main/java/com/darkcollective/relix/cli/command/Invocation.java index 8caaa0f..d3e80ac 100644 --- a/src/main/java/com/darkcollective/relix/cli/command/Invocation.java +++ b/src/main/java/com/darkcollective/relix/cli/command/Invocation.java @@ -17,9 +17,12 @@ import com.darkcollective.relix.cli.CommandFailure; import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.catalog.Catalog; +import com.darkcollective.relix.cli.catalog.Trust; import com.darkcollective.relix.cli.config.Placeholders; import com.darkcollective.relix.cli.config.Profiles; import com.darkcollective.relix.cli.config.RelixDirectories; +import com.darkcollective.relix.cli.config.Relixrc; import com.darkcollective.relix.cli.drivers.DownloadProgress; import com.darkcollective.relix.cli.io.FileSystemScriptLoader; import com.darkcollective.relix.cli.io.Host; @@ -30,6 +33,7 @@ import com.darkcollective.relix.embed.Sandbox; import com.darkcollective.relix.processor.connector.ConnectorProvisioner; +import java.io.File; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -38,6 +42,8 @@ import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Function; @@ -48,22 +54,40 @@ * *

Everything that can be wrong with the options is found here, before any script is * read, so a bad {@code --now} or an unknown profile fails the run with a usage error - * rather than part-way through. + * rather than part-way through. The catalog is read when a command first asks for it. */ final class Invocation { static final String PROFILE_VARIABLE = "RELIX_PROFILE"; static final String NOW_VARIABLE = "RELIX_NOW"; + static final String CATALOG_PATH_VARIABLE = "RELIX_CATALOG_PATH"; private final Host host; private final GlobalOptions options; private final Reporter reporter; private final Path directory; + private final List levels; + private final Trust trust; + private final Relixrc relixrc; + private final List catalogFiles; + private Catalog catalog; private final Function> placeholders; private final Clock clock; private final Sandbox sandbox; Invocation(Host host, GlobalOptions options) { + this(host, options, true); + } + + /** + * A run, settled. + * + * @param host the process + * @param options its global options + * @param announce whether to say which untrusted directories are skipped; not when + * the command is the one that trusts them + */ + Invocation(Host host, GlobalOptions options, boolean announce) { this.host = host; this.options = options; this.reporter = new Reporter(host.err(), verbosity(options)); @@ -73,10 +97,25 @@ final class Invocation { requirePositive("--max-materialized-rows", options.maxMaterializedRows); requirePositive("--max-processed-rows", options.maxProcessedRows); this.sandbox = options.sandbox == null ? null : sandbox(directory.resolve(options.sandbox)); - String profile = options.profile != null ? options.profile : host.variable(PROFILE_VARIABLE).orElse(null); + this.levels = RelixDirectories.discover(directory, host.home()); + this.trust = Trust.load(host.home(), trustAll(host)); + List trusted = levels.stream().filter(trust::trusts).toList(); + this.relixrc = options.noCatalog ? Relixrc.none() : Relixrc.read(trusted, reporter::warning); + this.catalogFiles = catalogFiles(host, options, directory); + String profile = options.profile != null ? options.profile + : host.variable(PROFILE_VARIABLE).or(() -> relixrc.get(Relixrc.PROFILE)).orElse(null); + if (announce) { + for (Path level : levels) { + if (!trust.trusts(level) && wouldRead(level, options.noCatalog, profile != null)) { + Path project = level.getParent(); + reporter.warning("skipping " + level + ", which is not trusted; to load it: " + + "relix catalog trust " + shellWord(project.toString())); + } + } + } Map values = profile == null ? Map.of() - : Profiles.select(profile, RelixDirectories.discover(directory, host.home())); + : Profiles.select(profile, trusted); this.placeholders = Placeholders.resolver(options.defines, values, host.environment()); } @@ -88,6 +127,56 @@ Reporter reporter() { return reporter; } + /** The defaults the {@code relixrc} files set; none with {@code -N}. */ + Relixrc relixrc() { + return relixrc; + } + + /** + * The catalog: the {@code .relix/} directories' files, unless {@code -N}, and then the + * {@code $RELIX_CATALOG_PATH} and {@code --catalog} files. Read on the first call. + * + * @return the catalog + */ + Catalog catalog() { + if (catalog == null) { + List discovered = options.noCatalog ? List.of() : levels; + catalog = discovered.isEmpty() && catalogFiles.isEmpty() + ? Catalog.empty() + : Catalog.load(discovered, catalogFiles, trust); + for (var file : catalog.files()) { + reporter.notice("catalog " + file.path()); + } + } + return catalog; + } + + /** + * The format a command's rows are written in: its options, else {@code $RELIX_OUTPUT}, + * else {@code relixrc}'s, else the terminal's. + * + * @param formatting the command's format options + * @return the format, settled + */ + FormatOptions.Format format(FormatOptions formatting) { + return formatting.settle(host, relixrc.get(Relixrc.OUTPUT)); + } + + /** What the user trusts. */ + Trust trust() { + return trust; + } + + /** The {@code .relix/} directories found from where the run starts, outermost first. */ + List levels() { + return levels; + } + + /** Whether {@code --remote} permits http(s) locations. */ + boolean remote() { + return options.remote; + } + /** The directory relative paths start from: the working directory, or {@code -C}. */ Path directory() { return directory; @@ -121,6 +210,8 @@ Relix.Builder session(Path base) { // script still runs: a command line over someone else's database must. .allowUnresolved() .placeholders(placeholders) + // Text, which the engine reads as the type each parameter was analysed to. + .parameters(options.arguments) .remoteFiles(options.remote); if (clock != null) { builder.clock(clock); @@ -168,6 +259,60 @@ private static Path directory(Host host, GlobalOptions options) { return dir; } + private static boolean trustAll(Host host) { + return host.variable(Trust.TRUST_ALL_VARIABLE) + .map(v -> v.equals("1") || v.equalsIgnoreCase("true")) + .orElse(false); + } + + /** Whether a run would read anything from a {@code .relix/} directory, were it trusted. */ + private static boolean wouldRead(Path level, boolean noCatalog, boolean profile) { + if (profile && Files.isRegularFile(level.resolve(Profiles.FILE_NAME))) { + return true; + } + return !noCatalog && (!Catalog.filesOf(level).isEmpty() + || Files.isRegularFile(level.resolve(Relixrc.FILE_NAME))); + } + + /** + * A path as a shell word: quoted when it holds anything a shell would split or expand. + * On Windows a backslash is the separator, not an escape, so it needs no quoting there. + */ + static String shellWord(String path) { + String plain = File.separatorChar == '\\' ? "[A-Za-z0-9_@%+=:,./~\\\\-]+" : "[A-Za-z0-9_@%+=:,./~-]+"; + if (path.matches(plain)) { + return path; + } + return "'" + path.replace("'", "'\\''") + "'"; + } + + /** + * The catalog files named outside the directory tree: {@code $RELIX_CATALOG_PATH}'s, + * then {@code --catalog}'s, the nearer last. + */ + private static List catalogFiles(Host host, GlobalOptions options, Path directory) { + List files = new ArrayList<>(); + host.variable(CATALOG_PATH_VARIABLE).ifPresent(path -> { + for (String entry : path.split(File.pathSeparator)) { + if (!entry.isBlank()) { + files.add(catalogFile("$" + CATALOG_PATH_VARIABLE, directory, Path.of(entry.strip()))); + } + } + }); + for (Path file : options.catalogs) { + files.add(catalogFile("--catalog", directory, file)); + } + return List.copyOf(files); + } + + private static Path catalogFile(String from, Path directory, Path file) { + Path resolved = directory.resolve(file).normalize(); + if (!Files.isRegularFile(resolved)) { + throw new CommandFailure(ExitCode.USAGE, from + ": " + file + ": no such file"); + } + return resolved; + } + private static Clock clock(String instant) { if (instant == null || instant.isBlank()) { return null; diff --git a/src/main/java/com/darkcollective/relix/cli/command/IrCommand.java b/src/main/java/com/darkcollective/relix/cli/command/IrCommand.java new file mode 100644 index 0000000..a438f98 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/IrCommand.java @@ -0,0 +1,47 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.io.RowSink; +import com.darkcollective.relix.embed.Relation; +import picocli.CommandLine.Command; + +import java.util.List; + +/** + * {@code relix ir}: the IR report of each script, the symbols, views and queries the + * analyser made of it, with their headings (design §3). + */ +@Command(name = "ir", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = "Prints the IR report: the symbols, views and queries the analyser made of each script.") +final class IrCommand extends ScriptCommand { + + @Override + ScriptRunner.View view(Invocation invocation) { + RowSink out = stdout(invocation); + return script -> { + // A relation's model is the whole script, its queries included; the session's + // is its declarations alone. + List queries = script.queries(); + out.text((queries.isEmpty() ? script.session().model() : queries.getFirst().model()).ir()); + return ExitCode.SUCCESS; + }; + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/Listing.java b/src/main/java/com/darkcollective/relix/cli/command/Listing.java new file mode 100644 index 0000000..c9a1516 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/Listing.java @@ -0,0 +1,89 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.io.RowSink; +import com.darkcollective.relix.processor.Row; +import com.darkcollective.relix.symbol.ColumnDefinition; +import com.darkcollective.relix.symbol.ScalarType; +import com.darkcollective.relix.symbol.Schema; +import com.darkcollective.relix.value.BooleanValue; +import com.darkcollective.relix.value.NullValue; +import com.darkcollective.relix.value.NumberValue; +import com.darkcollective.relix.value.StringValue; +import com.darkcollective.relix.value.Value; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; + +/** + * Rows a command makes itself, such as a catalog listing, written through the same + * renderers as a query's, so that they come out as a table on a terminal and as tsv or + * ndjson in a pipe. + */ +final class Listing { + + private final Schema schema; + private final List rows = new ArrayList<>(); + + /** + * A listing with the given heading. + * + * @param columns each column's name and type, in turn + */ + Listing(Object... columns) { + List definitions = new ArrayList<>(); + for (int i = 0; i < columns.length; i += 2) { + definitions.add(new ColumnDefinition((String) columns[i], (ScalarType) columns[i + 1])); + } + this.schema = new Schema(definitions); + } + + /** + * Adds a row: a {@link String}, a {@link Number}, a {@link Boolean} or {@code null} per + * column. + * + * @param values the row's values, in column order + * @return this + */ + Listing add(Object... values) { + List row = new ArrayList<>(values.length); + for (Object value : values) { + row.add(switch (value) { + case null -> NullValue.INSTANCE; + case String s -> new StringValue(s); + case Boolean b -> new BooleanValue(b); + case Number n -> new NumberValue(new BigDecimal(n.toString())); + default -> throw new IllegalArgumentException("not a listing value: " + value); + }); + } + rows.add(Row.of(schema, row)); + return this; + } + + /** + * Writes the listing to standard output. + * + * @param invocation the run, for its standard output + * @param format the format to write it in + * @param label what the listing is, for a format that names its result + */ + void print(Invocation invocation, FormatOptions.Format format, String label) { + new RowSink(invocation.host().out(), false) + .write(rows.stream(), format.format().encoder(label, schema, format.options())); + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/OptimizeCommand.java b/src/main/java/com/darkcollective/relix/cli/command/OptimizeCommand.java new file mode 100644 index 0000000..41811b9 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/OptimizeCommand.java @@ -0,0 +1,104 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.io.RowSink; +import com.darkcollective.relix.cli.io.ScriptText; +import com.darkcollective.relix.cli.report.OptimizationReport; +import com.darkcollective.relix.embed.Relation; +import com.darkcollective.relix.lang.ast.QueryStatement; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.util.ArrayList; +import java.util.List; + +/** + * {@code relix optimize}: the script, with each query rewritten as the engine's optimiser + * rewrites it, so that it works as a filter: {@code relix optimize slow.relix > fast.relix} + * (design §3). + * + *

Only the {@code query} statements change, each to {@code query { … };} around its + * rewritten expression, in which the views it reads are inlined. Every other statement, + * and every comment, is left as written. The engine writes every rewrite back as Relix + * that parses (DarkCollective/relix-core#100), so the output is a script that runs. With + * {@code --report}, what is printed instead is + * which rules fired, and each query before and after. + */ +@Command(name = "optimize", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Prints the script with each query rewritten by the optimiser, so that it works", + "as a filter: relix optimize slow.relix > fast.relix"}) +final class OptimizeCommand extends ScriptCommand { + + @Option(names = "--report", + description = "Print the rules that fired and each query before and after, instead.") + boolean report; + + @Override + ScriptRunner.View view(Invocation invocation) { + RowSink out = stdout(invocation); + return script -> { + List queries = script.queries(); + if (report) { + out.text(OptimizationReport.generate(queries.isEmpty() + ? script.session().model() : queries.getFirst().model(), entries(queries))); + } else { + out.text(rewritten(script, queries)); + } + return ExitCode.SUCCESS; + }; + } + + private static List entries(List queries) { + List entries = new ArrayList<>(); + for (int i = 0; i < queries.size(); i++) { + Relation written = queries.get(i); + Relation optimized = written.optimized(); + entries.add(new OptimizationReport.Entry(written.label().orElse(""), + written.node(), optimized.node(), optimized.rewrites())); + } + return entries; + } + + /** The text with each query statement replaced by its rewritten query, in order. */ + private static String rewritten(ScriptRunner.Script script, List queries) { + String text = script.source().text(); + List statements = ScriptText.of(text).statements().stream() + .filter(piece -> piece.statement() instanceof QueryStatement) + .toList(); + if (statements.size() != queries.size()) { + throw new CommandFailure(ExitCode.INTERNAL, "the script has " + statements.size() + + " query statements, but the engine returned " + queries.size() + " queries"); + } + StringBuilder out = new StringBuilder(); + int at = 0; + for (int i = 0; i < statements.size(); i++) { + ScriptText.Span span = statements.get(i).span(); + out.append(text, at, span.start()); + out.append("query { ").append(queries.get(i).optimized().render()).append(" };"); + at = span.end(); + } + out.append(text, at, text.length()); + return out.toString(); + } + +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/OutputOptions.java b/src/main/java/com/darkcollective/relix/cli/command/OutputOptions.java new file mode 100644 index 0000000..b1a3dec --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/OutputOptions.java @@ -0,0 +1,126 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.io.Host; +import com.darkcollective.relix.cli.render.OutputFormat; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; + +import java.util.Optional; + +/** + * Where a run's rows go and in what shape (design §3.3, §3.4, §3.6): the format, its + * modifiers, which queries a machine format prints, and the assertions on what they + * return. + */ +final class OutputOptions { + + @Mixin + FormatOptions formatting = new FormatOptions(); + + @Option(names = "--line-buffered", + description = "Flush each row as it is written, for a reader that wants it at once.") + boolean lineBuffered; + + @Option(names = "--query", paramLabel = "NAME", + description = { + "Print only the query NAME. A machine format (tsv, csv, ndjson, json) prints", + "one query: the script's last, unless this names another."}) + String query; + + @Option(names = "--all", + description = "With ndjson, print every query, each row tagged \"_query\": NAME.") + boolean all; + + @Option(names = "--fail-empty", + description = "Exit 1 when the queries printed return no rows.") + boolean failEmpty; + + @Option(names = "--fail-rows", + description = "Exit 1 when the queries printed return any row.") + boolean failRows; + + /** + * The output these options and the host settle on. + * + * @param host the process, for its environment and whether standard output is a terminal + * @param fallback the format a {@code relixrc} names, when one does + * @return the settled output + * @throws CommandFailure with {@link ExitCode#USAGE} for an unknown format or colour, + * or options that cannot go together + */ + Output settle(Host host, Optional fallback) { + FormatOptions.Format settled = formatting.settle(host, fallback); + OutputFormat chosen = settled.format(); + if (all && query != null) { + throw new CommandFailure(ExitCode.USAGE, "--all and --query cannot be given together"); + } + if (all && chosen.isMachineFormat() && chosen != OutputFormat.NDJSON) { + throw new CommandFailure(ExitCode.USAGE, "--all prints several queries, which " + + chosen.optionName() + " cannot hold; use -o ndjson, or --query=NAME for one"); + } + if (failEmpty && failRows) { + throw new CommandFailure(ExitCode.USAGE, "--fail-empty and --fail-rows cannot be given together"); + } + Assertion assertion = failEmpty ? Assertion.FAIL_EMPTY : failRows ? Assertion.FAIL_ROWS : Assertion.NONE; + OutputFormat.Options options = settled.options(); + return new Output(chosen, + new OutputFormat.Options(options.header(), options.nullText(), options.color(), + all && chosen == OutputFormat.NDJSON), + lineBuffered, query, all || !chosen.isMachineFormat(), assertion); + } + + /** What the rows printed must be for the run to succeed. */ + enum Assertion { + /** Nothing is asserted. */ + NONE, + /** {@code --fail-empty}: at least one row. */ + FAIL_EMPTY, + /** {@code --fail-rows}: no row at all. */ + FAIL_ROWS; + + /** + * Whether {@code rows} rows printed fail this assertion. + * + * @param rows how many rows were printed in all + * @return {@code true} when the run should exit 1 + */ + boolean failedBy(long rows) { + return switch (this) { + case NONE -> false; + case FAIL_EMPTY -> rows == 0; + case FAIL_ROWS -> rows > 0; + }; + } + } + + /** + * A run's output, settled. + * + * @param format the format + * @param options its modifiers + * @param lineBuffered whether each row is flushed as it is written + * @param query the one query to print, or {@code null} + * @param every whether every query is printed, rather than the last + * @param assertion what the printed rows must be + */ + record Output(OutputFormat format, OutputFormat.Options options, boolean lineBuffered, + String query, boolean every, Assertion assertion) { + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/ProvenanceCommand.java b/src/main/java/com/darkcollective/relix/cli/command/ProvenanceCommand.java new file mode 100644 index 0000000..069fa93 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/ProvenanceCommand.java @@ -0,0 +1,112 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.io.RowSink; +import com.darkcollective.relix.cli.report.ProvenanceOutput; +import com.darkcollective.relix.embed.Relation; +import com.darkcollective.relix.processor.provenance.AnnotatedRelation; +import com.darkcollective.relix.provenance.Semiring; +import com.darkcollective.relix.provenance.Semirings; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Locale; + +/** + * {@code relix provenance}: each query's rows, each annotated with where it came from in a + * semiring of the user's choice (design §3). + * + *

The annotation is never a data column. On a terminal it is a trailing {@code [prov]} + * column under a footer naming the semiring; in a pipe, or with {@code -o json}, it is a + * JSON document in which each tuple carries its {@code row} and its {@code provenance} + * under separate keys. + */ +@Command(name = "provenance", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Runs each query and prints its rows, each annotated with where it came from", + "in a provenance semiring."}) +final class ProvenanceCommand extends ScriptCommand { + + @Option(names = "--semiring", paramLabel = "NAME", defaultValue = "counting", + description = { + "The semiring: boolean, counting, tropical, security, lineage, cheapest-route,", + "or one an installed library offers (default: ${DEFAULT-VALUE})."}) + String semiring; + + @Option(names = "--weight", paramLabel = "COLUMN", + description = { + "The column a weighted closure takes each edge's weight from, such as", + "--semiring tropical --weight cost for shortest paths."}) + String weight; + + @Option(names = {"-o", "--output"}, paramLabel = "FORMAT", + description = "table or json (default: table on a terminal, json in a pipe).") + String format; + + @Option(names = "--query", paramLabel = "NAME", + description = "Annotate only the query NAME.") + String query; + + @Override + ScriptRunner.View view(Invocation invocation) { + Semiring chosen = Semirings.byName(semiring).orElseThrow(() -> new CommandFailure(ExitCode.USAGE, + "--semiring: '" + semiring + "' is not one of " + String.join(", ", Semirings.names()))); + boolean json = json(invocation); + RowSink out = stdout(invocation); + return script -> { + StringWriter text = new StringWriter(); + PrintWriter writer = new PrintWriter(text); + for (Relation relation : script.queries(query)) { + write(writer, json, relation, chosen); + } + writer.flush(); + out.text(text.toString()); + return ExitCode.SUCCESS; + }; + } + + private boolean json(Invocation invocation) { + if (format == null) { + return !invocation.host().stdoutIsTerminal(); + } + return switch (format.strip().toLowerCase(Locale.ROOT)) { + case "table" -> false; + case "json" -> true; + default -> throw new CommandFailure(ExitCode.USAGE, "-o: '" + format + "' is not one of table, json"); + }; + } + + /** Captures the semiring's annotation type. */ + private void write(PrintWriter out, boolean json, Relation relation, Semiring semiring) { + String label = relation.label().orElse(""); + AnnotatedRelation annotated = weight == null + ? relation.provenance(semiring) + : relation.provenance(semiring, weight); + if (json) { + ProvenanceOutput.writeJson(out, label, annotated, this.semiring); + } else { + ProvenanceOutput.writeHuman(out, label, annotated, this.semiring); + } + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/RelixCommand.java b/src/main/java/com/darkcollective/relix/cli/command/RelixCommand.java index a8926a8..6fa99a8 100644 --- a/src/main/java/com/darkcollective/relix/cli/command/RelixCommand.java +++ b/src/main/java/com/darkcollective/relix/cli/command/RelixCommand.java @@ -16,12 +16,17 @@ package com.darkcollective.relix.cli.command; import com.darkcollective.relix.cli.Main; +import com.darkcollective.relix.cli.config.Relixrc; import com.darkcollective.relix.cli.io.Host; +import com.darkcollective.relix.cli.io.Interruption; import com.darkcollective.relix.cli.io.ScriptSource; +import com.darkcollective.relix.cli.report.EventFeed; import picocli.CommandLine.Command; +import picocli.CommandLine.HelpCommand; import picocli.CommandLine.IVersionProvider; import picocli.CommandLine.Mixin; +import java.util.List; import java.util.Objects; import java.util.concurrent.Callable; @@ -40,14 +45,19 @@ "Runs Relix relational-algebra scripts as a stage of a Unix pipeline.", "Rows go to standard output; everything else goes to standard error.", ""}, - subcommands = RunCommand.class, + subcommands = { + RunCommand.class, CheckCommand.class, ExplainCommand.class, OptimizeCommand.class, + TraceCommand.class, BundleCommand.class, IrCommand.class, ProvenanceCommand.class, + FmtCommand.class, CatalogCommand.class, DriversCommand.class, ConnectorsCommand.class, + DocCommand.class, HelpCommand.class, VersionCommand.class, CompletionCommand.class}, exitCodeListHeading = "%nExit status:%n", exitCodeList = { " 0:success", + " 1:an assertion failed (--fail-empty, --fail-rows)", " 2:usage error (bad option, nothing to run)", " 3:the script did not parse or analyse", " 4:execution failed (a data error, a limit hit)", - " 5:environment (a missing driver, a loose profiles.json, a connection refused)", + " 5:environment (a missing driver, an untrusted catalog, a connection refused)", "70:internal error in relix", "130:interrupted (SIGINT)", "141:standard output closed (SIGPIPE)"}) @@ -59,33 +69,97 @@ public final class RelixCommand implements Callable { @Mixin ScriptOptions scripts; + @Mixin + InputOptions inputs; + + @Mixin + OutputOptions output; + + @Mixin + TraceOption trace; + private final Host host; + private final Interruption interruption; /** * The root command over a host. * - * @param host the process it runs in + * @param host the process it runs in + * @param interruption what an interrupt stops */ - public RelixCommand(Host host) { + public RelixCommand(Host host, Interruption interruption) { this.host = Objects.requireNonNull(host, "host"); + this.interruption = Objects.requireNonNull(interruption, "interruption"); } @Override public Integer call() { - return run(scripts); + return run(scripts, inputs, output, trace); } /** * Runs the scripts {@code scripts} names, with this invocation's global options. * * @param scripts where the scripts come from + * @param inputs the data bound to relation names + * @param output where their rows go + * @param trace where their event feed goes * @return the exit status */ - int run(ScriptOptions scripts) { - Invocation invocation = new Invocation(host, options); - return new ScriptRunner(invocation) - .run(ScriptSource.resolve(scripts.expressions, scripts.files, host, invocation.directory())) - .status(); + int run(ScriptOptions scripts, InputOptions inputs, OutputOptions output, TraceOption trace) { + Invocation invocation = invocation(true); + OutputOptions.Output settled = output.settle(host, invocation.relixrc().get(Relixrc.OUTPUT)); + InputOptions.Inputs bound = inputs.settle(invocation.directory(), invocation.remote()); + List sources = sources(invocation, scripts, bound); + try (EventFeed feed = trace.open(host, invocation.directory())) { + return runner(invocation, bound) + .run(sources, new ResultRows(invocation, settled, interruption, feed)) + .status(); + } + } + + /** + * The scripts a command reads. + * + * @param invocation the run + * @param scripts where they come from + * @param inputs the data bound to relation names, which may take standard input + * @return the scripts, in order + */ + List sources(Invocation invocation, ScriptOptions scripts, InputOptions.Inputs inputs) { + return ScriptSource.resolve(scripts.expressions, scripts.files, host, invocation.directory(), + inputs.readStdin()); + } + + /** + * What takes a command's scripts through the catalog, the inputs and analysis. + * + * @param invocation the run + * @param inputs the data bound to relation names + * @return the runner + */ + ScriptRunner runner(Invocation invocation, InputOptions.Inputs inputs) { + return new ScriptRunner(invocation, inputs, interruption); + } + + /** The process the command runs in. */ + Host host() { + return host; + } + + /** What an interrupt stops. */ + Interruption interruption() { + return interruption; + } + + /** + * This invocation's global options, settled. + * + * @param announce whether to say which untrusted directories are skipped + * @return the run + */ + Invocation invocation(boolean announce) { + return new Invocation(host, options, announce); } /** The command's and the engine's versions, for {@code --version}. */ diff --git a/src/main/java/com/darkcollective/relix/cli/command/ResultRows.java b/src/main/java/com/darkcollective/relix/cli/command/ResultRows.java new file mode 100644 index 0000000..49499c5 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/ResultRows.java @@ -0,0 +1,141 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.io.Interruption; +import com.darkcollective.relix.cli.io.Reporter; +import com.darkcollective.relix.cli.io.RowSink; +import com.darkcollective.relix.cli.report.EventFeed; +import com.darkcollective.relix.embed.Relation; +import com.darkcollective.relix.embed.Rows; +import com.darkcollective.relix.embed.Tuple; +import com.darkcollective.relix.events.QueryEvent; + +import java.time.Duration; +import java.util.List; +import java.util.stream.Stream; + +/** + * What {@code relix run} shows of a script: its queries' rows, on standard output. + * + *

A table or Markdown prints every query, each under its label. A machine format + * prints one result set, because a CSV with two headers is not a CSV: the script's last + * query, or the one {@code --query} names; {@code --all} with ndjson prints every one, + * tagged. A query not printed is not run. + * + *

With {@code --trace}, each query's event feed is written beside its rows, to standard + * error or a file, as the query runs. + */ +final class ResultRows implements ScriptRunner.View { + + private static final String TRUNCATED = "TRUNCATED"; + + private final OutputOptions.Output output; + private final Interruption interruption; + private final Reporter reporter; + private final RowSink sink; + private final EventFeed trace; + private long printed; + + /** + * The rows view. + * + * @param invocation the run + * @param output where the rows go and in what shape + * @param interruption what an interrupt stops + * @param trace where the event feed goes, or {@code null} for nowhere + */ + ResultRows(Invocation invocation, OutputOptions.Output output, Interruption interruption, EventFeed trace) { + this.output = output; + this.interruption = interruption; + this.reporter = invocation.reporter(); + this.sink = new RowSink(invocation.host().out(), output.lineBuffered()); + this.trace = trace; + } + + @Override + public ExitCode show(ScriptRunner.Script script) { + for (Relation relation : chosen(script)) { + print(script, relation); + } + return ExitCode.SUCCESS; + } + + @Override + public ExitCode finish(ExitCode result) { + if (result == ExitCode.SUCCESS && output.assertion().failedBy(printed)) { + return ExitCode.ASSERTION; + } + return result; + } + + /** The queries to print, in order. */ + private List chosen(ScriptRunner.Script script) { + List queries = script.queries(output.query()); + if (output.query() != null || output.every() || queries.isEmpty()) { + return queries; + } + return List.of(queries.getLast()); + } + + private void print(ScriptRunner.Script script, Relation relation) { + String name = script.source().name(); + long start = System.nanoTime(); + String label = relation.label().orElse(""); + String query = label.isEmpty() ? "query" : label; + var encoder = output.format().encoder(label, relation.schema(), output.options()); + boolean truncated; + long count; + if (output.format().streams() || trace != null) { + // A traced table is collected from the stream, which is what reports events. + boolean[] cut = {false}; + try (Stream rows = relation.stream(event -> { + cut[0] |= isTruncation(event); + if (trace != null) { + trace.event(event, query); + } + })) { + interruption.stream(rows); + Stream written = output.format().streams() ? rows : rows.toList().stream(); + count = sink.write(written, encoder); + } finally { + interruption.stream(null); + } + truncated = cut[0]; + } else { + // A table aligns its columns over every row, so it collects them first, and a + // result that never ends is refused rather than collected forever. + Rows rows = relation.run(); + count = sink.write(rows.rows().stream(), encoder); + truncated = rows.truncated(); + } + printed += count; + Duration elapsed = Duration.ofNanos(System.nanoTime() - start); + if (trace != null) { + trace.delivered(query, count, elapsed); + } + if (truncated) { + reporter.error(name + ": " + (label.isEmpty() ? "a query" : label) + + ": the sandbox cut the result at " + count + " rows"); + } + reporter.timing(name + ": " + query, elapsed); + } + + private static boolean isTruncation(QueryEvent event) { + return event.stage() == QueryEvent.Stage.EXECUTE && TRUNCATED.equals(event.code()); + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/RunCommand.java b/src/main/java/com/darkcollective/relix/cli/command/RunCommand.java index 9e7fbff..79aba41 100644 --- a/src/main/java/com/darkcollective/relix/cli/command/RunCommand.java +++ b/src/main/java/com/darkcollective/relix/cli/command/RunCommand.java @@ -29,7 +29,7 @@ mixinStandardHelpOptions = true, sortOptions = false, usageHelpAutoWidth = true, - description = "Runs scripts and writes every query's rows to standard output.") + description = "Runs scripts and writes their queries' rows to standard output.") final class RunCommand implements Callable { @ParentCommand @@ -38,8 +38,17 @@ final class RunCommand implements Callable { @Mixin ScriptOptions scripts; + @Mixin + InputOptions inputs; + + @Mixin + OutputOptions output; + + @Mixin + TraceOption trace; + @Override public Integer call() { - return parent.run(scripts); + return parent.run(scripts, inputs, output, trace); } } diff --git a/src/main/java/com/darkcollective/relix/cli/command/ScriptCommand.java b/src/main/java/com/darkcollective/relix/cli/command/ScriptCommand.java new file mode 100644 index 0000000..a8b270d --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/ScriptCommand.java @@ -0,0 +1,70 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.io.RowSink; +import com.darkcollective.relix.cli.io.ScriptSource; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.ParentCommand; + +import java.util.List; +import java.util.concurrent.Callable; + +/** + * A command that shows something of each script it reads other than its rows: a plan, + * a rewrite, a report (design §3). The scripts come and are analysed as {@code run}'s + * do, with the catalog and the {@code -i} inputs; what is shown is the command's + * {@link ScriptRunner.View}. + */ +abstract class ScriptCommand implements Callable { + + @ParentCommand + RelixCommand root; + + @Mixin + ScriptOptions scripts; + + @Mixin + InputOptions inputs; + + @Override + public Integer call() { + Invocation invocation = root.invocation(true); + InputOptions.Inputs bound = inputs.settle(invocation.directory(), invocation.remote()); + ScriptRunner.View view = view(invocation); + List sources = root.sources(invocation, scripts, bound); + return root.runner(invocation, bound).run(sources, view).status(); + } + + /** + * What this command shows of each script. Its options are checked here, before any + * script is read. + * + * @param invocation the run + * @return the view + */ + abstract ScriptRunner.View view(Invocation invocation); + + /** + * Standard output, for a whole text at a time. + * + * @param invocation the run + * @return the sink + */ + static RowSink stdout(Invocation invocation) { + return new RowSink(invocation.host().out(), false); + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/ScriptRunner.java b/src/main/java/com/darkcollective/relix/cli/command/ScriptRunner.java index 42e7c87..bb68ae8 100644 --- a/src/main/java/com/darkcollective/relix/cli/command/ScriptRunner.java +++ b/src/main/java/com/darkcollective/relix/cli/command/ScriptRunner.java @@ -15,84 +15,271 @@ */ package com.darkcollective.relix.cli.command; +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.EngineMessages; import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.catalog.Declaration; +import com.darkcollective.relix.cli.io.InputBinding; +import com.darkcollective.relix.cli.io.Interruption; import com.darkcollective.relix.cli.io.Reporter; import com.darkcollective.relix.cli.io.ScriptSource; -import com.darkcollective.relix.cli.render.QueryResultFormatter; import com.darkcollective.relix.embed.Diagnostic; import com.darkcollective.relix.embed.Relation; import com.darkcollective.relix.embed.Relix; import com.darkcollective.relix.embed.RelixException; -import com.darkcollective.relix.embed.Rows; -import com.darkcollective.relix.processor.Row; -import java.io.PrintStream; +import java.nio.file.Path; import java.time.Duration; import java.util.List; /** - * Runs scripts, each in a session of its own, and prints every query's rows. + * Takes scripts through what every command that reads Relix does before it shows anything: + * each in a session of its own, with the catalog, then the inputs, then the script + * analysed. What is then shown of a script is a {@link View}'s: rows for {@code run}, a + * plan for {@code explain}, and so on. * - *

A script is checked before it runs, and every diagnostic is reported, so a script - * with three mistakes reports three; one with an error does not run. Each script is - * independent: a failure in one does not stop the next, and the run exits with the first - * failure's code. + *

Each session first defines the catalog (design §5), then declares the inputs + * {@code -i} binds, then analyses the script, so that each of these can replace a name the + * one before it declared. An input that cannot be read fails the script as execution does. * - *

Output here is the aligned table for every query. Choosing the format, the default in - * a pipe and which queries a machine format prints are the output slice's (S2). + *

Every diagnostic is reported, so a script with three mistakes reports three; one with + * an error is not shown, unless the view shows broken scripts too, as the bundle does. Each + * script is independent: a failure in one does not stop the next, and the run exits with + * the first failure's code. + * + *

A script that fails for want of a name only an untrusted {@code .relix/} declares + * exits 5, the environment's code, with the command that trusts it. + * + *

Two things end the whole run at once, whatever script it is in: standard output + * closing (exit 141) and an interrupt (exit 130). Neither says anything. */ final class ScriptRunner { + /** What a command shows of each script. */ + interface View { + + /** + * Shows one analysed script. + * + * @param script the script, its session and its diagnostics + * @return the script's exit code + */ + ExitCode show(Script script); + + /** + * Whether a script with errors is still shown. The script's exit code is then 3 + * whatever {@link #show} returns. + * + * @return {@code true} for a view that describes a broken script too + */ + default boolean showsBroken() { + return false; + } + + /** + * Whether diagnostics are reported on standard error; not by a view that writes + * them to standard output itself. + * + * @return {@code true} to report them + */ + default boolean reportsDiagnostics() { + return true; + } + + /** + * Ends the run, once every script has been shown. + * + * @param result the exit code of the scripts, the first failure's + * @return the run's exit code + */ + default ExitCode finish(ExitCode result) { + return result; + } + } + + /** + * One script, analysed: its session holds the catalog and the inputs. + * + * @param source where it came from + * @param session its session + * @param diagnostics what analysing it found + */ + record Script(ScriptSource source, Relix session, List diagnostics) { + + /** + * Whether analysing it found an error. + * + * @return {@code true} when a diagnostic is an error + */ + boolean broken() { + return diagnostics.stream().anyMatch(Diagnostic::isError); + } + + /** + * Its queries, in order. + * + * @return one relation per {@code query} statement + */ + List queries() { + return session.script(source.text()); + } + + /** + * The queries a view shows: the one {@code --query} names, or every one. + * + * @param name the query to show, or {@code null} for all + * @return the queries + * @throws CommandFailure with {@link ExitCode#USAGE} when no query has that name + */ + List queries(String name) { + List queries = queries(); + if (name == null) { + return queries; + } + List named = queries.stream() + .filter(q -> q.label().filter(name::equals).isPresent()) + .toList(); + if (named.isEmpty()) { + throw new CommandFailure(ExitCode.USAGE, source.name() + ": no query named '" + name + "'"); + } + return named; + } + } + private final Invocation invocation; + private final InputOptions.Inputs inputs; + private final Interruption interruption; - ScriptRunner(Invocation invocation) { + ScriptRunner(Invocation invocation, InputOptions.Inputs inputs, Interruption interruption) { this.invocation = invocation; + this.inputs = inputs; + this.interruption = interruption; } - ExitCode run(List sources) { + /** + * Shows every script through a view. + * + * @param sources the scripts, in order + * @param view what to show of each + * @return the run's exit code + */ + ExitCode run(List sources, View view) { invocation.loadInstalledDrivers(); ExitCode result = ExitCode.SUCCESS; for (ScriptSource source : sources) { - ExitCode code = run(source); + ExitCode code = run(source, view); if (result == ExitCode.SUCCESS) { result = code; } } - return result; + return view.finish(result); } - private ExitCode run(ScriptSource source) { + private ExitCode run(ScriptSource source, View view) { Reporter reporter = invocation.reporter(); - PrintStream out = invocation.host().out(); try (Relix session = invocation.session(source.directory()).build()) { + interruption.session(session); + if (!defineCatalog(source, session)) { + return ExitCode.ANALYSIS; + } + if (!declareInputs(source, session)) { + return ExitCode.EXECUTION; + } long start = System.nanoTime(); List diagnostics = session.validate(source.text()); reporter.timing(source.name() + ": analysed", Duration.ofNanos(System.nanoTime() - start)); - boolean errors = false; - for (Diagnostic diagnostic : diagnostics) { - reporter.diagnostic(source.place(diagnostic.location()), diagnostic); - errors |= diagnostic.isError(); - } - if (errors) { - return ExitCode.ANALYSIS; + if (view.reportsDiagnostics()) { + for (Diagnostic diagnostic : diagnostics) { + reporter.diagnostic(source.place(diagnostic.location()), diagnostic); + } } - for (Relation relation : session.script(source.text())) { - long queryStart = System.nanoTime(); - Rows rows = relation.run(); - String label = relation.label().orElse(""); - out.print(QueryResultFormatter.format(label, rows.schema(), rows.rows().stream().map(Row.class::cast))); - out.flush(); - if (rows.truncated()) { - reporter.error(source.name() + ": " + (label.isEmpty() ? "a query" : label) - + ": the sandbox cut the result at " + rows.size() + " rows"); + Script script = new Script(source, session, diagnostics); + if (script.broken()) { + ExitCode code = withheld(diagnostics) ? ExitCode.ENVIRONMENT : ExitCode.ANALYSIS; + if (view.showsBroken()) { + view.show(script); } - reporter.timing(source.name() + ": " + (label.isEmpty() ? "query" : label), - Duration.ofNanos(System.nanoTime() - queryStart)); + return code; } - return ExitCode.SUCCESS; + return view.show(script); + } catch (RuntimeException e) { + // Closing the session under a running query fails it in whatever way it fails; + // once interrupted, every such failure is the interrupt's. + if (interruption.interrupted()) { + throw new CommandFailure(ExitCode.INTERRUPTED, null); + } + if (e instanceof CommandFailure failure && failure.exitCode() == ExitCode.USAGE + && failure.getMessage() != null) { + reporter.error(failure.getMessage()); + return ExitCode.USAGE; + } + if (e instanceof RelixException failure) { + reporter.error(source.name() + ": " + EngineMessages.of(failure)); + return ExitCode.of(failure); + } + throw e; + } finally { + interruption.session(null); + } + } + + /** + * Whether a script failed for a name only an untrusted directory declares, saying so: + * then the script is not wrong, the environment is. + * + * @param diagnostics the script's diagnostics, with an error among them + * @return {@code true} when an error names a withheld declaration + */ + private boolean withheld(List diagnostics) { + boolean found = false; + for (Declaration declaration : invocation.catalog().withheld()) { + String quoted = "'" + declaration.name() + "'"; + if (diagnostics.stream().anyMatch(d -> d.isError() && d.message().contains(quoted))) { + Path project = declaration.file().level().getParent(); + invocation.reporter().error(declaration.name() + " is declared in " + + declaration.file().path() + ", which is not trusted; to load it: relix catalog trust " + + Invocation.shellWord(project.toString())); + found = true; + } + } + return found; + } + + /** + * Defines the catalog's declarations on a session, before its inputs and its script, + * reporting why when they do not analyse. + * + * @return whether they were defined + */ + private boolean defineCatalog(ScriptSource source, Relix session) { + try { + invocation.catalog().define(session); + return true; } catch (RelixException e) { - reporter.error(source.name() + ": " + e.getMessage()); - return ExitCode.of(e); + invocation.reporter().error(source.name() + ": the catalog does not analyse: " + e.getMessage()); + return false; + } + } + + /** + * Declares every bound input on a session, reporting the first that cannot be read. + * + * @return whether all were declared + */ + private boolean declareInputs(ScriptSource source, Relix session) { + for (InputBinding binding : inputs.bindings()) { + try { + binding.declare(session, invocation.host().in(), inputs.sample(), + inputs.unbounded().contains(binding.name())); + } catch (RelixException e) { + if (interruption.interrupted()) { + throw new CommandFailure(ExitCode.INTERRUPTED, null); + } + invocation.reporter().error(source.name() + ": -i " + binding.name() + "=" + + binding.location() + ": " + e.getMessage()); + return false; + } } + return true; } } diff --git a/src/main/java/com/darkcollective/relix/cli/command/TraceCommand.java b/src/main/java/com/darkcollective/relix/cli/command/TraceCommand.java new file mode 100644 index 0000000..4ab5392 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/TraceCommand.java @@ -0,0 +1,72 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.io.Interruption; +import com.darkcollective.relix.cli.report.EventFeed; +import com.darkcollective.relix.embed.Relation; +import com.darkcollective.relix.embed.Tuple; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.time.Duration; +import java.util.stream.Stream; + +/** + * {@code relix trace}: runs each query for its event feed, what the engine rewrote, + * planned and ran, and discards its rows (design §3). {@code relix run --trace} writes the + * same feed beside the rows. + * + *

The rows are pulled and dropped one at a time, never collected, so tracing a large + * result costs no more memory than running it. + */ +@Command(name = "trace", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Runs each query and prints its event feed, what the engine rewrote, planned", + "and ran, discarding the rows. relix run --trace keeps the rows too."}) +final class TraceCommand extends ScriptCommand { + + @Option(names = "--query", paramLabel = "NAME", + description = "Trace only the query NAME.") + String query; + + @Override + ScriptRunner.View view(Invocation invocation) { + EventFeed feed = EventFeed.stdout(invocation.host().out()); + Interruption interruption = root.interruption(); + return script -> { + for (Relation relation : script.queries(query)) { + String label = relation.label().orElse("query"); + long start = System.nanoTime(); + long delivered = 0; + try (Stream rows = relation.stream(event -> feed.event(event, label))) { + interruption.stream(rows); + for (var it = rows.iterator(); it.hasNext(); it.next()) { + delivered++; + } + } finally { + interruption.stream(null); + } + feed.delivered(label, delivered, Duration.ofNanos(System.nanoTime() - start)); + } + return ExitCode.SUCCESS; + }; + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/TraceOption.java b/src/main/java/com/darkcollective/relix/cli/command/TraceOption.java new file mode 100644 index 0000000..c382107 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/TraceOption.java @@ -0,0 +1,68 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.io.Host; +import com.darkcollective.relix.cli.report.EventFeed; +import picocli.CommandLine.Option; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * {@code run --trace[=FILE]}: the event feed, written beside the rows (design §4). + * + *

The file is attached with {@code =} only, so that {@code relix --trace q.relix} traces + * {@code q.relix} rather than writing its trace over it; {@code Main} makes a bare + * {@code --trace} an empty value before the line is parsed. + */ +final class TraceOption { + + /** The option's name. */ + static final String NAME = "--trace"; + + @Option(names = NAME, paramLabel = "FILE", + description = { + "Write the event feed (what was rewritten, planned and run) to standard", + "error, or to FILE with --trace=FILE, while the rows go to standard output."}) + String trace; + + /** + * Where the feed goes, if anywhere. + * + * @param host the process, for standard error + * @param directory where a relative FILE is + * @return the feed, or {@code null} without {@code --trace}; a file is opened here, and + * closing the feed closes it + */ + EventFeed open(Host host, Path directory) { + if (trace == null) { + return null; + } + if (trace.isEmpty()) { + return EventFeed.stderr(host.err()); + } + Path file = directory.resolve(trace); + try { + return EventFeed.file(Files.newOutputStream(file)); + } catch (IOException e) { + throw new CommandFailure(ExitCode.USAGE, NAME + "=" + trace + ": " + e.getMessage()); + } + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/TrustCommand.java b/src/main/java/com/darkcollective/relix/cli/command/TrustCommand.java new file mode 100644 index 0000000..98887ce --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/TrustCommand.java @@ -0,0 +1,110 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.catalog.Trust; +import com.darkcollective.relix.cli.config.RelixDirectories; +import com.darkcollective.relix.symbol.ScalarType; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.ParentCommand; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.Callable; + +/** + * {@code relix catalog trust}: records that a project's {@code .relix/} may be loaded, as + * its files are now (design §5.4). + */ +@Command(name = "trust", + mixinStandardHelpOptions = true, + sortOptions = false, + usageHelpAutoWidth = true, + description = { + "Trusts the .relix/ directory of DIR (default: the working directory), as its", + "files are now: a project's catalog, relixrc and profiles.json are loaded only", + "once it is trusted, and changing, adding or removing a file there means trusting", + "it again. ~/.relix, --catalog files and the script itself are always trusted;", + "RELIX_TRUST_ALL=1 trusts everything, for a CI container."}) +final class TrustCommand implements Callable { + + @ParentCommand + CatalogCommand catalog; + + @Parameters(paramLabel = "DIR", arity = "0..1", + description = "The project to trust, or its .relix/ directory.") + Path directory; + + @Option(names = "--list", + description = "List the trusted directories and whether each has changed since.") + boolean list; + + @Option(names = "--revoke", paramLabel = "DIR", + description = "Stop trusting DIR.") + Path revoke; + + @Mixin + FormatOptions formatting; + + @Override + public Integer call() { + if ((list ? 1 : 0) + (revoke != null ? 1 : 0) + (directory != null ? 1 : 0) > 1) { + throw new CommandFailure(ExitCode.USAGE, "give one of DIR, --list and --revoke DIR"); + } + Invocation invocation = catalog.root.invocation(false); + Trust trust = invocation.trust(); + if (list) { + Listing listing = new Listing("directory", ScalarType.STRING, "state", ScalarType.STRING); + for (Path level : trust.directories()) { + listing.add(level.getParent().toString(), trust.state(level).label()); + } + listing.print(invocation, invocation.format(formatting), "trusted"); + return ExitCode.SUCCESS.status(); + } + if (revoke != null) { + Path level = level(invocation.directory().resolve(revoke).normalize()); + if (!trust.revoke(level)) { + throw new CommandFailure(ExitCode.USAGE, revoke + " is not trusted"); + } + invocation.reporter().notice("no longer trusting " + level); + return ExitCode.SUCCESS.status(); + } + Path given = directory == null ? invocation.directory() : invocation.directory().resolve(directory).normalize(); + Path level = level(given); + if (!Files.isDirectory(level)) { + throw new CommandFailure(ExitCode.USAGE, (directory == null ? given : directory) + + " has no " + RelixDirectories.NAME + " directory to trust"); + } + if (level.equals(invocation.host().home().resolve(RelixDirectories.NAME))) { + invocation.reporter().notice(level + " is your own, and always trusted"); + return ExitCode.SUCCESS.status(); + } + trust.trust(level); + invocation.reporter().notice("trusting " + level); + return ExitCode.SUCCESS.status(); + } + + /** The {@code .relix/} directory a path names: itself, or the one inside it. */ + private static Path level(Path path) { + Path name = path.getFileName(); + return name != null && name.toString().equals(RelixDirectories.NAME) ? path : path.resolve(RelixDirectories.NAME); + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/command/VersionCommand.java b/src/main/java/com/darkcollective/relix/cli/command/VersionCommand.java new file mode 100644 index 0000000..1d725fb --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/command/VersionCommand.java @@ -0,0 +1,46 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.ExitCode; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Spec; + +import java.util.concurrent.Callable; + +/** + * {@code relix version}: the command's version and the engine's, printed by the same + * version help as {@code relix --version}, so the two are the same text, line ending + * included. + */ +@Command(name = "version", + mixinStandardHelpOptions = true, + usageHelpAutoWidth = true, + description = "Prints the versions of relix and of the Relix engine it runs.") +final class VersionCommand implements Callable { + + @Spec + CommandSpec spec; + + @Override + public Integer call() { + CommandLine root = spec.root().commandLine(); + root.printVersionHelp(root.getOut()); + return ExitCode.SUCCESS.status(); + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/config/Relixrc.java b/src/main/java/com/darkcollective/relix/cli/config/Relixrc.java new file mode 100644 index 0000000..5282a6d --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/config/Relixrc.java @@ -0,0 +1,133 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.config; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; + +/** + * The defaults a {@code .relix/relixrc} file sets (design §5.1): {@code key = value} lines, + * with {@code #} starting a comment line. + * + *

{@code
+ * # acme's defaults
+ * output  = csv
+ * profile = staging
+ * }
+ * + *

The keys are {@value #OUTPUT} and {@value #PROFILE}. The nearest file's value for a key + * wins, and an option or environment variable on the command line beats them all. A key + * the command does not know, or a line that is not {@code key = value}, is reported as a + * warning and ignored, so a file written for a later version still works. + */ +public final class Relixrc { + + /** The file's name inside a {@code .relix/} directory. */ + public static final String FILE_NAME = "relixrc"; + + /** The default output format, as {@code -o} names it. */ + public static final String OUTPUT = "output"; + + /** The default profile, as {@code -P} names it. */ + public static final String PROFILE = "profile"; + + private static final Set KEYS = Set.of(OUTPUT, PROFILE); + + private static final Relixrc NONE = new Relixrc(Map.of()); + + private final Map values; + + private Relixrc(Map values) { + this.values = Map.copyOf(values); + } + + /** + * No defaults at all. + * + * @return the empty defaults + */ + public static Relixrc none() { + return NONE; + } + + /** + * The defaults of the given directories, the nearest file's value winning. + * + * @param directories the {@code .relix/} directories, outermost first + * @param warn where a line that is ignored is reported + * @return the merged defaults + * @throws CommandFailure with {@link ExitCode#ENVIRONMENT} when a file cannot be read + */ + public static Relixrc read(List directories, Consumer warn) { + Map values = new HashMap<>(); + for (Path dir : directories) { + Path file = dir.resolve(FILE_NAME); + if (Files.isRegularFile(file)) { + values.putAll(parse(file, warn)); + } + } + return new Relixrc(values); + } + + private static Map parse(Path file, Consumer warn) { + List lines; + try { + lines = Files.readAllLines(file, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new CommandFailure(ExitCode.ENVIRONMENT, file + ": cannot read: " + e.getMessage()); + } + Map values = new HashMap<>(); + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i).strip(); + if (line.isEmpty() || line.startsWith("#")) { + continue; + } + int equals = line.indexOf('='); + String key = equals < 0 ? "" : line.substring(0, equals).strip(); + String value = equals < 0 ? "" : line.substring(equals + 1).strip(); + if (key.isEmpty() || value.isEmpty()) { + warn.accept(file + ":" + (i + 1) + ": ignored: expected key = value"); + } else if (!KEYS.contains(key)) { + warn.accept(file + ":" + (i + 1) + ": ignored: unknown key '" + key + + "' (known: " + OUTPUT + ", " + PROFILE + ")"); + } else { + values.put(key, value); + } + } + return values; + } + + /** + * A default, when a file sets it. + * + * @param key {@value #OUTPUT} or {@value #PROFILE} + * @return its value + */ + public Optional get(String key) { + return Optional.ofNullable(values.get(key)); + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/io/Host.java b/src/main/java/com/darkcollective/relix/cli/io/Host.java index 1af72c4..cf7d7b6 100644 --- a/src/main/java/com/darkcollective/relix/cli/io/Host.java +++ b/src/main/java/com/darkcollective/relix/cli/io/Host.java @@ -15,7 +15,10 @@ */ package com.darkcollective.relix.cli.io; +import java.io.FileDescriptor; +import java.io.FileOutputStream; import java.io.InputStream; +import java.io.OutputStream; import java.io.PrintStream; import java.nio.file.Path; import java.util.Map; @@ -24,23 +27,27 @@ /** * What the command reads from the process it runs in: its streams, its environment, where - * it was started and whose home that is, and whether standard input is a terminal. + * it was started and whose home that is, and whether standard input and standard output + * are terminals. * *

Everything outside the arguments arrives through here rather than from * {@link System}, so that a test runs the whole command over streams and an environment * of its own. * * @param in standard input - * @param out standard output: data only + * @param out standard output: data only. A raw stream, unbuffered and not a + * {@link PrintStream}, so that a write to a closed pipe fails + * rather than being swallowed * @param err standard error: everything else * @param environment the process environment * @param workingDirectory where the command was started, absolute * @param home the user's home directory, absolute * @param stdinIsTerminal whether standard input is a terminal rather than a pipe or file + * @param stdoutIsTerminal whether standard output is a terminal rather than a pipe or file */ -public record Host(InputStream in, PrintStream out, PrintStream err, +public record Host(InputStream in, OutputStream out, PrintStream err, Map environment, Path workingDirectory, Path home, - boolean stdinIsTerminal) { + boolean stdinIsTerminal, boolean stdoutIsTerminal) { /** * A host, checked. @@ -60,12 +67,16 @@ public record Host(InputStream in, PrintStream out, PrintStream err, * @return the real streams, environment and directories */ public static Host system() { - return new Host(System.in, System.out, System.err, System.getenv(), + // On Java 21 a null console is the pipe check: System.console() is null when + // standard input or output is redirected, and it cannot tell which. Both are taken + // for pipes then, so `echo … | relix` writes tsv to a terminal; pass -o to choose. + // Java 22 returns a console for a pipe too, and this has to move to + // Console.isTerminal() with it. + boolean terminal = System.console() != null; + return new Host(System.in, new FileOutputStream(FileDescriptor.out), System.err, + System.getenv(), Path.of(System.getProperty("user.dir")), Path.of(System.getProperty("user.home")), - // On Java 21 a null console is the pipe check: System.console() is null - // when standard input or output is redirected. Java 22 returns a console - // for a pipe too, and this has to move to Console.isTerminal() with it. - System.console() != null); + terminal, terminal); } /** diff --git a/src/main/java/com/darkcollective/relix/cli/io/InputBinding.java b/src/main/java/com/darkcollective/relix/cli/io/InputBinding.java new file mode 100644 index 0000000..55b58b0 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/io/InputBinding.java @@ -0,0 +1,279 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.io; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.embed.Input; +import com.darkcollective.relix.embed.InputFormat; +import com.darkcollective.relix.embed.Relix; +import com.darkcollective.relix.lang.ast.ScriptParseException; +import com.darkcollective.relix.lang.ast.SourceDeclaration; +import com.darkcollective.relix.lang.ast.Statement; +import com.darkcollective.relix.lang.ast.source.ColumnSpec; +import com.darkcollective.relix.lang.ast.source.CsvFileSourceConfig; +import com.darkcollective.relix.symbol.ColumnDefinition; +import com.darkcollective.relix.symbol.Schema; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; + +/** + * A relation bound to data for one run: {@code -i NAME=[FORMAT:]LOCATION} (design §3.2). + * + *

The location is a path, {@code -} for standard input, or an {@code http(s)} URL, + * which {@code --remote} must permit. The format is {@code csv}, {@code tsv}, {@code json} + * or {@code ndjson}; without one the extension decides, and standard input, which has + * none, must name one. A file or URL is opened afresh for each read of the relation; + * standard input is read once, by the one session that binds it. + * + *

The heading is inferred from the first records ({@code --infer-rows}) or given with + * {@code --schema NAME='{ id: NUMBER, … }'}. An input marked {@code --unbounded} never + * ends: it streams, and the engine refuses a blocking operator over it before it starts. + * + * @param name the relation's name + * @param format how its text is written + * @param location what {@code -i} gave: a path, {@code -} or a URL + * @param open opens a file or URL afresh; {@code null} for standard input + * @param schema the declared heading, or {@code null} to infer one + */ +public record InputBinding(String name, InputFormat format, String location, + Supplier open, Schema schema) { + + /** The location that is standard input. */ + public static final String STDIN = "-"; + + /** + * A binding, checked. + */ + public InputBinding { + Objects.requireNonNull(name, "name"); + Objects.requireNonNull(format, "format"); + Objects.requireNonNull(location, "location"); + } + + /** + * Whether this input is standard input. + * + * @return {@code true} for {@code -} + */ + public boolean isStdin() { + return open == null; + } + + /** + * Declares this input on a session. + * + * @param session the session + * @param stdin standard input, for a binding of {@code -} + * @param sample how many records a heading is inferred from + * @param unbounded whether the input never ends + * @throws com.darkcollective.relix.embed.RelixException when the text cannot be read as + * its format, or its heading cannot be inferred + */ + public void declare(Relix session, InputStream stdin, int sample, boolean unbounded) { + Input input = isStdin() ? Input.of(format, stdin) : Input.of(format, open); + input = input.sample(sample); + if (schema != null) { + input = input.schema(schema); + } + if (unbounded) { + input = input.unbounded(); + } + session.input(name, input); + } + + /** + * The inputs a command line binds. + * + * @param specs the {@code -i} values, in order + * @param schemas the {@code --schema} values, by input name + * @param unbounded the {@code --unbounded} names + * @param directory where relative paths start from + * @param remote whether {@code --remote} permits a URL + * @return one binding per {@code -i}, in order + * @throws CommandFailure with {@link ExitCode#USAGE} for a malformed binding, a format + * that cannot be told, a file that is not there, a URL without {@code --remote}, + * a name bound twice, standard input bound twice, or a {@code --schema} or + * {@code --unbounded} for a name no {@code -i} binds + */ + public static List resolve(List specs, Map schemas, + List unbounded, Path directory, boolean remote) { + List bindings = new ArrayList<>(); + Set names = new HashSet<>(); + for (String spec : specs) { + InputBinding binding = parse(spec, schemas, directory, remote); + if (!names.add(binding.name())) { + throw usage("-i " + spec + ": '" + binding.name() + "' is bound twice"); + } + bindings.add(binding); + } + if (bindings.stream().filter(InputBinding::isStdin).count() > 1) { + throw usage("standard input (-) can be bound to one input only"); + } + for (String name : schemas.keySet()) { + if (!names.contains(name)) { + throw usage("--schema " + name + ": no input of that name is bound with -i"); + } + } + for (String name : unbounded) { + InputBinding binding = bindings.stream().filter(b -> b.name().equals(name)).findFirst() + .orElseThrow(() -> usage("--unbounded " + name + ": no input of that name is bound with -i")); + if (binding.format() == InputFormat.JSON) { + throw usage("--unbounded " + name + ": a JSON array is read whole, so it cannot be " + + "unbounded; write one object per line and bind it as ndjson"); + } + } + return List.copyOf(bindings); + } + + private static InputBinding parse(String spec, Map schemas, Path directory, + boolean remote) { + int eq = spec.indexOf('='); + if (eq <= 0 || eq == spec.length() - 1) { + throw usage("-i " + spec + ": expected NAME=[FORMAT:]LOCATION"); + } + String name = spec.substring(0, eq).strip(); + String rest = spec.substring(eq + 1); + InputFormat format = null; + int colon = rest.indexOf(':'); + if (colon > 0) { + format = format(rest.substring(0, colon)); + if (format != null) { + rest = rest.substring(colon + 1); + } + } + if (rest.isEmpty()) { + throw usage("-i " + spec + ": expected NAME=[FORMAT:]LOCATION"); + } + Schema schema = schemas.containsKey(name) ? schema(name, schemas.get(name)) : null; + if (rest.equals(STDIN)) { + if (format == null) { + throw usage("-i " + spec + ": standard input has no extension to tell its format by; " + + "name one, as " + name + "=ndjson:-"); + } + return new InputBinding(name, format, rest, null, schema); + } + String lower = rest.toLowerCase(Locale.ROOT); + if (lower.startsWith("http://") || lower.startsWith("https://")) { + if (!remote) { + throw usage("-i " + spec + ": reading a URL needs --remote"); + } + URI uri; + try { + uri = new URI(rest); + } catch (URISyntaxException e) { + throw usage("-i " + spec + ": " + e.getMessage()); + } + InputFormat chosen = format != null ? format : byExtension(spec, uri.getPath()); + return new InputBinding(name, chosen, rest, () -> open(uri), schema); + } + Path path; + try { + path = directory.resolve(rest).normalize(); + } catch (InvalidPathException e) { + throw usage("-i " + spec + ": " + e.getMessage()); + } + if (!Files.isRegularFile(path)) { + throw usage("-i " + spec + ": " + rest + ": no such file"); + } + InputFormat chosen = format != null ? format : byExtension(spec, path.getFileName().toString()); + return new InputBinding(name, chosen, rest, () -> open(path), schema); + } + + /** The format a prefix names, or {@code null} when it names none, as {@code C} in {@code C:\…}. */ + private static InputFormat format(String prefix) { + for (InputFormat format : InputFormat.values()) { + if (format.name().equalsIgnoreCase(prefix)) { + return format; + } + } + return null; + } + + private static InputFormat byExtension(String spec, String file) { + String lower = file == null ? "" : file.toLowerCase(Locale.ROOT); + int dot = lower.lastIndexOf('.'); + String extension = dot < 0 ? "" : lower.substring(dot + 1); + return switch (extension) { + case "csv" -> InputFormat.CSV; + case "tsv", "tab" -> InputFormat.TSV; + case "json" -> InputFormat.JSON; + case "ndjson", "jsonl" -> InputFormat.NDJSON; + default -> throw usage("-i " + spec + ": cannot tell the format from " + + (extension.isEmpty() ? "a name with no extension" : "the extension ." + extension) + + "; name it, as NAME=csv:LOCATION (csv, tsv, json or ndjson)"); + }; + } + + /** + * A heading as {@code --schema} writes it: a source's {@code schema:} block, such as + * {@code { id: NUMBER, name: STRING }}, read by the engine's own parser. + */ + private static Schema schema(String name, String text) { + List statements; + try { + statements = Relix.parse("source " + name + " from csv(\"-\") { schema: " + text + " };") + .statements(); + } catch (ScriptParseException e) { + throw usage("--schema " + name + ": expected a heading such as '{ id: NUMBER, name: STRING }': " + + e.getMessage()); + } + if (statements.size() == 1 && statements.getFirst() instanceof SourceDeclaration source + && source.config() instanceof CsvFileSourceConfig csv && !csv.columns().isEmpty()) { + List columns = new ArrayList<>(); + for (ColumnSpec column : csv.columns()) { + columns.add(new ColumnDefinition(column.name(), column.type())); + } + return new Schema(columns); + } + throw usage("--schema " + name + ": expected a heading such as '{ id: NUMBER, name: STRING }'"); + } + + private static InputStream open(Path path) { + try { + return Files.newInputStream(path); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static InputStream open(URI uri) { + try { + return uri.toURL().openStream(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static CommandFailure usage(String message) { + return new CommandFailure(ExitCode.USAGE, message); + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/io/Interruption.java b/src/main/java/com/darkcollective/relix/cli/io/Interruption.java new file mode 100644 index 0000000..2678350 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/io/Interruption.java @@ -0,0 +1,99 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.io; + +import com.darkcollective.relix.embed.Relix; + +import java.util.stream.Stream; + +/** + * What an interrupt ({@code SIGINT}) stops: the session running now, and the row stream + * being read from it (design §3.6). + * + *

The run registers each as it opens it. {@link #interrupt()}, called from a shutdown + * hook on another thread, cancels what the session is running, closes the stream and then + * the session, so that a database statement in flight is cancelled rather than abandoned. + * The run, finding its query cancelled, sees {@link #interrupted()} and ends with + * {@link com.darkcollective.relix.cli.ExitCode#INTERRUPTED}. + */ +public final class Interruption { + + private Relix session; + private Stream stream; + private boolean interrupted; + + /** + * Notes the session now running, or none. + * + * @param session the session, or {@code null} once it is closed + */ + public synchronized void session(Relix session) { + this.session = session; + } + + /** + * Notes the row stream now being read, or none. + * + * @param stream the stream, or {@code null} once it is closed + */ + public synchronized void stream(Stream stream) { + this.stream = stream; + } + + /** + * Whether the run has been interrupted. + * + * @return {@code true} once {@link #interrupt()} has been called + */ + public synchronized boolean interrupted() { + return interrupted; + } + + /** + * Stops the run: cancels the session's queries, then closes the stream and the + * session. Safe from any thread, and more than once. + */ + public void interrupt() { + Relix s; + Stream r; + synchronized (this) { + interrupted = true; + s = session; + r = stream; + session = null; + stream = null; + } + // Each step is attempted whatever the one before it did: an interrupt is the last + // chance to let go of a database statement. + if (s != null) { + quietly(s::cancel); + } + if (r != null) { + quietly(r::close); + } + if (s != null) { + quietly(s::close); + } + } + + private static void quietly(Runnable step) { + try { + step.run(); + } catch (RuntimeException ignored) { + // The process is ending; there is no one left to tell. + } + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/io/Pager.java b/src/main/java/com/darkcollective/relix/cli/io/Pager.java new file mode 100644 index 0000000..ef045be --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/io/Pager.java @@ -0,0 +1,114 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.io; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Shows a long text a screen at a time, through {@code $PAGER}, when standard output is a + * terminal, as {@code git} and {@code man} do. + * + *

The pager is {@code $PAGER}, else {@code less}, run by the shell; {@code less} is given + * {@code LESS=FRX} unless {@code $LESS} is set, so a page that fits the screen is printed + * and left there. On Windows there is a pager only when {@code $PAGER} names one. An empty + * {@code $PAGER} or {@code cat} means none, and so does a pager that will not start: the + * text is then written to standard output as it is. + */ +public final class Pager { + + /** The variable naming the pager. */ + public static final String PAGER_VARIABLE = "PAGER"; + + private Pager() { + } + + /** + * Shows a text, through the pager when there is one. + * + * @param host the process + * @param text the text + * @param sink standard output, for when there is no pager + */ + public static void show(Host host, String text, RowSink sink) { + Optional> command = paged(host) ? command(host.environment(), isWindows()) : Optional.empty(); + if (command.isEmpty() || !run(command.get(), host.environment(), text)) { + sink.text(text); + } + } + + /** + * Whether a host's standard output can be paged: it is a terminal, and it is this + * process's own, which is where a pager writes. + */ + private static boolean paged(Host host) { + return host.stdoutIsTerminal() && host.out() instanceof FileOutputStream; + } + + /** + * The pager to run, as a command line, if any. + * + * @param environment the process environment + * @param windows whether this is Windows + * @return the command, or empty for none + */ + public static Optional> command(Map environment, boolean windows) { + String pager = environment.get(PAGER_VARIABLE); + if (pager == null) { + return windows ? Optional.empty() : Optional.of(List.of("sh", "-c", "less")); + } + if (pager.isBlank() || pager.strip().equals("cat")) { + return Optional.empty(); + } + return Optional.of(windows ? List.of("cmd", "/c", pager) : List.of("sh", "-c", pager)); + } + + private static boolean run(List command, Map environment, String text) { + ProcessBuilder builder = new ProcessBuilder(command) + .redirectOutput(ProcessBuilder.Redirect.INHERIT) + .redirectError(ProcessBuilder.Redirect.INHERIT); + if (!environment.containsKey("LESS")) { + builder.environment().put("LESS", "FRX"); + } + Process process; + try { + process = builder.start(); + } catch (IOException e) { + return false; + } + try (OutputStream in = process.getOutputStream()) { + in.write(text.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + // The reader quit before the end, as a pager may. + } + try { + process.waitFor(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return true; + } + + private static boolean isWindows() { + return File.separatorChar == '\\'; + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/io/Reporter.java b/src/main/java/com/darkcollective/relix/cli/io/Reporter.java index 15d16d5..0b9dfa9 100644 --- a/src/main/java/com/darkcollective/relix/cli/io/Reporter.java +++ b/src/main/java/com/darkcollective/relix/cli/io/Reporter.java @@ -82,6 +82,28 @@ public void error(String message) { err.print("relix: " + message + "\n"); } + /** + * Prints a warning that has no place in a script, unless warnings are silenced. + * + * @param message what is amiss + */ + public void warning(String message) { + if (verbosity != Verbosity.QUIET) { + err.print("relix: warning: " + message + "\n"); + } + } + + /** + * Prints what a command did, such as what it installed, unless only errors are wanted. + * + * @param message what was done + */ + public void info(String message) { + if (verbosity != Verbosity.QUIET) { + err.print("relix: " + message + "\n"); + } + } + /** * Prints a notice when {@code -v} is given. * diff --git a/src/main/java/com/darkcollective/relix/cli/io/RowSink.java b/src/main/java/com/darkcollective/relix/cli/io/RowSink.java new file mode 100644 index 0000000..ff090b3 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/io/RowSink.java @@ -0,0 +1,114 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.io; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.render.OutputFormat; +import com.darkcollective.relix.processor.Row; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Iterator; +import java.util.Objects; +import java.util.stream.Stream; + +/** + * Where result rows go: standard output, buffered, and watched for the reader going away + * (design §3.6). + * + *

Each row is written as soon as it is encoded, and a failed write is noticed after + * that row: the row stream is closed, which the engine takes as cancellation, so a plan + * stops pulling and a database cursor is closed, and the run ends with + * {@link ExitCode#PIPE_CLOSED} and nothing said. {@code relix … | head} therefore ends + * when {@code head} does. + * + *

Output is buffered and flushed at the end of each result set, or after every row when + * line-buffered ({@code --line-buffered}, as {@code grep}'s), for a reader that wants each + * row the moment it is made. A buffered write reaches the pipe only when the buffer fills, + * so it is then that a closed pipe is noticed. + */ +public final class RowSink { + + private static final int BUFFER = 1 << 16; + + private final OutputStream out; + private final boolean lineBuffered; + + /** + * A sink over standard output. + * + * @param out standard output, unbuffered + * @param lineBuffered whether to flush after every row + */ + public RowSink(OutputStream out, boolean lineBuffered) { + this.out = new BufferedOutputStream(Objects.requireNonNull(out, "out"), BUFFER); + this.lineBuffered = lineBuffered; + } + + /** + * Writes one result set and flushes it. + * + * @param rows the rows, pulled one at a time; closed here if the write fails + * @param encoder the result set's encoder + * @return how many rows were written + * @throws CommandFailure with {@link ExitCode#PIPE_CLOSED} when standard output has + * gone away + */ + public long write(Stream rows, OutputFormat.Encoder encoder) { + long count = 0; + try { + write(encoder.begin()); + for (Iterator it = rows.iterator(); it.hasNext(); ) { + write(encoder.row(it.next())); + count++; + if (lineBuffered) { + out.flush(); + } + } + write(encoder.end()); + out.flush(); + } catch (IOException e) { + rows.close(); + throw new CommandFailure(ExitCode.PIPE_CLOSED, null); + } + return count; + } + + /** + * Writes a whole text, such as a report or a plan, and flushes it. + * + * @param text the text + * @throws CommandFailure with {@link ExitCode#PIPE_CLOSED} when standard output has + * gone away + */ + public void text(String text) { + try { + write(text); + out.flush(); + } catch (IOException e) { + throw new CommandFailure(ExitCode.PIPE_CLOSED, null); + } + } + + private void write(String text) throws IOException { + if (!text.isEmpty()) { + out.write(text.getBytes(StandardCharsets.UTF_8)); + } + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/io/ScriptSource.java b/src/main/java/com/darkcollective/relix/cli/io/ScriptSource.java index 6597248..b6edbb3 100644 --- a/src/main/java/com/darkcollective/relix/cli/io/ScriptSource.java +++ b/src/main/java/com/darkcollective/relix/cli/io/ScriptSource.java @@ -113,13 +113,16 @@ public Reporter.Place place(Optional location) { * @param files the {@code SCRIPT} arguments, in order; {@code -} is standard input * @param host the process, for standard input and whether it is a terminal * @param directory the directory relative paths start from ({@code -C}), absolute + * @param stdinBound whether an input is bound to standard input ({@code -i NAME=…:-}), + * so that the script must come from {@code -e} or a file * @return one source per script, in order * @throws CommandFailure with {@link ExitCode#USAGE} when there is nothing to run, when * both {@code -e} and files are given, when standard input is - * named twice, or when a file cannot be read + * named twice or is both data and script, or when a file cannot + * be read */ public static List resolve(List expressions, List files, - Host host, Path directory) { + Host host, Path directory, boolean stdinBound) { if (!expressions.isEmpty() && !files.isEmpty()) { throw new CommandFailure(ExitCode.USAGE, "give the script with -e or as files, not both"); @@ -127,10 +130,18 @@ public static List resolve(List expressions, List if (!expressions.isEmpty()) { return List.of(fromExpressions(expressions, directory)); } + if (stdinBound && (files.isEmpty() || files.contains("-"))) { + throw new CommandFailure(ExitCode.USAGE, "standard input is bound with -i, so it cannot " + + "be the script too; give the script with -e or as a file"); + } if (!files.isEmpty()) { if (files.stream().filter("-"::equals).count() > 1) { throw new CommandFailure(ExitCode.USAGE, "standard input (-) can be read only once"); } + if (stdinBound && files.size() > 1) { + throw new CommandFailure(ExitCode.USAGE, "standard input is bound with -i and can be " + + "read once, so it can be given to one script only"); + } List sources = new ArrayList<>(); for (String file : files) { sources.add(file.equals("-") ? fromStdin(host, directory) : fromFile(file, directory)); diff --git a/src/main/java/com/darkcollective/relix/cli/io/ScriptText.java b/src/main/java/com/darkcollective/relix/cli/io/ScriptText.java new file mode 100644 index 0000000..11fc694 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/io/ScriptText.java @@ -0,0 +1,249 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.io; + +import com.darkcollective.relix.ast.SourceLocation; +import com.darkcollective.relix.embed.Relix; +import com.darkcollective.relix.lang.ast.Script; +import com.darkcollective.relix.lang.ast.Statement; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * A script's text together with what it parses to: where in the text each statement is. + * + *

The parser gives each statement the place it starts; this finds where it ends, at the + * semicolon that closes it, passing over the comments a semicolon may be in. Those are the + * language's, read as its two lexers read them: {@code --} to the end of the line and {@code /* *}{@code /}, outside + * a string. Within an inline table only a line that begins with one is a comment, since a + * cell may hold {@code --} as data, and a table's {@code |---|} rule is not one. + */ +public final class ScriptText { + + /** + * A stretch of the text, from {@code start} up to but not including {@code end}. + * + * @param start the offset of its first character + * @param end the offset just after its last + */ + public record Span(int start, int end) { + + /** + * The text this stretch covers. + * + * @param text the whole text + * @return the stretch of it + */ + public String of(String text) { + return text.substring(start, end); + } + } + + /** + * One statement and where it is written. + * + * @param statement what it parses to + * @param span its text, from its first character to the semicolon that ends it + */ + public record Piece(Statement statement, Span span) { + + /** Checks every component. */ + public Piece { + Objects.requireNonNull(statement, "statement"); + Objects.requireNonNull(span, "span"); + } + } + + private final String text; + private final List pieces; + + private ScriptText(String text, List pieces) { + this.text = text; + this.pieces = List.copyOf(pieces); + } + + /** + * Parses a script and places its statements in its text. + * + * @param text the script + * @return the script and its layout + * @throws com.darkcollective.relix.embed.RelixException when the text does not parse + */ + public static ScriptText of(String text) { + Script script = Relix.parse(text); + int[] lines = lineStarts(text); + List pieces = new ArrayList<>(); + for (Statement statement : script.statements()) { + int start = offset(statement.location(), lines, text); + Scan scan = new Scan(text, start); + scan.toEndOfStatement(); + pieces.add(new Piece(statement, new Span(start, scan.position))); + } + return new ScriptText(text, pieces); + } + + /** {@return the text} */ + public String text() { + return text; + } + + /** {@return each statement and where it is written, in order} */ + public List statements() { + return pieces; + } + + private static int[] lineStarts(String text) { + List starts = new ArrayList<>(); + starts.add(0); + for (int i = 0; i < text.length(); i++) { + if (text.charAt(i) == '\n') { + starts.add(i + 1); + } + } + return starts.stream().mapToInt(Integer::intValue).toArray(); + } + + private static int offset(SourceLocation location, int[] lines, String text) { + int line = Math.max(1, Math.min(location.line(), lines.length)); + return Math.min(text.length(), lines[line - 1] + Math.max(0, location.column() - 1)); + } + + /** + * Reads the text as the lexers do, as far as finding the end of a statement needs: + * through comments, strings, delimited names, braces (an expression body) and brackets + * (an inline table). + */ + private static final class Scan { + + private enum Context { TOP, BRACE, BRACKET } + + private final String text; + private final List stack = new ArrayList<>(List.of(Context.TOP)); + private int position; + + Scan(String text, int start) { + this.text = text; + this.position = start; + } + + /** Reads up to and including the semicolon that ends the statement. */ + void toEndOfStatement() { + while (position < text.length()) { + if (stack.getLast() == Context.TOP && text.charAt(position) == ';') { + position++; + return; + } + step(); + } + } + + private void step() { + char c = text.charAt(position); + Context context = stack.getLast(); + if (comment(context)) { + return; + } + switch (c) { + case '"' -> quoted('"'); + case '\'' -> { + // A quote is a string in an expression; in a table it is a cell's text. + if (context == Context.BRACE) { + quoted('\''); + } else { + position++; + } + } + case '`' -> { + if (context == Context.BRACKET) { + position++; + } else { + quoted('`'); + } + } + case '{' -> { + if (context != Context.BRACKET) { + stack.add(Context.BRACE); + } + position++; + } + case '}' -> { + if (context == Context.BRACE) { + stack.removeLast(); + } + position++; + } + case '[' -> { + // Brackets inside an expression are an array, not a table. + if (context != Context.BRACE) { + stack.add(Context.BRACKET); + } + position++; + } + case ']' -> { + if (context == Context.BRACKET) { + stack.removeLast(); + } + position++; + } + default -> position++; + } + } + + /** Reads past a comment starting here, if one does. */ + private boolean comment(Context context) { + if (!text.startsWith("--", position) && !text.startsWith("/*", position)) { + return false; + } + if (context == Context.BRACKET && !atLineStart()) { + return false; + } + if (text.startsWith("--", position)) { + int end = text.indexOf('\n', position); + position = end < 0 ? text.length() : end; + } else { + int end = text.indexOf("*/", position + 2); + position = end < 0 ? text.length() : end + 2; + } + return true; + } + + private boolean atLineStart() { + for (int i = position - 1; i >= 0; i--) { + char c = text.charAt(i); + if (c == '\n') { + return true; + } + if (c != ' ' && c != '\t' && c != '\r') { + return false; + } + } + return true; + } + + private void quoted(char quote) { + position++; + while (position < text.length() && text.charAt(position) != quote) { + if (text.charAt(position) == '\\' && quote != '`') { + position++; + } + position++; + } + position = Math.min(text.length(), position + 1); + } + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/io/package-info.java b/src/main/java/com/darkcollective/relix/cli/io/package-info.java index 92870b7..b79226f 100644 --- a/src/main/java/com/darkcollective/relix/cli/io/package-info.java +++ b/src/main/java/com/darkcollective/relix/cli/io/package-info.java @@ -14,7 +14,8 @@ * limitations under the License. */ /** - * Where scripts come from: a {@code ScriptLoader} over files, so that {@code import} - * resolves relative to the importing file. + * The command's streams: where scripts come from, with a {@code ScriptLoader} over files + * so that {@code import} resolves relative to the importing file; where rows go, watched + * for the reader going away; and what an interrupt stops. */ package com.darkcollective.relix.cli.io; diff --git a/src/main/java/com/darkcollective/relix/cli/render/OutputFormat.java b/src/main/java/com/darkcollective/relix/cli/render/OutputFormat.java index f94a396..df25036 100644 --- a/src/main/java/com/darkcollective/relix/cli/render/OutputFormat.java +++ b/src/main/java/com/darkcollective/relix/cli/render/OutputFormat.java @@ -31,108 +31,284 @@ import com.darkcollective.relix.symbol.Schema; import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; import java.util.stream.Stream; /** - * Output format for {@code --exec} query results. + * The formats a query's rows are written in (design §3.3). * - *

Each constant knows how to {@link #write} one query result to a - * {@link PrintWriter}, consuming the lazy row {@link Stream} as it goes. Three of - * the four formats are fully streaming — they emit each row as it arrives - * and run in constant memory regardless of result size: + *

Each constant makes an {@link Encoder} for one result set, which turns it into text a + * row at a time: what comes before the rows, each row, and what comes after. Every format + * but {@link #TABLE} is streaming: a row's text is ready as soon as the row is, + * so a result larger than memory goes straight through, and a reader at the other end of + * a pipe sees each row as it is made: *

    + *
  • {@link #TSV} — IANA tab-separated values with a header row, its escapes keeping + * each row on one line;
  • *
  • {@link #CSV} — RFC 4180 comma-separated values with a header row;
  • + *
  • {@link #NDJSON} — one JSON object per line;
  • *
  • {@link #JSON} — a JSON array of one object per row;
  • *
  • {@link #MARKDOWN} — a GitHub-flavoured Markdown table.
  • *
- * The default {@link #TABLE} format renders an aligned ASCII table and must buffer - * each query's rows to compute column widths. + * {@link #TABLE} aligns its columns, so it must see every row before it writes any. * *

Every format ends its lines with {@code \n}, on Windows too: the output is data for * the next stage of a pipeline, and must not change with the platform it was made on. */ public enum OutputFormat { - /** Aligned ASCII table (default). Buffers each query for column-width alignment. */ + /** Aligned table for a person to read. Buffers each result set to align its columns. */ TABLE { @Override - public void write(PrintWriter out, String label, Schema schema, Stream rows) { - out.print(QueryResultFormatter.format(label, schema, rows)); + public Encoder encoder(String label, Schema schema, Options options) { + String nullText = options.nullText() != null ? options.nullText() : "NULL"; + List rows = new ArrayList<>(); + return new Encoder() { + @Override + public String row(Row row) { + rows.add(row); + return ""; + } + + @Override + public String end() { + return QueryResultFormatter.format(label, schema, rows, nullText, options.color()); + } + }; + } + }, + + /** + * IANA tab-separated values with a header row. A tab, line feed, carriage return or + * backslash in a value is written {@code \t}, {@code \n}, {@code \r} or {@code \\}, so + * that a row is always one line, as {@code cut}, {@code awk} and {@code sort} expect. + */ + TSV { + @Override + public Encoder encoder(String label, Schema schema, Options options) { + return delimited(schema, options, '\t', OutputFormat::tsvField); } }, - /** RFC 4180 CSV with a header row, its lines ended by {@code \n}. Fully streaming. */ + /** RFC 4180 comma-separated values with a header row, its lines ended by {@code \n}. */ CSV { @Override - public void write(PrintWriter out, String label, Schema schema, Stream rows) { - List cols = schema.columns(); - StringBuilder header = new StringBuilder(); - for (int i = 0; i < cols.size(); i++) { - if (i > 0) header.append(','); - header.append(csvField(cols.get(i).name())); + public Encoder encoder(String label, Schema schema, Options options) { + return delimited(schema, options, ',', OutputFormat::csvField); + } + }, + + /** + * One JSON object per line ({@code {column: value}}), the values encoded as + * {@link #JSON} encodes them. There is no header, and a NULL is JSON's {@code null}. + * {@link Options#tagQuery()} adds a {@code "_query"} member naming the result set, so + * that several can share one stream. + */ + NDJSON { + @Override + public Encoder encoder(String label, Schema schema, Options options) { + if (options.tagQuery()) { + String tag = "{" + JsonText.quote(QUERY_MEMBER) + ":" + JsonText.quote(label); + return row -> { + String object = jsonObject(schema, row); + return tag + (object.length() > 2 ? "," : "") + object.substring(1) + "\n"; + }; } - out.print(header + "\n"); - rows.forEach(row -> { - StringBuilder line = new StringBuilder(); - for (int i = 0; i < cols.size(); i++) { - if (i > 0) line.append(','); - line.append(csvField(plainCell(row.get(i)))); - } - out.print(line + "\n"); - }); + return row -> jsonObject(schema, row) + "\n"; } }, - /** JSON array of one object per row ({@code {column: value}}). Fully streaming. */ + /** A JSON array of one object per row ({@code {column: value}}). */ JSON { @Override - public void write(PrintWriter out, String label, Schema schema, Stream rows) { - List cols = schema.columns(); - out.print('['); - boolean[] first = {true}; - rows.forEach(row -> { - out.print(first[0] ? "\n " : ",\n "); - first[0] = false; - StringBuilder obj = new StringBuilder("{"); - for (int i = 0; i < cols.size(); i++) { - if (i > 0) obj.append(','); - obj.append(JsonText.quote(cols.get(i).name())).append(':').append(jsonValue(row.get(i))); + public Encoder encoder(String label, Schema schema, Options options) { + return new Encoder() { + private boolean first = true; + + @Override + public String begin() { + return "["; + } + + @Override + public String row(Row row) { + String separator = first ? "\n " : ",\n "; + first = false; + return separator + jsonObject(schema, row); + } + + @Override + public String end() { + return first ? "]\n" : "\n]\n"; } - out.print(obj.append('}')); - }); - out.print(first[0] ? "]\n" : "\n]\n"); + }; } }, - /** GitHub-flavoured Markdown table, under a {@code ###} heading. Fully streaming. */ + /** A GitHub-flavoured Markdown table, under a {@code ###} heading. */ MARKDOWN { @Override - public void write(PrintWriter out, String label, Schema schema, Stream rows) { + public Encoder encoder(String label, Schema schema, Options options) { + String nullText = options.nullText() != null ? options.nullText() : ""; List cols = schema.columns(); - out.print("### " + label + "\n"); - out.print("\n"); - StringBuilder header = new StringBuilder("|"); - StringBuilder rule = new StringBuilder("|"); - for (ColumnDefinition col : cols) { - header.append(' ').append(mdCell(col.name())).append(" |"); - rule.append(" --- |"); - } - out.print(header + "\n"); - out.print(rule + "\n"); - rows.forEach(row -> { - StringBuilder line = new StringBuilder("|"); - for (int i = 0; i < cols.size(); i++) { - line.append(' ').append(mdCell(plainCell(row.get(i)))).append(" |"); + return new Encoder() { + @Override + public String begin() { + StringBuilder header = new StringBuilder("|"); + StringBuilder rule = new StringBuilder("|"); + for (ColumnDefinition col : cols) { + header.append(' ').append(mdCell(col.name())).append(" |"); + rule.append(" --- |"); + } + return "### " + label + "\n\n" + header + "\n" + rule + "\n"; + } + + @Override + public String row(Row row) { + StringBuilder line = new StringBuilder("|"); + for (int i = 0; i < cols.size(); i++) { + line.append(' ').append(mdCell(plainCell(row.get(i), nullText))).append(" |"); + } + return line.append('\n').toString(); } - out.print(line + "\n"); - }); - out.print("\n"); + + @Override + public String end() { + return "\n"; + } + }; } }; - /** Writes one query result, consuming {@code rows}. */ - public abstract void write(PrintWriter out, String label, Schema schema, Stream rows); + /** The member {@link Options#tagQuery()} adds to each NDJSON object. */ + public static final String QUERY_MEMBER = "_query"; + + /** + * How a result set is written, beyond its format. + * + * @param header whether the delimited formats ({@link #TSV}, {@link #CSV}) write a + * header row + * @param nullText how a NULL is written, or {@code null} for the format's own: empty in + * the delimited formats and Markdown, {@code NULL} in a table. The JSON + * formats always write JSON's {@code null}. + * @param color whether a {@link #TABLE} uses terminal colours: bold column names, dim + * NULLs. The other formats are never coloured. + * @param tagQuery whether each {@link #NDJSON} object names its result set in a + * {@value #QUERY_MEMBER} member + */ + public record Options(boolean header, String nullText, boolean color, boolean tagQuery) { + + /** Each format's own defaults: a header row, its own NULL, no colour, no tag. */ + public static final Options DEFAULTS = new Options(true, null, false, false); + } + + /** + * One result set, turned into text a row at a time. + * + *

The text a call returns is what to write next; an encoder writes nothing itself. + * An encoder is for one result set, and is called {@link #begin} once, {@link #row} once + * per row, then {@link #end} once. + */ + @FunctionalInterface + public interface Encoder { + + /** + * What comes before the first row: a header, an opening bracket. + * + * @return the text, possibly empty + */ + default String begin() { + return ""; + } + + /** + * One row. + * + * @param row the row + * @return its text, possibly empty when the format holds rows back + */ + String row(Row row); + + /** + * What comes after the last row. + * + * @return the text, possibly empty + */ + default String end() { + return ""; + } + } + + /** + * An encoder for one result set. + * + * @param label the result's label: a query's name, or {@code } + * @param schema the result's heading + * @param options the header and NULL choices + * @return an encoder for its rows + */ + public abstract Encoder encoder(String label, Schema schema, Options options); + + /** + * Writes one result set in this format's defaults, consuming {@code rows}. + * + * @param out where to write + * @param label the result's label + * @param schema the result's heading + * @param rows the rows; consumed, a row written as soon as it arrives + */ + public void write(PrintWriter out, String label, Schema schema, Stream rows) { + write(out, label, schema, rows, Options.DEFAULTS); + } + + /** + * Writes one result set, consuming {@code rows}. + * + * @param out where to write + * @param label the result's label + * @param schema the result's heading + * @param rows the rows; consumed, a row written as soon as it arrives + * @param options the header and NULL choices + */ + public void write(PrintWriter out, String label, Schema schema, Stream rows, + Options options) { + Encoder encoder = encoder(label, schema, options); + out.print(encoder.begin()); + rows.forEach(row -> out.print(encoder.row(row))); + out.print(encoder.end()); + } + + /** + * Whether this format writes each row as it arrives, rather than holding them back. + * + * @return {@code false} for {@link #TABLE} alone + */ + public boolean streams() { + return this != TABLE; + } + + /** + * Whether this format is for a program rather than a person: one result set, nothing + * around it but the format's own syntax. + * + * @return {@code true} for {@link #TSV}, {@link #CSV}, {@link #NDJSON} and {@link #JSON} + */ + public boolean isMachineFormat() { + return this != TABLE && this != MARKDOWN; + } + + /** + * The format's name as an option gives it. + * + * @return the lower-case name + */ + public String optionName() { + return name().toLowerCase(Locale.ROOT); + } /** * Resolves a format name (case-insensitive) to a constant. @@ -147,15 +323,46 @@ public static OutputFormat fromString(String name) { return format; } } - throw new IllegalArgumentException( - "unknown output format: '" + name + "' (expected one of: table, csv, json, markdown)"); + throw new IllegalArgumentException("unknown output format: '" + name + "' (expected one of: " + + Arrays.stream(values()).map(OutputFormat::optionName).collect(Collectors.joining(", ")) + + ")"); } // ── shared value rendering ────────────────────────────────────────────── - /** Plain text for a cell; SQL {@code NULL} renders as the empty string. */ - private static String plainCell(Value value) { - return value.isNull() ? "" : value.asDisplayString(); + /** A delimited format: a header row unless asked not to, then one line per row. */ + private static Encoder delimited(Schema schema, Options options, char delimiter, + java.util.function.UnaryOperator field) { + String nullText = options.nullText() != null ? options.nullText() : ""; + List cols = schema.columns(); + return new Encoder() { + @Override + public String begin() { + if (!options.header()) { + return ""; + } + return cols.stream().map(c -> field.apply(c.name())) + .collect(Collectors.joining(String.valueOf(delimiter), "", "\n")); + } + + @Override + public String row(Row row) { + StringBuilder line = new StringBuilder(); + for (int i = 0; i < cols.size(); i++) { + if (i > 0) line.append(delimiter); + // The NULL spelling is written as given, so that one a reader is told + // to expect, such as \N, arrives exactly so. + Value value = row.get(i); + line.append(value.isNull() ? nullText : field.apply(value.asDisplayString())); + } + return line.append('\n').toString(); + } + }; + } + + /** Plain text for a cell, with a NULL written as {@code nullText}. */ + private static String plainCell(Value value, String nullText) { + return value.isNull() ? nullText : value.asDisplayString(); } /** Quotes a CSV field when it contains a comma, quote, or line break (RFC 4180). */ @@ -167,12 +374,59 @@ private static String csvField(String s) { return s; } + /** Escapes a TSV field: the backslash first, then tab and the line breaks. */ + private static String tsvField(String s) { + if (s.indexOf('\\') < 0 && s.indexOf('\t') < 0 && s.indexOf('\n') < 0 && s.indexOf('\r') < 0) { + return s; + } + StringBuilder out = new StringBuilder(s.length() + 8); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\\' -> out.append("\\\\"); + case '\t' -> out.append("\\t"); + case '\n' -> out.append("\\n"); + case '\r' -> out.append("\\r"); + default -> out.append(c); + } + } + return out.toString(); + } + /** Escapes a Markdown table cell: pipes are escaped and line breaks become spaces. */ private static String mdCell(String s) { return s.replace("|", "\\|").replace('\n', ' ').replace('\r', ' '); } - /** Renders a value as JSON: scalars as literals, structs/arrays recursively. */ + /** + * A row as one JSON object. A closed heading gives its columns in order; an open one, + * read schema-on-read, gives each row's own. + */ + private static String jsonObject(Schema schema, Row row) { + StringBuilder obj = new StringBuilder("{"); + if (schema.isOpen()) { + List names = row.columnNames(); + for (int i = 0; i < names.size(); i++) { + if (i > 0) obj.append(','); + obj.append(JsonText.quote(names.get(i))).append(':').append(jsonValue(row.get(names.get(i)))); + } + } else { + List cols = schema.columns(); + for (int i = 0; i < cols.size(); i++) { + if (i > 0) obj.append(','); + obj.append(JsonText.quote(cols.get(i).name())).append(':').append(jsonValue(row.get(i))); + } + } + return obj.append('}').toString(); + } + + /** + * Renders a value as JSON: scalars as literals, structs and arrays recursively, and the + * temporal values as their ISO-8601 strings. + * + * @param value the value + * @return its JSON text + */ public static String jsonValue(Value value) { return switch (value) { case NullValue ignored -> "null"; @@ -181,17 +435,14 @@ public static String jsonValue(Value value) { case StringValue s -> JsonText.quote(s.value()); case StructValue st -> st.fields().entrySet().stream() .map(e -> JsonText.quote(e.getKey()) + ":" + jsonValue(e.getValue())) - .collect(java.util.stream.Collectors.joining(",", "{", "}")); + .collect(Collectors.joining(",", "{", "}")); case ArrayValue ar -> ar.elements().stream() .map(OutputFormat::jsonValue) - .collect(java.util.stream.Collectors.joining(",", "[", "]")); - // Temporal values render as their canonical ISO-8601 string. + .collect(Collectors.joining(",", "[", "]")); case DateValue d -> JsonText.quote(d.asDisplayString()); case TimeValue t -> JsonText.quote(t.asDisplayString()); case TimestampValue ts -> JsonText.quote(ts.asDisplayString()); case DurationValue du -> JsonText.quote(du.asDisplayString()); }; } - - /** Quotes and escapes a string as a JSON string literal. */ } diff --git a/src/main/java/com/darkcollective/relix/cli/render/QueryResultFormatter.java b/src/main/java/com/darkcollective/relix/cli/render/QueryResultFormatter.java index 25ee3b9..15419f2 100644 --- a/src/main/java/com/darkcollective/relix/cli/render/QueryResultFormatter.java +++ b/src/main/java/com/darkcollective/relix/cli/render/QueryResultFormatter.java @@ -17,6 +17,7 @@ import com.darkcollective.relix.processor.Row; import com.darkcollective.relix.value.NumberValue; +import com.darkcollective.relix.value.Value; import com.darkcollective.relix.symbol.Schema; import java.util.List; @@ -41,7 +42,7 @@ *

  • Column widths are the maximum of the header name and the widest value, * capped at {@value #MAX_COLUMN_WIDTH} characters.
  • *
  • Values longer than the cap are truncated with {@code …}.
  • - *
  • {@code NULL} values display as {@code NULL}.
  • + *
  • {@code NULL} values display as {@code NULL}, or as the caller says.
  • * * *

    This class is stateless and cannot be instantiated. @@ -67,10 +68,21 @@ private QueryResultFormatter() {} * @return the formatted table as a multi-line string; never null or empty */ public static String format(String label, Schema schema, Stream rows) { - return format(label, schema, rows.toList()); + return format(label, schema, rows.toList(), "NULL", false); } - private static String format(String label, Schema schema, List rows) { + /** + * Formats a query's rows, already collected, as a printable ASCII table. + * + * @param label the result's display label; must not be {@code null} + * @param schema the result's output schema; must not be {@code null} + * @param rows the result rows + * @param nullText how a NULL is shown + * @param color whether to colour it for a terminal: bold column names, dim NULLs + * @return the formatted table as a multi-line string; never null or empty + */ + public static String format(String label, Schema schema, List rows, String nullText, + boolean color) { // Closed schemas enumerate their declared columns; open (schema-on-read) // schemas have none, so we discover the column set from the rows — the // ordered union of each document's field names (see DocumentRow). @@ -86,7 +98,7 @@ private static String format(String label, Schema schema, List rows) { } for (Row row : rows) { for (int i = 0; i < ncols; i++) { - int len = row.get(columns.get(i)).asDisplayString().length(); + int len = cell(row.get(columns.get(i)), nullText).length(); widths[i] = Math.min(MAX_COLUMN_WIDTH, Math.max(widths[i], len)); } } @@ -103,7 +115,7 @@ private static String format(String label, Schema schema, List rows) { sb.append(' '); for (int i = 0; i < ncols; i++) { if (i > 0) sb.append(" "); - sb.append(pad(columns.get(i), widths[i], false)); + sb.append(paint(pad(columns.get(i), widths[i], false), BOLD, color)); } sb.append('\n'); @@ -121,9 +133,9 @@ private static String format(String label, Schema schema, List rows) { for (int i = 0; i < ncols; i++) { if (i > 0) sb.append(" "); var value = row.get(columns.get(i)); - String cell = truncate(value.asDisplayString(), widths[i]); + String cell = truncate(cell(value, nullText), widths[i]); boolean rightAlign = value instanceof NumberValue; - sb.append(pad(cell, widths[i], rightAlign)); + sb.append(paint(pad(cell, widths[i], rightAlign), DIM, color && value.isNull())); } sb.append('\n'); } @@ -136,7 +148,7 @@ private static String format(String label, Schema schema, List rows) { } /** The ordered union of every row's column names (first-seen order). */ - private static List discoverColumns(List rows) { + private static List discoverColumns(List rows) { var columns = new java.util.LinkedHashSet(); for (Row row : rows) { columns.addAll(row.columnNames()); @@ -146,6 +158,23 @@ private static List discoverColumns(List rows) { // ── private helpers ────────────────────────────────────────────────────── + private static final String BOLD = "\u001b[1m"; + private static final String DIM = "\u001b[2m"; + private static final String RESET = "\u001b[0m"; + + /** + * Text in a terminal style, or as it is. Styled after padding, so that the escapes take + * no part in the alignment. + */ + private static String paint(String text, String style, boolean on) { + return on ? style + text + RESET : text; + } + + /** A cell's text, with a NULL shown as {@code nullText}. */ + private static String cell(Value value, String nullText) { + return value.isNull() ? nullText : value.asDisplayString(); + } + /** Pads {@code s} to {@code width} chars, right- or left-aligning. */ private static String pad(String s, int width, boolean rightAlign) { int gap = width - s.length(); diff --git a/src/main/java/com/darkcollective/relix/cli/render/ScriptFormatter.java b/src/main/java/com/darkcollective/relix/cli/render/ScriptFormatter.java new file mode 100644 index 0000000..989cc74 --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/render/ScriptFormatter.java @@ -0,0 +1,169 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.render; + +import com.darkcollective.relix.ast.Spelling; +import com.darkcollective.relix.embed.Relix; +import com.darkcollective.relix.lang.ast.ScriptPrinter; + +import java.util.ArrayList; +import java.util.List; + +/** + * Prints a script canonically: as the engine's script printer writes it, which keeps the + * script's comments, before, inside and after each statement, and its blank lines + * (DarkCollective/relix-core#99), with the columns of its inline tables aligned. + * + *

    Operators are written as glyphs, the printer's default, or with + * {@link Spelling#KEYWORDS} as ASCII keywords. Formatting is idempotent: formatting its + * own output changes nothing. + */ +public final class ScriptFormatter { + + private ScriptFormatter() { + } + + /** + * Formats a script. + * + * @param text the script + * @param source the name to report a parse error against + * @param spelling how operators are written + * @return the script, formatted, ending in one line break + * @throws com.darkcollective.relix.lang.ast.ScriptParseException when it does not parse + */ + public static String format(String text, String source, Spelling spelling) { + return Tables.align(ScriptPrinter.print(Relix.parse(text, source), spelling)); + } + + /** + * Aligns the columns of the inline tables in a printed script: each run of rows the + * printer writes on the lines after a table's opening {@code [}, so that a comment + * whose lines look like rows is left as written. + */ + private static final class Tables { + + private Tables() { + } + + static String align(String printed) { + if (!printed.contains("\n|")) { + return printed; + } + List lines = new ArrayList<>(List.of(printed.split("\n", -1))); + int i = 0; + while (i < lines.size()) { + if (!isRow(lines.get(i)) || i == 0 || !lines.get(i - 1).endsWith("[")) { + i++; + continue; + } + int end = i; + while (end < lines.size() && isRow(lines.get(end))) { + end++; + } + align(lines.subList(i, end)); + i = end; + } + return String.join("\n", lines); + } + + /** Whether a line is a table row: cells between pipes, perhaps a {@code --} comment after. */ + private static boolean isRow(String line) { + String cells = cellsOf(line); + return cells.length() >= 2 && cells.startsWith("|") && cells.endsWith("|"); + } + + /** A row without the {@code --} comment after its last pipe, as the parser reads it. */ + private static String cellsOf(String line) { + int last = line.lastIndexOf('|'); + return last >= 0 && line.substring(last + 1).strip().startsWith("--") + ? line.substring(0, last + 1) + : line; + } + + /** The {@code --} comment after a row's last pipe, or {@code ""}. */ + private static String commentOf(String line) { + return line.substring(cellsOf(line).length()).strip(); + } + + private static void align(List rows) { + List> cells = new ArrayList<>(); + List rules = new ArrayList<>(); + List widths = new ArrayList<>(); + for (String row : rows) { + List split = cells(cellsOf(row)); + boolean rule = split.stream().allMatch(Tables::isRule); + cells.add(split); + rules.add(rule); + for (int c = 0; c < split.size(); c++) { + int width = rule ? 3 : width(split.get(c)); + if (c == widths.size()) { + widths.add(width); + } else { + widths.set(c, Math.max(widths.get(c), width)); + } + } + } + for (int r = 0; r < rows.size(); r++) { + StringBuilder line = new StringBuilder("|"); + List split = cells.get(r); + for (int c = 0; c < split.size(); c++) { + String cell = split.get(c); + if (rules.get(r)) { + line.append("-".repeat(widths.get(c) + 2)).append('|'); + } else { + line.append(' ').append(cell).append(" ".repeat(widths.get(c) - width(cell))).append(" |"); + } + } + String comment = commentOf(rows.get(r)); + if (!comment.isEmpty()) { + line.append(" ").append(comment); + } + rows.set(r, line.toString()); + } + } + + /** A row's cells, trimmed, split at the pipes that are not in a string or escaped. */ + private static List cells(String row) { + List cells = new ArrayList<>(); + StringBuilder cell = new StringBuilder(); + boolean quoted = false; + for (int i = 1; i < row.length(); i++) { + char c = row.charAt(i); + if (c == '\\' && i + 1 < row.length()) { + cell.append(c).append(row.charAt(++i)); + } else if (c == '"') { + quoted = !quoted; + cell.append(c); + } else if (c == '|' && !quoted) { + cells.add(cell.toString().strip()); + cell.setLength(0); + } else { + cell.append(c); + } + } + return cells; + } + + private static boolean isRule(String cell) { + return !cell.isEmpty() && cell.chars().allMatch(ch -> ch == '-' || ch == ':'); + } + + private static int width(String cell) { + return cell.codePointCount(0, cell.length()); + } + } +} diff --git a/src/main/java/com/darkcollective/relix/cli/report/EventFeed.java b/src/main/java/com/darkcollective/relix/cli/report/EventFeed.java new file mode 100644 index 0000000..e20d2ef --- /dev/null +++ b/src/main/java/com/darkcollective/relix/cli/report/EventFeed.java @@ -0,0 +1,147 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.report; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.events.EventMetrics; +import com.darkcollective.relix.events.QueryEvent; + +import java.io.OutputStream; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Locale; +import java.util.Objects; + +/** + * The event feed: what the engine reports while it rewrites, plans and runs a query, one + * line per event, written as each event happens. + * + *

    A line reads {@code [STAGE ] target CODE description}, with the time the event + * measured after it when it measured one. An event that names no target names the query + * it belongs to. A query's feed ends with a {@code ROWS} event of the engine's own shape, + * saying how many rows it delivered and how long that took. + * + *

    {@code relix trace} writes the feed to standard output, and {@code relix run --trace} + * to standard error or a file, beside the rows. + */ +public final class EventFeed implements AutoCloseable { + + private final PrintWriter out; + private final boolean stdout; + private final boolean owned; + + private EventFeed(OutputStream out, boolean stdout, boolean owned) { + this.out = new PrintWriter(Objects.requireNonNull(out, "out"), false, StandardCharsets.UTF_8); + this.stdout = stdout; + this.owned = owned; + } + + /** + * A feed on standard output, whose reader going away ends the run with + * {@link ExitCode#PIPE_CLOSED}. + * + * @param out standard output + * @return the feed + */ + public static EventFeed stdout(OutputStream out) { + return new EventFeed(out, true, false); + } + + /** + * A feed on standard error, which closing the feed leaves open. + * + * @param err standard error + * @return the feed + */ + public static EventFeed stderr(OutputStream err) { + return new EventFeed(err, false, false); + } + + /** + * A feed written to a file, which closing the feed closes. + * + * @param file the file, open + * @return the feed + */ + public static EventFeed file(OutputStream file) { + return new EventFeed(file, false, true); + } + + /** + * Writes one event. + * + * @param event the event + * @param query the label of the query it belongs to, for an event that names no target + */ + public void event(QueryEvent event, String query) { + write(line(event.target().isPresent() ? event : event.withTarget(query))); + } + + /** + * Writes the event that ends a query's feed. + * + * @param query the query's label + * @param delivered how many rows it delivered + * @param elapsed how long it took + */ + public void delivered(String query, long delivered, Duration elapsed) { + event(QueryEvent.of(QueryEvent.Stage.EXECUTE, "ROWS", + "query delivered " + delivered + " row" + (delivered == 1 ? "" : "s"), query) + .withMetrics(EventMetrics.of(delivered, elapsed)), query); + } + + /** + * One event as a line of the feed, without its line break. + * + * @param event the event + * @return the line + */ + public static String line(QueryEvent event) { + String target = event.target().map(t -> " " + t).orElse(""); + return String.format(Locale.ROOT, "[%-8s]%s %s %s%s", + event.stage(), target, event.code(), event.description(), elapsed(event)); + } + + /** + * The time an event measured, as a trailing {@code (12.3 ms)}, or nothing for an + * event that measured none. It is rendered from the event's metrics rather than its + * description, which stays free of wall-clock readings a test could not assert on. + */ + private static String elapsed(QueryEvent event) { + return event.metrics().duration() + .map(d -> String.format(Locale.ROOT, " (%.1f ms)", d.toNanos() / 1_000_000.0)) + .orElse(""); + } + + /** Flushes the feed, and closes it when it is a file's. */ + @Override + public void close() { + out.flush(); + if (owned) { + out.close(); + } + } + + private void write(String line) { + out.print(line + "\n"); + out.flush(); + if (stdout && out.checkError()) { + throw new CommandFailure(ExitCode.PIPE_CLOSED, null); + } + } +} diff --git a/src/packageTest/java/com/darkcollective/relix/cli/packaging/ReleaseArchiveTest.java b/src/packageTest/java/com/darkcollective/relix/cli/packaging/ReleaseArchiveTest.java new file mode 100644 index 0000000..5b9f3fd --- /dev/null +++ b/src/packageTest/java/com/darkcollective/relix/cli/packaging/ReleaseArchiveTest.java @@ -0,0 +1,261 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.packaging; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The release archive's contents, run as a user runs them: through the launcher, in a + * process of its own (design §7.3). What a packaging mistake breaks — a provider jlink + * left out, a driver DriverManager cannot see, a CDS archive the JVM will not map — the + * unit tests cannot see, because they run on the build's class path. + * + *

    {@code relix.release} names the unpacked archive; {@code ./gradlew + * packageTest} builds and unpacks it first. + */ +class ReleaseArchiveTest { + + private static final boolean WINDOWS = System.getProperty("os.name").startsWith("Windows"); + + private final Path release = Path.of(System.getProperty("relix.release")); + + @TempDir + Path tree; + + private Path home; + + @BeforeEach + void home() throws IOException { + home = Files.createDirectories(tree.resolve("home")); + } + + @Test + @DisplayName("runs in a pipe: a script on stdin, a catalog from the directory tree, tsv out") + void pipeline() throws Exception { + Path project = Files.createDirectories(home.resolve("project")); + Path catalog = Files.createDirectories(project.resolve(".relix/catalog")); + // The connection is to an H2 database, so the bundled driver has to be found. + // Its names are folded to lower case, as the engine writes them unquoted. + String url = "jdbc:h2:file:" + project.resolve("shop").toAbsolutePath().toString().replace('\\', '/') + + ";DATABASE_TO_LOWER=TRUE"; + try (Connection connection = DriverManager.getConnection(url); + Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE cities (city VARCHAR(20), country VARCHAR(20))"); + statement.execute("INSERT INTO cities VALUES ('london', 'uk'), ('paris', 'fr')"); + } + Files.writeString(catalog.resolve("people.csv"), "id,name,city\n1,ada,london\n2,bob,paris\n"); + Files.writeString(catalog.resolve("10-shop.relix"), """ + connection shop from database { url: "%s" }; + source People from csv("people.csv") { + header: true, + schema: { id: NUMBER, name: STRING, city: STRING } + }; + """.formatted(url)); + + Run trust = relix(project, "", "catalog", "trust"); + assertThat(trust.status()).as(trust.toString()).isZero(); + + Run run = relix(project, "query { τ name (π name, country (People ⋈ shop.cities)) };\n"); + + assertThat(run.status()).as(run.toString()).isZero(); + assertThat(run.out()).isEqualTo("name\tcountry\nada\tuk\nbob\tfr\n"); + } + + @Test + @DisabledOnOs(OS.WINDOWS) + @DisplayName("runs through a relative symbolic link, as Homebrew installs it") + void symbolicLink() throws Exception { + Path bin = Files.createDirectories(tree.resolve("prefix/bin")).toRealPath(); + Path link = Files.createSymbolicLink(bin.resolve("relix"), + bin.relativize(release.resolve("bin/relix").toRealPath())); + + Run run = run(home, "", List.of(), link, "version"); + + assertThat(run.status()).as(run.toString()).isZero(); + assertThat(run.out()).startsWith("relix "); + } + + @Test + @DisplayName("finds the solver: OPTIMIZE plans") + void solver() throws Exception { + Run run = relix(home, """ + Items := [ + | item | value | weight | + |------|-------|--------| + | keep | 6 | 2 | + | drop | 5 | 3 | + ]; + query { OPTIMIZE MAXIMIZE SUM(value) SUBJECT TO SUM(weight) <= 4 (Items) }; + """); + + assertThat(run.status()).as(run.toString()).isZero(); + assertThat(run.out()).isEqualTo("item\tvalue\tweight\nkeep\t6\t2\n"); + } + + @Test + @DisplayName("finds the reference: doc shows a page") + void reference() throws Exception { + Run run = relix(home, "", "doc", "select"); + + assertThat(run.status()).as(run.toString()).isZero(); + assertThat(run.out()).contains("Selection (σ / SELECT)"); + } + + @Test + @DisplayName("maps its CDS archive: -Xshare:on, which fails without one, starts it from the archive") + void cdsArchive() throws Exception { + // Relative to the run's directory: -Xlog splits on ':', which a Windows path holds. + Run run = relix(home, "", List.of("-Xshare:on", "-Xlog:class+load=info:file=class-load.log"), "version"); + Path log = home.resolve("class-load.log"); + + assertThat(run.status()).as(run.toString()).isZero(); + assertThat(run.out()).startsWith("relix "); + assertThat(Files.readString(log)) + .as("the command's own classes come from the application's archive") + .containsPattern("com\\.darkcollective\\.relix\\.cli\\.Main source: shared objects file \\(top\\)"); + } + + @Test + @DisplayName("-Xshare:on refuses to start without its archive, so the test above can fail") + void cdsArchiveRequired() throws Exception { + Path java = release.resolve("bin").resolve(WINDOWS ? "java.exe" : "java"); + Process process = new ProcessBuilder(java.toString(), "-Xshare:on", + "-XX:SharedArchiveFile=" + tree.resolve("missing.jsa"), "-version") + .redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + + assertThat(process.waitFor(60, TimeUnit.SECONDS)).isTrue(); + assertThat(process.exitValue()).as(output).isNotZero(); + } + + @Test + @DisabledOnOs(OS.WINDOWS) + @DisplayName("ships a man page per command, each of which man renders") + void manPages() throws Exception { + List pages; + try (Stream files = Files.list(release.resolve("man/man1"))) { + pages = files.sorted().toList(); + } + assertThat(pages).extracting(page -> page.getFileName().toString()) + .contains("relix.1", "relix-run.1", "relix-catalog-trust.1", "relix-completion.1"); + + for (Path page : pages) { + // man-db's man takes a file with -l; macOS's (mandoc's) takes one as a path. + List command = OS.LINUX.isCurrentOs() + ? List.of("man", "-l", page.toString()) + : List.of("man", page.toAbsolutePath().toString()); + ProcessBuilder builder = new ProcessBuilder(command).redirectErrorStream(true); + builder.environment().put("MANPAGER", "cat"); + builder.environment().put("MANWIDTH", "80"); + Process process = builder.start(); + process.getOutputStream().close(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8) + .replaceAll(".\b", ""); + + assertThat(process.waitFor(60, TimeUnit.SECONDS)).isTrue(); + assertThat(process.exitValue()).as(page + "\n" + output).isZero(); + assertThat(output).as(page.toString()).contains("NAME", "SYNOPSIS", "relix"); + } + } + + @Test + @DisplayName("ships a completion script per shell, zsh's loadable from $fpath") + void completions() throws IOException { + Path completions = release.resolve("completions"); + + for (String file : List.of("relix.bash", "_relix", "relix.fish", "relix.ps1")) { + assertThat(completions.resolve(file)).isNotEmptyFile(); + } + assertThat(Files.readAllLines(completions.resolve("_relix")).getFirst()).isEqualTo("#compdef relix"); + } + + // ------------------------------------------------------------------------- + + private record Run(int status, String out, String err) { + } + + private Run relix(Path directory, String stdin, String... args) throws Exception { + return relix(directory, stdin, List.of(), args); + } + + /** + * Runs the archive's launcher in {@code directory}, with {@code stdin} on standard + * input, the temporary home as its home, and {@code jvmOptions} for its JVM. + */ + private Run relix(Path directory, String stdin, List jvmOptions, String... args) throws Exception { + return run(directory, stdin, jvmOptions, release.resolve("bin").resolve(WINDOWS ? "relix.bat" : "relix"), args); + } + + private Run run(Path directory, String stdin, List jvmOptions, Path launcher, String... args) + throws Exception { + List command = new ArrayList<>(List.of(launcher.toString())); + command.addAll(List.of(args)); + ProcessBuilder builder = new ProcessBuilder(command).directory(directory.toFile()); + Map environment = builder.environment(); + Map kept = new HashMap<>(); + for (String name : List.of("PATH", "SystemRoot", "TEMP", "TMP", "COMSPEC", "PATHEXT")) { + if (environment.containsKey(name)) { + kept.put(name, environment.get(name)); + } + } + environment.clear(); + environment.putAll(kept); + environment.put("HOME", home.toString()); + List options = new ArrayList<>(List.of("-Duser.home=" + home)); + options.addAll(jvmOptions); + environment.put("JDK_JAVA_OPTIONS", String.join(" ", options)); + Process process = builder.start(); + CompletableFuture out = CompletableFuture.supplyAsync(() -> read(process, true)); + CompletableFuture err = CompletableFuture.supplyAsync(() -> read(process, false)); + try (OutputStream in = process.getOutputStream()) { + in.write(stdin.getBytes(StandardCharsets.UTF_8)); + } + assertThat(process.waitFor(120, TimeUnit.SECONDS)).as("relix finished").isTrue(); + return new Run(process.exitValue(), new String(out.get(), StandardCharsets.UTF_8), + new String(err.get(), StandardCharsets.UTF_8)); + } + + private static byte[] read(Process process, boolean out) { + try { + return (out ? process.getInputStream() : process.getErrorStream()).readAllBytes(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java b/src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java new file mode 100644 index 0000000..cff1f97 --- /dev/null +++ b/src/packaging/java/com/darkcollective/relix/cli/packaging/CommandPages.java @@ -0,0 +1,420 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.packaging; + +import com.darkcollective.relix.cli.command.RelixCommand; +import com.darkcollective.relix.cli.io.Host; +import com.darkcollective.relix.cli.io.Interruption; +import picocli.CommandLine; +import picocli.CommandLine.Help; +import picocli.CommandLine.Model.ArgSpec; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Model.OptionSpec; +import picocli.CommandLine.Model.PositionalParamSpec; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Writes the command reference of the programming guide's command-line section: a + * Markdown page per command, from the same picocli model {@code --help} and the man pages + * are printed from, so the three cannot drift (design §7.4), and {@code index.json}, the + * section's machine-readable index. + * + *

    A page is the command's description, usage, arguments, options, subcommands and exit + * status, then the hand-written part from {@code docs/command-examples/.md}: its + * examples, which the examples test runs like every other page's. The root command's page, + * {@code relix.md}, holds the global options every command takes; the others link to it + * rather than repeat them. + * + *

    The pages are checked in, so that a change to the command's options shows up in its + * documentation in the same diff; {@code CommandPagesTest} fails when they are stale, and + * {@code ./gradlew commandPages} rewrites them. + */ +public final class CommandPages { + + /** The section's pages, relative to the repository. */ + public static final String GUIDE = "docs/guide/command-line"; + + /** The hand-written part of each command page, relative to the repository. */ + public static final String EXAMPLES = "docs/command-examples"; + + /** Where the command pages go, relative to the section. */ + public static final String COMMANDS = "commands"; + + /** The section's machine-readable index, relative to the section. */ + public static final String INDEX = "index.json"; + + /** The width the usage synopsis is wrapped to. */ + private static final int WIDTH = 80; + + private static final Pattern INDEX_CHAPTER = Pattern.compile("^##\\s+(.+?)\\s*$"); + private static final Pattern INDEX_TITLE = Pattern.compile("^#\\s+(.+?)\\s*$"); + private static final Pattern LINK = Pattern.compile("\\[([^\\]]+)]\\(([^)#\\s]+\\.md)\\)"); + + private CommandPages() { + } + + /** + * Writes the pages and the index. + * + * @param args the repository's directory + * @throws IOException if one cannot be written + */ + public static void main(String[] args) throws IOException { + if (args.length != 1) { + throw new IllegalArgumentException("usage: CommandPages REPOSITORY"); + } + Path repository = Path.of(args[0]); + Path guide = repository.resolve(GUIDE); + Map files = generate(repository.resolve(EXAMPLES), guide); + Path commands = guide.resolve(COMMANDS); + Files.createDirectories(commands); + try (var stale = Files.list(commands)) { + for (Path file : stale.toList()) { + if (!files.containsKey(COMMANDS + "/" + file.getFileName())) { + Files.delete(file); + } + } + } + for (Map.Entry file : files.entrySet()) { + Files.writeString(guide.resolve(file.getKey()), file.getValue(), StandardCharsets.UTF_8); + } + } + + /** + * Every file this writes, by its path relative to the section. + * + * @param examples the directory of the command pages' hand-written parts + * @param guide the section, whose {@code README.md} the index is read from + * @return the command pages and {@code index.json} + */ + public static Map generate(Path examples, Path guide) { + CommandSpec root = new CommandLine(new RelixCommand(Host.system(), new Interruption())).getCommandSpec(); + Map files = new LinkedHashMap<>(); + List commands = new ArrayList<>(); + commands.add(root); + commands.addAll(visible(root)); + for (CommandSpec command : commands) { + String name = fileName(command); + files.put(COMMANDS + "/" + name, page(command, root, examples.resolve(name))); + } + files.put(INDEX, index(guide.resolve("README.md"), commands)); + return files; + } + + // ── a command's page ───────────────────────────────────────────────────────────── + + private static String page(CommandSpec command, CommandSpec root, Path examples) { + boolean isRoot = command == root; + StringBuilder md = new StringBuilder(); + md.append("# ").append(command.qualifiedName()).append("\n\n"); + description(md, command); + section(md, command, "##", isRoot); + if (isRoot) { + md.append("## Global options\n\n"); + md.append("Every command takes these, before or after its name.\n\n"); + options(md, global(root)); + md.append("## Commands\n\n"); + md.append("| Command | What it does |\n|---|---|\n"); + for (CommandSpec sub : visible(root)) { + md.append("| [`").append(sub.qualifiedName()).append("`](").append(fileName(sub)).append(") | ") + .append(cell(firstSentence(sub))).append(" |\n"); + } + md.append('\n'); + exitStatus(md, root, "##"); + } + for (CommandSpec sub : visible(command)) { + if (isRoot) { + break; + } + md.append("## ").append(sub.qualifiedName()).append("\n\n"); + description(md, sub); + section(md, sub, "###", false); + } + String handWritten = read(examples); + if (handWritten != null) { + md.append(handWritten.strip()).append('\n'); + } + // picocli wraps the synopsis with the platform's line separator; a page has LF + // endings everywhere, as it is checked in. + return md.toString().replace("\r\n", "\n").replaceAll("\n{3,}", "\n\n").strip() + "\n"; + } + + /** Usage, arguments, options and exit status, under headings of {@code level}. */ + private static void section(StringBuilder md, CommandSpec command, String level, boolean isRoot) { + md.append(level).append(" Usage").append("\n\n```text\n") + .append(synopsis(command)).append("```\n\n"); + List positionals = command.positionalParameters().stream() + .filter(p -> !p.hidden()).toList(); + if (!positionals.isEmpty()) { + md.append(level).append(" Arguments\n\n| Argument | Description |\n|---|---|\n"); + for (PositionalParamSpec p : positionals) { + md.append("| `").append(p.paramLabel()).append(p.arity().max() > 1 ? "...`" : "`").append(" | ") + .append(cell(description(p))).append(" |\n"); + } + md.append('\n'); + } + List own = command.options().stream() + .filter(o -> !o.hidden() && !o.inherited() && !o.usageHelp() && !o.versionHelp()) + .filter(o -> !isRoot || !global(command).contains(o)) + .toList(); + if (!own.isEmpty()) { + md.append(level).append(isRoot ? " Run options" : " Options").append("\n\n"); + if (isRoot) { + md.append("With no command, `relix` is `relix run`, and takes its options:\n\n"); + } + options(md, own); + } + if (!isRoot) { + md.append("It also takes the [global options](relix.md#global-options).\n\n"); + } + if (!isRoot) { + exitStatus(md, command, level); + } + } + + private static void exitStatus(StringBuilder md, CommandSpec command, String level) { + Map exits = command.usageMessage().exitCodeList(); + if (!exits.isEmpty()) { + md.append(level).append(" Exit status\n\n| Status | Meaning |\n|---|---|\n"); + exits.forEach((code, meaning) -> md.append("| ").append(code.strip()).append(" | ") + .append(cell(meaning)).append(" |\n")); + md.append('\n'); + } + } + + private static void options(StringBuilder md, List options) { + md.append("| Option | Description |\n|---|---|\n"); + for (OptionSpec o : options) { + md.append("| ").append(optionNames(o)).append(" | ").append(cell(description(o))).append(" |\n"); + } + md.append('\n'); + } + + private static String optionNames(OptionSpec option) { + String label = option.typeInfo().isBoolean() || option.arity().max() == 0 ? "" : option.paramLabel(); + List names = new ArrayList<>(); + for (String name : option.names()) { + boolean last = name.equals(option.longestName()); + if (label.isEmpty() || !last) { + names.add("`" + name + "`"); + } else { + String separator = name.startsWith("--") ? "=" : " "; + names.add("`" + name + separator + label + "`"); + } + } + return String.join(", ", names); + } + + private static List global(CommandSpec root) { + CommandSpec mixin = root.mixins().get("options"); + Set names = mixin.options().stream().map(OptionSpec::longestName).collect(Collectors.toSet()); + List global = new ArrayList<>(); + for (OptionSpec o : root.options()) { + if (names.contains(o.longestName()) || o.usageHelp() || o.versionHelp()) { + global.add(o); + } + } + return global; + } + + private static void description(StringBuilder md, CommandSpec command) { + List code = new ArrayList<>(); + List prose = new ArrayList<>(); + for (String line : command.usageMessage().description()) { + for (String part : line.split("%n", -1)) { + if (part.startsWith(" ") && !part.isBlank()) { + flush(md, prose); + code.add(part.strip()); + continue; + } + if (!code.isEmpty()) { + md.append("```text\n").append(String.join("\n", code)).append("\n```\n\n"); + code.clear(); + } + if (part.isBlank()) { + flush(md, prose); + } else { + prose.add(part.strip()); + } + } + } + flush(md, prose); + if (!code.isEmpty()) { + md.append("```text\n").append(String.join("\n", code)).append("\n```\n\n"); + } + } + + private static void flush(StringBuilder md, List prose) { + if (!prose.isEmpty()) { + md.append(String.join(" ", prose)).append("\n\n"); + prose.clear(); + } + } + + private static String synopsis(CommandSpec command) { + command.usageMessage().autoWidth(false).width(WIDTH).abbreviateSynopsis(true); + Help help = new Help(command, Help.defaultColorScheme(Help.Ansi.OFF)); + return help.synopsis(0); + } + + private static String description(ArgSpec arg) { + return String.join(" ", arg.description()).replaceAll("\\s+", " ").strip(); + } + + private static String firstSentence(CommandSpec command) { + String text = String.join(" ", command.usageMessage().description()) + .replace("%n", " ").replaceAll("\\s+", " ").strip(); + int stop = text.indexOf(". "); + String sentence = stop < 0 ? text : text.substring(0, stop + 1); + return sentence.endsWith(".") ? sentence : sentence + "."; + } + + private static String cell(String text) { + return text.replace("|", "\\|"); + } + + private static List visible(CommandSpec command) { + return command.subcommands().entrySet().stream() + .filter(e -> e.getKey().equals(e.getValue().getCommandSpec().name())) + .map(e -> e.getValue().getCommandSpec()) + .filter(c -> !c.usageMessage().hidden()) + .toList(); + } + + /** {@code relix.md} for the root, {@code run.md} for {@code relix run}. */ + private static String fileName(CommandSpec command) { + return command.name() + ".md"; + } + + // ── index.json ─────────────────────────────────────────────────────────────────── + + /** + * The section's index as JSON: each page in reading order, with its chapter, its title + * and what the index says it covers, and for a command page the command and its + * subcommands, each with the anchor of its section. + */ + private static String index(Path readme, List commands) { + StringBuilder json = new StringBuilder(); + json.append("{\n \"title\": ").append(string(title(readme))).append(",\n \"pages\": [\n"); + List entries = new ArrayList<>(); + Map byFile = new LinkedHashMap<>(); + for (CommandSpec command : commands) { + byFile.put(COMMANDS + "/" + fileName(command), command); + } + String chapter = null; + for (String line : lines(readme)) { + Matcher heading = INDEX_CHAPTER.matcher(line); + if (heading.matches()) { + chapter = heading.group(1); + continue; + } + Matcher link = LINK.matcher(line); + if (chapter == null || !line.startsWith("|") || !link.find()) { + continue; + } + String path = link.group(2); + String title = link.group(1).replace("`", ""); + String[] cells = line.strip().split("(? 2 ? cells[2].strip().replace("\\|", "|") : ""; + StringBuilder entry = new StringBuilder(); + entry.append(" {\"path\": ").append(string(path)) + .append(", \"chapter\": ").append(string(chapter)) + .append(", \"title\": ").append(string(title)) + .append(", \"summary\": ").append(string(summary)); + CommandSpec command = byFile.get(path); + if (command != null) { + entry.append(", \"command\": ").append(string(command.qualifiedName())); + List subs = visible(command).stream() + .filter(c -> command.parent() != null) + .map(c -> "{\"command\": " + string(c.qualifiedName()) + ", \"anchor\": " + + string(slug(c.qualifiedName())) + "}") + .toList(); + if (!subs.isEmpty()) { + entry.append(", \"subcommands\": [").append(String.join(", ", subs)).append(']'); + } + } + entry.append('}'); + entries.add(entry.toString()); + } + json.append(String.join(",\n", entries)).append("\n ]\n}\n"); + return json.toString(); + } + + private static String title(Path readme) { + for (String line : lines(readme)) { + Matcher title = INDEX_TITLE.matcher(line); + if (title.matches()) { + return title.group(1); + } + } + throw new IllegalStateException(readme + " has no title"); + } + + private static String slug(String text) { + return text.toLowerCase(java.util.Locale.ROOT).replaceAll("[^a-z0-9]+", "-").replaceAll("(^-|-$)", ""); + } + + private static String string(String text) { + StringBuilder out = new StringBuilder("\""); + for (char c : text.toCharArray()) { + switch (c) { + case '"' -> out.append("\\\""); + case '\\' -> out.append("\\\\"); + case '\n' -> out.append("\\n"); + default -> { + if (c < 0x20) { + out.append(String.format("\\u%04x", (int) c)); + } else { + out.append(c); + } + } + } + } + return out.append('"').toString(); + } + + private static List lines(Path file) { + try { + return Files.readAllLines(file, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException("cannot read " + file, e); + } + } + + private static String read(Path file) { + if (!Files.isRegularFile(file)) { + return null; + } + try { + return Files.readString(file, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException("cannot read " + file, e); + } + } +} diff --git a/src/packaging/java/com/darkcollective/relix/cli/packaging/ReleaseFiles.java b/src/packaging/java/com/darkcollective/relix/cli/packaging/ReleaseFiles.java new file mode 100644 index 0000000..7ab53e6 --- /dev/null +++ b/src/packaging/java/com/darkcollective/relix/cli/packaging/ReleaseFiles.java @@ -0,0 +1,116 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.packaging; + +import com.darkcollective.relix.cli.Main; +import com.darkcollective.relix.cli.command.RelixCommand; +import com.darkcollective.relix.cli.io.Host; +import com.darkcollective.relix.cli.io.Interruption; +import org.asciidoctor.Asciidoctor; +import org.asciidoctor.Options; +import org.asciidoctor.SafeMode; +import picocli.CommandLine; +import picocli.codegen.docgen.manpage.ManPageGenerator; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +/** + * Writes what the release archive carries beside the image: a man page per command and a + * completion script per shell, both from the command's picocli model, so that neither can + * drift from what {@code --help} prints (design §7.3). It runs in the build and is not + * shipped. + * + *

    The man pages are picocli's {@code ManPageGenerator} AsciiDoc, converted to troff by + * Asciidoctor's {@code manpage} backend. + */ +public final class ReleaseFiles { + + /** Each shell's script, and the name its file has in the archive. */ + private static final Map COMPLETIONS = Map.of( + "bash", "relix.bash", + "zsh", "_relix", + "fish", "relix.fish", + "powershell", "relix.ps1"); + + private ReleaseFiles() { + } + + /** + * Writes the man pages and the completion scripts. + * + * @param args the directory for the man pages' AsciiDoc, the directory for the pages + * themselves ({@code man1}), and the directory for the completion scripts + * @throws IOException if one cannot be written + */ + public static void main(String[] args) throws IOException { + if (args.length != 3) { + throw new IllegalArgumentException("usage: ReleaseFiles ADOC_DIR MAN1_DIR COMPLETIONS_DIR"); + } + manPages(Path.of(args[0]), Path.of(args[1])); + completions(Path.of(args[2])); + } + + private static void manPages(Path adoc, Path man1) throws IOException { + Files.createDirectories(adoc); + Files.createDirectories(man1); + CommandLine command = new CommandLine(new RelixCommand(Host.system(), new Interruption())); + int status = ManPageGenerator.generateManPage(adoc.toFile(), null, new boolean[0], true, + command.getCommandSpec()); + if (status != 0) { + throw new IOException("ManPageGenerator exited " + status); + } + List pages; + try (Stream files = Files.list(adoc)) { + pages = files.filter(file -> file.toString().endsWith(".adoc")).sorted().toList(); + } + try (Asciidoctor asciidoctor = Asciidoctor.Factory.create()) { + for (Path page : pages) { + asciidoctor.convertFile(page.toFile(), Options.builder() + .backend("manpage") + .safe(SafeMode.UNSAFE) + .toDir(man1.toFile()) + .mkDirs(true) + .build()); + } + } + } + + private static void completions(Path directory) throws IOException { + Files.createDirectories(directory); + for (Map.Entry shell : COMPLETIONS.entrySet()) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayOutputStream err = new ByteArrayOutputStream(); + Host host = new Host(new ByteArrayInputStream(new byte[0]), out, + new PrintStream(err, true, StandardCharsets.UTF_8), Map.of(), + Path.of("").toAbsolutePath(), Path.of(System.getProperty("user.home")), false, false); + int status = Main.run(host, "completion", shell.getKey()); + if (status != 0) { + throw new IOException("relix completion " + shell.getKey() + " exited " + status + ": " + + err.toString(StandardCharsets.UTF_8)); + } + Files.write(directory.resolve(shell.getValue()), out.toByteArray()); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/Cli.java b/src/test/java/com/darkcollective/relix/cli/Cli.java index efce818..22ed4e7 100644 --- a/src/test/java/com/darkcollective/relix/cli/Cli.java +++ b/src/test/java/com/darkcollective/relix/cli/Cli.java @@ -16,9 +16,12 @@ package com.darkcollective.relix.cli; import com.darkcollective.relix.cli.io.Host; +import com.darkcollective.relix.cli.io.Interruption; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; import java.io.PrintStream; import java.nio.charset.StandardCharsets; import java.nio.file.Path; @@ -33,7 +36,10 @@ public final class Cli { private final Path workingDirectory; private Path home; - private String stdin; + private InputStream stdin; + private boolean stdoutIsTerminal = true; + private OutputStream stdout; + private Interruption interruption = new Interruption(); private final Map environment = new HashMap<>(); private Cli(Path workingDirectory) { @@ -43,7 +49,7 @@ private Cli(Path workingDirectory) { /** * A command started in {@code workingDirectory}, which is also its home, with a - * terminal on standard input and an empty environment. + * terminal on standard input and standard output and an empty environment. * * @param workingDirectory where it starts * @return the command, to configure and run @@ -70,7 +76,50 @@ public Cli home(Path home) { * @return this */ public Cli stdin(String text) { - this.stdin = text; + return stdin(new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8))); + } + + /** + * Connects standard input to {@code in}, which is then not a terminal. + * + * @param in what standard input reads + * @return this + */ + public Cli stdin(InputStream in) { + this.stdin = in; + return this; + } + + /** + * Makes standard output a pipe rather than a terminal. + * + * @return this + */ + public Cli piped() { + this.stdoutIsTerminal = false; + return this; + } + + /** + * Sends standard output to {@code out}, a pipe, as well as capturing it. + * + * @param out the reader at the other end; a write it refuses is a closed pipe + * @return this + */ + public Cli stdout(OutputStream out) { + this.stdout = out; + return piped(); + } + + /** + * Runs with {@code interruption} standing for the shutdown hook, so a test can + * interrupt the run as {@code SIGINT} would. + * + * @param interruption what an interrupt stops + * @return this + */ + public Cli interruption(Interruption interruption) { + this.interruption = interruption; return this; } @@ -95,15 +144,43 @@ public Cli env(String name, String value) { public Result run(String... args) { ByteArrayOutputStream out = new ByteArrayOutputStream(); ByteArrayOutputStream err = new ByteArrayOutputStream(); - byte[] in = stdin == null ? new byte[0] : stdin.getBytes(StandardCharsets.UTF_8); - Host host = new Host(new ByteArrayInputStream(in), - new PrintStream(out, true, StandardCharsets.UTF_8), + InputStream in = stdin == null ? new ByteArrayInputStream(new byte[0]) : stdin; + OutputStream target = stdout == null ? out : new Tee(stdout, out); + Host host = new Host(in, target, new PrintStream(err, true, StandardCharsets.UTF_8), - environment, workingDirectory, home, stdin == null); - int status = Main.run(host, args); + environment, workingDirectory, home, stdin == null, stdoutIsTerminal); + int status = Main.run(host, interruption, args); return new Result(status, out.toString(StandardCharsets.UTF_8), err.toString(StandardCharsets.UTF_8)); } + /** Writes to the reader first, and captures only what it accepted. */ + private static final class Tee extends OutputStream { + private final OutputStream reader; + private final OutputStream capture; + + Tee(OutputStream reader, OutputStream capture) { + this.reader = reader; + this.capture = capture; + } + + @Override + public void write(int b) throws java.io.IOException { + reader.write(b); + capture.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws java.io.IOException { + reader.write(b, off, len); + capture.write(b, off, len); + } + + @Override + public void flush() throws java.io.IOException { + reader.flush(); + } + } + /** * What a run did. * diff --git a/src/test/java/com/darkcollective/relix/cli/Examples.java b/src/test/java/com/darkcollective/relix/cli/Examples.java new file mode 100644 index 0000000..0f8815b --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/Examples.java @@ -0,0 +1,76 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** + * The example scripts under {@code src/test/resources/examples}: real scripts, with + * comments, inline tables, views and several queries each, that need nothing but + * themselves to run. None reaches the network: a test over a live service fails when the + * service does. + */ +public final class Examples { + + /** Every example, by file name. */ + public static final List NAMES = + List.of("introspection.relix", "league.relix", "library.relix"); + + private Examples() { + } + + /** + * One example's text. + * + * @param name its file name + * @return the script + */ + public static String text(String name) { + try (InputStream in = Examples.class.getResourceAsStream("/examples/" + name)) { + if (in == null) { + throw new IllegalArgumentException("no example " + name); + } + return new String(in.readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Copies every example into a directory. + * + * @param directory where to put them + * @return their paths, in {@link #NAMES}' order + */ + public static List copyTo(Path directory) { + List copies = new ArrayList<>(); + for (String name : NAMES) { + try { + copies.add(Files.writeString(directory.resolve(name), text(name), StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return copies; + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/command/CatalogDiscoveryTest.java b/src/test/java/com/darkcollective/relix/cli/command/CatalogDiscoveryTest.java new file mode 100644 index 0000000..36cbbe2 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/command/CatalogDiscoveryTest.java @@ -0,0 +1,419 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.Cli; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFilePermissions; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The catalog the {@code .relix/} directories supply (design §5.1, §5.2), end to end + * through {@code Main}: which directories are read, in what order, and which declaration + * of a name wins. + */ +@DisplayName("Catalog discovery") +class CatalogDiscoveryTest { + + @TempDir + Path root; + + /** {@code $HOME}. */ + Path home; + /** {@code ~/work/acme}, a project. */ + Path acme; + /** {@code ~/work/acme/reports}, a sub-project and the working directory. */ + Path reports; + + @BeforeEach + void tree() throws IOException { + home = Files.createDirectories(root.resolve("home")); + acme = Files.createDirectories(home.resolve("work/acme")); + reports = Files.createDirectories(acme.resolve("reports")); + } + + /** A one-row table named {@code name} whose {@code v} is {@code value}. */ + private static String table(String name, String value) { + return name + " := [\n| v |\n|---|\n| " + value + " |\n];\n"; + } + + private static Path write(Path file, String text) throws IOException { + Files.createDirectories(file.getParent()); + return Files.writeString(file, text, StandardCharsets.UTF_8); + } + + private static Path catalog(Path project, String file, String text) throws IOException { + return write(project.resolve(".relix/catalog").resolve(file), text); + } + + /** + * The command in {@code ~/work/acme/reports}, writing tsv, with every directory trusted: + * trust is {@link CatalogTrustTest}'s subject. + */ + private Cli cli() { + return in(reports); + } + + private Cli in(Path dir) { + return Cli.in(dir).home(home).piped().env("RELIX_TRUST_ALL", "1"); + } + + @Nested + @DisplayName("which directories are read") + class Walk { + + @Test + @DisplayName("every level from the working directory up to $HOME, and ~/.relix") + void everyLevel() throws IOException { + catalog(home, "user.relix", table("U", "user")); + catalog(home.resolve("work"), "work.relix", table("W", "work")); + catalog(acme, "acme.relix", table("A", "acme")); + catalog(reports, "reports.relix", table("R", "reports")); + + var result = cli().run("-e", "U ∪ W ∪ A ∪ R"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("v\nuser\nwork\nacme\nreports\n"); + } + + @Test + @DisplayName("nothing above $HOME") + void homeCeiling() throws IOException { + catalog(root, "above.relix", table("Above", "x")); + + var result = cli().run("-e", "Above"); + + assertThat(result.status()).as(result.toString()).isEqualTo(3); + assertThat(result.err()).contains("Above"); + } + + @Test + @DisplayName("a root marker ends the walk at its directory; ~/.relix is still read") + void rootMarker() throws IOException { + catalog(home, "user.relix", table("U", "user")); + catalog(home.resolve("work"), "work.relix", table("W", "work")); + catalog(acme, "acme.relix", table("A", "acme")); + write(acme.resolve(".relix/root"), ""); + + var inside = cli().run("-e", "U ∪ A"); + var above = cli().run("-e", "W"); + + assertThat(inside.status()).as(inside.toString()).isZero(); + assertThat(inside.out()).isEqualTo("v\nuser\nacme\n"); + assertThat(above.status()).as(above.toString()).isEqualTo(3); + } + + @Test + @DisplayName("outside $HOME, only the working directory's .relix/") + void outsideHome() throws IOException { + Path elsewhere = Files.createDirectories(root.resolve("srv/app")); + catalog(root.resolve("srv"), "srv.relix", table("Parent", "p")); + catalog(elsewhere, "app.relix", table("App", "app")); + + var own = in(elsewhere).run("-e", "App"); + var parent = in(elsewhere).run("-e", "Parent"); + + assertThat(own.status()).as(own.toString()).isZero(); + assertThat(own.out()).isEqualTo("v\napp\n"); + assertThat(parent.status()).as(parent.toString()).isEqualTo(3); + } + + @Test + @DisplayName("-C moves where the walk starts") + void minusC() throws IOException { + catalog(acme, "acme.relix", table("A", "acme")); + + var result = in(home).run("-C", "work/acme", "-e", "A"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("v\nacme\n"); + } + + @Test + @DisplayName("only catalog/*.relix is read") + void onlyRelixFiles() throws IOException { + catalog(acme, "notes.txt", "this is not Relix"); + write(acme.resolve(".relix/stray.relix"), "not read either"); + catalog(acme, "acme.relix", table("A", "acme")); + + var result = cli().run("-e", "A"); + + assertThat(result.status()).as(result.toString()).isZero(); + } + } + + @Nested + @DisplayName("which declaration wins") + class Overlay { + + @Test + @DisplayName("within a directory, files load in lexical order, so the later wins") + void lexicalOrder() throws IOException { + catalog(acme, "10-later.relix", table("T", "later")); + catalog(acme, "00-first.relix", table("T", "first")); + + var result = cli().run("-e", "T"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("v\nlater\n"); + } + + @Test + @DisplayName("a nearer level replaces a farther one's declaration, without a duplicate-name error") + void nearestWins() throws IOException { + catalog(home, "user.relix", table("T", "user")); + catalog(acme, "acme.relix", table("T", "acme")); + catalog(reports, "reports.relix", table("T", "reports")); + + var result = cli().run("-e", "T"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("v\nreports\n"); + } + + @Test + @DisplayName("an outer view reads the inner replacement of its source") + void outerViewReadsInnerSource() throws IOException { + catalog(acme, "acme.relix", "Orders := [\n| id | amount |\n|----|--------|\n| 1 | 500 |\n| 2 | 50 |\n];\n" + + "Big := { σ amount > 100 (Orders) };\n"); + catalog(reports, "sample.relix", "Orders := [\n| id | amount |\n|----|--------|\n| 7 | 700 |\n];\n"); + + var inReports = cli().run("-e", "π id (Big)"); + var inAcme = in(acme).run("-e", "π id (Big)"); + + assertThat(inReports.status()).as(inReports.toString()).isZero(); + assertThat(inReports.out()).isEqualTo("id\n7\n"); + assertThat(inAcme.out()).isEqualTo("id\n1\n"); + } + + @Test + @DisplayName("a relative path in a catalog file is read from beside that file") + void relativePathBesideFile() throws IOException { + write(acme.resolve(".relix/catalog/data/orders.csv"), "id,amount\n1,10\n"); + catalog(acme, "orders.relix", "source Orders from csv(\"data/orders.csv\") " + + "{ header: true, schema: { id: NUMBER, amount: NUMBER } };\n"); + + var result = cli().run("-e", "Orders"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("id\tamount\n1\t10\n"); + } + + @Test + @DisplayName("an input bound with -i replaces the catalog's relation of that name") + void inputShadows() throws IOException { + catalog(acme, "acme.relix", "Orders := [\n| id |\n|----|\n| 1 |\n];\nAll := { Orders };\n"); + write(reports.resolve("sample.csv"), "id\n9\n"); + + var result = cli().run("-i", "Orders=sample.csv", "-e", "All"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("id\n9\n"); + } + + @Test + @DisplayName("the script's own declaration replaces the catalog's") + void scriptShadows() throws IOException { + catalog(acme, "acme.relix", table("T", "catalog")); + + var result = cli().run("-e", table("T", "script") + "query { T };"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("v\nscript\n"); + } + } + + @Nested + @DisplayName("files named outside the tree") + class Named { + + @Test + @DisplayName("-N ignores the directories") + void noCatalog() throws IOException { + catalog(acme, "acme.relix", table("A", "acme")); + + var result = cli().run("-N", "-e", "A"); + + assertThat(result.status()).as(result.toString()).isEqualTo(3); + } + + @Test + @DisplayName("--catalog adds a file, nearer than every directory, and loads under -N too") + void catalogOption() throws IOException { + catalog(acme, "acme.relix", table("T", "acme") + table("A", "acme")); + write(root.resolve("extra.relix"), table("T", "extra")); + + var overlaid = cli().run("--catalog=" + root.resolve("extra.relix"), "-e", "T ∪ A"); + var alone = cli().run("-N", "--catalog", root.resolve("extra.relix").toString(), "-e", "T"); + + assertThat(overlaid.status()).as(overlaid.toString()).isZero(); + assertThat(overlaid.out()).isEqualTo("v\nextra\nacme\n"); + assertThat(alone.out()).isEqualTo("v\nextra\n"); + } + + @Test + @DisplayName("$RELIX_CATALOG_PATH adds files, farther than --catalog") + void catalogPath() throws IOException { + write(root.resolve("a.relix"), table("A", "a") + table("T", "a")); + write(root.resolve("b.relix"), table("B", "b") + table("T", "b")); + write(root.resolve("c.relix"), table("T", "c")); + String path = root.resolve("a.relix") + File.pathSeparator + root.resolve("b.relix"); + + var fromPath = cli().env("RELIX_CATALOG_PATH", path).run("-e", "A ∪ B ∪ T"); + var withOption = cli().env("RELIX_CATALOG_PATH", path) + .run("--catalog", root.resolve("c.relix").toString(), "-e", "T"); + + assertThat(fromPath.status()).as(fromPath.toString()).isZero(); + assertThat(fromPath.out()).isEqualTo("v\na\nb\n"); + assertThat(withOption.out()).isEqualTo("v\nc\n"); + } + + @Test + @DisplayName("a --catalog file that is not there is a usage error") + void missingCatalogFile() { + var result = cli().run("--catalog", "nowhere.relix", "-e", "1"); + + assertThat(result.status()).isEqualTo(2); + assertThat(result.err()).contains("--catalog: nowhere.relix: no such file"); + } + } + + @Test + @DisplayName("a piped script resolves catalog names") + void pipedScript() throws IOException { + catalog(acme, "acme.relix", table("A", "acme")); + + var result = cli().stdin("query { A };").run(); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("v\nacme\n"); + } + + @Test + @DisplayName("a catalog file that does not parse fails the run with 3, naming the file") + void catalogDoesNotParse() throws IOException { + Path bad = catalog(acme, "bad.relix", "Orders := [ oops"); + + var result = cli().run("-e", "1"); + + assertThat(result.status()).as(result.toString()).isEqualTo(3); + assertThat(result.err()).contains(bad.toString()); + } + + @Test + @DisplayName("a catalog file may not hold a query") + void catalogHoldsQuery() throws IOException { + catalog(acme, "q.relix", table("T", "x") + "query { T };\n"); + + var result = cli().run("-e", "T"); + + assertThat(result.status()).as(result.toString()).isEqualTo(3); + assertThat(result.err()).contains("a query belongs in a script"); + } + + @Test + @DisplayName("-v names each catalog file read") + void verboseNamesFiles() throws IOException { + Path file = catalog(acme, "acme.relix", table("A", "acme")); + + var result = cli().run("-v", "-e", "A"); + + assertThat(result.err()).contains("relix: catalog " + file); + } + + @Nested + @DisplayName("relixrc") + class Rc { + + /** Two columns, so that csv and tsv differ. */ + static final String PAIR = "P := [\n| a | b |\n|---|---|\n| 1 | 2 |\n];\n"; + + @Test + @DisplayName("sets the output format, the nearest file winning") + void output() throws IOException { + write(acme.resolve(".relix/relixrc"), "# defaults\noutput = ndjson\n"); + write(reports.resolve(".relix/relixrc"), "output = csv\n"); + catalog(acme, "acme.relix", PAIR); + + var result = cli().run("-e", "P"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("a,b\n1,2\n"); + } + + @Test + @DisplayName("-o and $RELIX_OUTPUT beat it") + void optionBeatsIt() throws IOException { + write(acme.resolve(".relix/relixrc"), "output = csv\n"); + catalog(acme, "acme.relix", table("A", "acme")); + + var option = cli().run("-o", "ndjson", "-e", "A"); + var variable = cli().env("RELIX_OUTPUT", "ndjson").run("-e", "A"); + + assertThat(option.out()).isEqualTo("{\"v\":\"acme\"}\n"); + assertThat(variable.out()).isEqualTo("{\"v\":\"acme\"}\n"); + } + + @Test + @DisplayName("sets the profile; -P beats it") + void profile() throws IOException { + write(acme.resolve(".relix/relixrc"), "profile = staging\n"); + Path profiles = write(home.resolve(".relix/profiles.json"), + "{ \"staging\": { \"WHO\": \"stage\" }, \"production\": { \"WHO\": \"prod\" } }"); + if (Files.getFileAttributeView(profiles, PosixFileAttributeView.class) != null) { + Files.setPosixFilePermissions(profiles, PosixFilePermissions.fromString("rw-------")); + } + catalog(acme, "acme.relix", "source Who from csv(\"${WHO}.csv\") { header: true, schema: { v: STRING } };\n"); + write(acme.resolve(".relix/catalog/stage.csv"), "v\nfrom staging\n"); + write(acme.resolve(".relix/catalog/prod.csv"), "v\nfrom production\n"); + + var fromRc = cli().run("-e", "Who"); + var fromOption = cli().run("-P", "production", "-e", "Who"); + + assertThat(fromRc.status()).as(fromRc.toString()).isZero(); + assertThat(fromRc.out()).isEqualTo("v\nfrom staging\n"); + assertThat(fromOption.out()).isEqualTo("v\nfrom production\n"); + } + + @Test + @DisplayName("an unknown key is a warning, and -N ignores the file") + void unknownKey() throws IOException { + write(acme.resolve(".relix/relixrc"), "colour = always\noutput = csv\n"); + write(root.resolve("pair.relix"), PAIR); + String pair = "--catalog=" + root.resolve("pair.relix"); + + var warned = cli().run(pair, "-e", "P"); + var ignored = cli().run("-N", pair, "-e", "P"); + + assertThat(warned.err()).contains("relixrc:1: ignored: unknown key 'colour'"); + assertThat(warned.out()).isEqualTo("a,b\n1,2\n"); + assertThat(ignored.err()).isEmpty(); + assertThat(ignored.out()).isEqualTo("a\tb\n1\t2\n"); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/command/CatalogInspectionTest.java b/src/test/java/com/darkcollective/relix/cli/command/CatalogInspectionTest.java new file mode 100644 index 0000000..67096c9 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/command/CatalogInspectionTest.java @@ -0,0 +1,253 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.Cli; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * {@code relix catalog files}, {@code ls}, {@code schema} and {@code where} (design §5.3), + * end to end through {@code Main}, over three levels: {@code ~/.relix}, a project and an + * untrusted sub-project. + */ +@DisplayName("Catalog inspection") +class CatalogInspectionTest { + + @TempDir + Path root; + + Path home; + Path acme; + Path reports; + + Path base; + Path project; + Path secret; + + @BeforeEach + void tree() throws IOException { + home = Files.createDirectories(root.resolve("home")); + acme = Files.createDirectories(home.resolve("acme")); + reports = Files.createDirectories(acme.resolve("reports")); + base = write(home.resolve(".relix/catalog/base.relix"), + "Orders := [\n| id | amount |\n|----|--------|\n| 1 | 5 |\n];\n" + + "Big := { σ amount > 1 (Orders) };\n"); + project = write(acme.resolve(".relix/catalog/10-acme.relix"), + "-- the project's own Orders\n" + + "source Orders from csv(\"orders.csv\") " + + "{ header: true, schema: { id: NUMBER, amount: NUMBER, note: STRING } };\n" + + "Customers := [\n| cid | name |\n|-----|------|\n| 1 | ada |\n];\n"); + secret = write(reports.resolve(".relix/catalog/secret.relix"), + "Secret := [\n| s |\n|---|\n| 1 |\n];\nOrders := [\n| id |\n|----|\n| 9 |\n];\n"); + var trusted = Cli.in(acme).home(home).run("catalog", "trust"); + assertThat(trusted.status()).as(trusted.toString()).isZero(); + } + + /** A path as tsv writes it: a backslash, Windows' separator, escaped. */ + private static String escaped(Path path) { + return path.toString().replace("\\", "\\\\"); + } + + private static Path write(Path file, String text) throws IOException { + Files.createDirectories(file.getParent()); + return Files.writeString(file, text, StandardCharsets.UTF_8); + } + + /** In the sub-project, whose own level is not trusted, writing tsv. */ + private Cli piped() { + return Cli.in(reports).home(home).piped(); + } + + /** In the sub-project, on a terminal, so writing a table. */ + private Cli terminal() { + return Cli.in(reports).home(home); + } + + @Nested + @DisplayName("files") + class Files_ { + + @Test + @DisplayName("every file in load order, its names and whether it is trusted, as tsv") + void tsv() { + var result = piped().run("catalog", "files"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("file\ttrusted\tdeclares\n" + + escaped(base) + "\ttrue\tOrders Big\n" + + escaped(project) + "\ttrue\tOrders Customers\n" + + escaped(secret) + "\tfalse\tSecret Orders\n"); + assertThat(result.err()).contains("relix catalog trust " + reports); + } + + @Test + @DisplayName("as a table") + void table() { + var result = terminal().run("catalog", "files", "--color=never"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).contains("file").contains("trusted").contains("declares") + .contains("Orders Customers").contains("false").contains("(3 rows)"); + } + + @Test + @DisplayName("-N leaves only the --catalog files") + void noCatalog() throws IOException { + Path extra = write(root.resolve("extra.relix"), "X := [\n| x |\n|---|\n| 1 |\n];\n"); + + var result = piped().run("-N", "--catalog", extra.toString(), "catalog", "files"); + + assertThat(result.out()).isEqualTo("file\ttrusted\tdeclares\n" + escaped(extra) + "\ttrue\tX\n"); + } + } + + @Nested + @DisplayName("ls") + class Ls { + + @Test + @DisplayName("name, kind, arity and the winning file, from the engine's relix.relations, as tsv") + void tsv() { + var result = piped().run("catalog", "ls"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("name\tkind\tarity\tfile\n" + + "Big\tQR\t3\t" + escaped(base) + "\n" + + "Customers\tINL\t2\t" + escaped(project) + "\n" + + "Orders\tSRC\t3\t" + escaped(project) + "\n"); + } + + @Test + @DisplayName("as a table") + void table() { + var result = terminal().run("catalog", "ls", "--color=never"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).contains("── relations").contains("Customers").contains("INL") + .contains("(3 rows)").doesNotContain("Secret"); + } + + @Test + @DisplayName("as ndjson") + void ndjson() { + var result = piped().run("catalog", "ls", "-o", "ndjson"); + + assertThat(result.out()).startsWith("{\"name\":\"Big\",\"kind\":\"QR\",\"arity\":3,\"file\":"); + } + } + + @Nested + @DisplayName("schema") + class Schema { + + @Test + @DisplayName("the winning declaration's columns and types, in order, as tsv") + void tsv() { + var result = piped().run("catalog", "schema", "Orders"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("column\ttype\nid\tN\namount\tN\nnote\tS\n"); + } + + @Test + @DisplayName("of a view, as a table") + void table() { + var result = terminal().run("catalog", "schema", "Big", "--color=never"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).contains("── Big").contains("column").contains("note") + .contains("(3 rows)"); + } + + @Test + @DisplayName("of an unknown name exits 3") + void unknown() { + var result = piped().run("catalog", "schema", "Nowhere"); + + assertThat(result.status()).as(result.toString()).isEqualTo(3); + assertThat(result.out()).isEmpty(); + assertThat(result.err()).contains("the catalog declares no 'Nowhere'"); + } + + @Test + @DisplayName("of a name only an untrusted level declares exits 3") + void untrusted() { + var result = piped().run("catalog", "schema", "Secret"); + + assertThat(result.status()).as(result.toString()).isEqualTo(3); + } + } + + @Nested + @DisplayName("where") + class Where { + + @Test + @DisplayName("the declaration that won, then those it shadowed, and an untrusted one, as tsv") + void tsv() { + var result = piped().run("catalog", "where", "Orders"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("file\tline\tkind\tstate\n" + + escaped(secret) + "\t6\ttable\tuntrusted\n" + + escaped(project) + "\t2\tsource\twins\n" + + escaped(base) + "\t1\ttable\tshadowed\n"); + } + + @Test + @DisplayName("as a table") + void table() { + var result = terminal().run("catalog", "where", "Orders", "--color=never"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).contains("── Orders").contains("wins").contains("shadowed") + .contains("untrusted").contains("(3 rows)"); + } + + @Test + @DisplayName("once the sub-project is trusted, its declaration wins and shadows both") + void trusted() { + assertThat(Cli.in(reports).home(home).run("catalog", "trust").status()).isZero(); + + var result = piped().run("catalog", "where", "Orders"); + + assertThat(result.out()).isEqualTo("file\tline\tkind\tstate\n" + + escaped(secret) + "\t6\ttable\twins\n" + + escaped(project) + "\t2\tsource\tshadowed\n" + + escaped(base) + "\t1\ttable\tshadowed\n"); + } + + @Test + @DisplayName("of an unknown name exits 3") + void unknown() { + var result = piped().run("catalog", "where", "Nowhere"); + + assertThat(result.status()).isEqualTo(3); + assertThat(result.err()).contains("the catalog declares no 'Nowhere'"); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/command/CatalogTrustTest.java b/src/test/java/com/darkcollective/relix/cli/command/CatalogTrustTest.java new file mode 100644 index 0000000..ca4fb82 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/command/CatalogTrustTest.java @@ -0,0 +1,305 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.Cli; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Trusting a project's {@code .relix/} before it is loaded (design §5.4), end to end + * through {@code Main}. + */ +@DisplayName("Catalog trust") +class CatalogTrustTest { + + @TempDir + Path root; + + Path home; + Path acme; + + @BeforeEach + void tree() throws IOException { + home = Files.createDirectories(root.resolve("home")); + acme = Files.createDirectories(home.resolve("work/acme")); + } + + private static String table(String name, String value) { + return name + " := [\n| v |\n|---|\n| " + value + " |\n];\n"; + } + + /** A path as tsv writes it: a backslash, Windows' separator, escaped. */ + private static String escaped(Path path) { + return path.toString().replace("\\", "\\\\"); + } + + private static Path write(Path file, String text) throws IOException { + Files.createDirectories(file.getParent()); + return Files.writeString(file, text, StandardCharsets.UTF_8); + } + + private Path catalog(String file, String text) throws IOException { + return write(acme.resolve(".relix/catalog").resolve(file), text); + } + + private Cli cli() { + return Cli.in(acme).home(home).piped(); + } + + private Cli.Result trust() { + var result = cli().run("catalog", "trust"); + assertThat(result.status()).as(result.toString()).isZero(); + return result; + } + + @Nested + @DisplayName("an untrusted directory") + class Untrusted { + + private HttpServer server; + private final AtomicInteger requests = new AtomicInteger(); + + @BeforeEach + void start() throws IOException { + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/", exchange -> { + requests.incrementAndGet(); + byte[] body = "[{\"v\":\"leaked\"}]".getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.start(); + } + + @AfterEach + void stop() { + server.stop(0); + } + + private void hook() throws IOException { + catalog("hook.relix", "source Hook from http { url: \"http://127.0.0.1:" + + server.getAddress().getPort() + "/collect\", schema: { v: STRING } };\n"); + } + + @Test + @DisplayName("is not defined at all: its connection is never opened, and the run exits 5") + void neverOpened() throws IOException { + hook(); + + var result = cli().run("-e", "Hook"); + + assertThat(requests).hasValue(0); + assertThat(result.status()).as(result.toString()).isEqualTo(5); + assertThat(result.err()) + .contains("relix: warning: skipping " + acme.resolve(".relix") + ", which is not trusted;" + + " to load it: relix catalog trust " + Invocation.shellWord(acme.toString())) + .contains("Hook is declared in " + acme.resolve(".relix/catalog/hook.relix") + + ", which is not trusted; to load it: relix catalog trust " + + Invocation.shellWord(acme.toString())); + } + + @Test + @DisplayName("is loaded once trusted") + void loadedOnceTrusted() throws IOException { + hook(); + trust(); + + var result = cli().run("-e", "Hook"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("v\nleaked\n"); + assertThat(requests.get()).isPositive(); + assertThat(result.err()).doesNotContain("not trusted"); + } + + @Test + @DisplayName("is skipped with one line, and a script that does not need it still runs") + void scriptThatDoesNotNeedIt() throws IOException { + hook(); + catalog("more.relix", table("Other", "x")); + write(root.resolve("mine.relix"), table("Mine", "m")); + + var result = cli().run("--catalog", root.resolve("mine.relix").toString(), "-e", "Mine"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.err().lines()).hasSize(1); + } + + @Test + @DisplayName("an unknown name it does not declare is still the script's mistake: exit 3") + void unrelatedMistake() throws IOException { + hook(); + + var result = cli().run("-e", "Nowhere"); + + assertThat(result.status()).as(result.toString()).isEqualTo(3); + } + + @Test + @DisplayName("supplies no relixrc and no profile values") + void noRelixrcOrProfiles() throws IOException { + write(acme.resolve(".relix/relixrc"), "output = csv\n"); + write(root.resolve("pair.relix"), "P := [\n| a | b |\n|---|---|\n| 1 | 2 |\n];\n"); + + var result = cli().run("--catalog", root.resolve("pair.relix").toString(), "-e", "P"); + + assertThat(result.out()).isEqualTo("a\tb\n1\t2\n"); + } + + @Test + @DisplayName("is loaded under RELIX_TRUST_ALL=1") + void trustAll() throws IOException { + hook(); + + var result = cli().env("RELIX_TRUST_ALL", "1").run("-e", "Hook"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("v\nleaked\n"); + } + } + + @Nested + @DisplayName("a trusted directory") + class Trusted { + + @Test + @DisplayName("is untrusted again when a file is edited, added or removed") + void changeUntrusts() throws IOException { + Path file = catalog("a.relix", table("A", "a")); + + trust(); + assertThat(cli().run("-e", "A").status()).isZero(); + + write(file, table("A", "edited")); + assertThat(cli().run("-e", "A").status()).isEqualTo(5); + + trust(); + catalog("b.relix", table("B", "b")); + assertThat(cli().run("-e", "A").status()).isEqualTo(5); + + trust(); + Files.delete(acme.resolve(".relix/catalog/b.relix")); + assertThat(cli().run("-e", "A").status()).isEqualTo(5); + + trust(); + write(acme.resolve(".relix/relixrc"), "output = csv\n"); + assertThat(cli().run("-e", "A").status()).isEqualTo(5); + } + + @Test + @DisplayName("is trusted by naming the project or its .relix/, from anywhere") + void namedDirectory() throws IOException { + catalog("a.relix", table("A", "a")); + + var byProject = Cli.in(home).home(home).run("catalog", "trust", "work/acme"); + var run = cli().run("-e", "A"); + var byLevel = Cli.in(home).home(home).run("catalog", "trust", acme.resolve(".relix").toString()); + + assertThat(byProject.status()).as(byProject.toString()).isZero(); + assertThat(run.status()).as(run.toString()).isZero(); + assertThat(byLevel.status()).as(byLevel.toString()).isZero(); + } + + @Test + @DisplayName("is no longer trusted once revoked") + void revoke() throws IOException { + catalog("a.relix", table("A", "a")); + trust(); + + var revoked = Cli.in(home).home(home).run("catalog", "trust", "--revoke", "work/acme"); + var run = cli().run("-e", "A"); + var again = Cli.in(home).home(home).run("catalog", "trust", "--revoke", "work/acme"); + + assertThat(revoked.status()).as(revoked.toString()).isZero(); + assertThat(run.status()).as(run.toString()).isEqualTo(5); + assertThat(again.status()).isEqualTo(2); + assertThat(again.err()).contains(Path.of("work", "acme") + " is not trusted"); + } + + @Test + @DisplayName("--list shows each directory and whether it has changed since") + void list() throws IOException { + Path other = Files.createDirectories(home.resolve("work/other")); + write(other.resolve(".relix/catalog/o.relix"), table("O", "o")); + Path file = catalog("a.relix", table("A", "a")); + trust(); + assertThat(Cli.in(other).home(home).run("catalog", "trust").status()).isZero(); + write(file, table("A", "edited")); + + var tsv = cli().run("catalog", "trust", "--list"); + var table = Cli.in(acme).home(home).run("catalog", "trust", "--list", "--color=never"); + + assertThat(tsv.status()).as(tsv.toString()).isZero(); + assertThat(tsv.out()).isEqualTo("directory\tstate\n" + escaped(acme) + "\tchanged\n" + escaped(other) + "\ttrusted\n"); + assertThat(table.out()).contains("directory").contains("changed").contains("(2 rows)"); + } + } + + @Test + @DisplayName("~/.relix is trusted without asking") + void userLevel() throws IOException { + write(home.resolve(".relix/catalog/u.relix"), table("U", "u")); + + var result = cli().run("-e", "U"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.err()).isEmpty(); + } + + @Test + @DisplayName("trusting a directory with no .relix/ is a usage error") + void nothingToTrust() { + var result = cli().run("catalog", "trust"); + + assertThat(result.status()).isEqualTo(2); + assertThat(result.err()).contains("has no .relix directory to trust"); + } + + @Test + @DisplayName("DIR, --list and --revoke go one at a time") + void oneAtATime() { + var result = cli().run("catalog", "trust", "--list", "x"); + + assertThat(result.status()).isEqualTo(2); + } + + @Test + @DisplayName("relix catalog alone prints its usage and exits 2") + void catalogAlone() { + var result = cli().run("catalog"); + + assertThat(result.status()).isEqualTo(2); + assertThat(result.err()).contains("Usage: relix catalog"); + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/command/FmtTest.java b/src/test/java/com/darkcollective/relix/cli/command/FmtTest.java new file mode 100644 index 0000000..208cfc4 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/command/FmtTest.java @@ -0,0 +1,333 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.Cli; +import com.darkcollective.relix.cli.Examples; +import com.darkcollective.relix.embed.Relix; +import com.darkcollective.relix.lang.ast.ScriptPrinter; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.FieldSource; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * {@code relix fmt} (design §3), end to end through {@code Main}: over every example + * script, formatting is idempotent, keeps the statements and keeps the comments. + */ +@DisplayName("relix fmt") +class FmtTest { + + static final List EXAMPLES = Examples.NAMES; + + /** A line comment or a block comment, as the examples write them. */ + private static final Pattern COMMENT = Pattern.compile("--[^\\n]*|/\\*.*?\\*/", Pattern.DOTALL); + + @TempDir + Path dir; + + private String fmt(String text, String... options) { + String[] args = new String[options.length + 1]; + args[0] = "fmt"; + System.arraycopy(options, 0, args, 1, options.length); + var result = Cli.in(dir).stdin(text).run(args); + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.err()).isEmpty(); + return result.out(); + } + + /** The statements a text parses to, each as the printer writes it, which ignores where they were. */ + private static List statements(String text) { + return Relix.parse(text).statements().stream().map(ScriptPrinter::print).toList(); + } + + /** The comments in a text, leaving out its inline tables' rows, whose rules are not comments. */ + private static List comments(String text) { + String code = text.lines().filter(line -> !line.strip().startsWith("|")) + .collect(java.util.stream.Collectors.joining("\n")); + Matcher m = COMMENT.matcher(code); + List found = new java.util.ArrayList<>(); + while (m.find()) { + found.add(m.group().stripTrailing()); + } + return found; + } + + @Nested + @DisplayName("over every example") + class OverTheExamples { + + @ParameterizedTest(name = "{0}") + @FieldSource("com.darkcollective.relix.cli.command.FmtTest#EXAMPLES") + @DisplayName("is idempotent") + void idempotent(String example) { + String once = fmt(Examples.text(example)); + + assertThat(fmt(once)).isEqualTo(once); + } + + @ParameterizedTest(name = "{0}") + @FieldSource("com.darkcollective.relix.cli.command.FmtTest#EXAMPLES") + @DisplayName("parses to the same statements") + void sameStatements(String example) { + String original = Examples.text(example); + + assertThat(statements(fmt(original))).isEqualTo(statements(original)); + assertThat(Relix.parse(fmt(original)).namespace()).isEqualTo(Relix.parse(original).namespace()); + } + + @ParameterizedTest(name = "{0}") + @FieldSource("com.darkcollective.relix.cli.command.FmtTest#EXAMPLES") + @DisplayName("keeps every comment, in order") + void keepsComments(String example) { + String original = Examples.text(example); + + assertThat(comments(fmt(original))).isEqualTo(comments(original)); + } + + @ParameterizedTest(name = "{0}") + @FieldSource("com.darkcollective.relix.cli.command.FmtTest#EXAMPLES") + @DisplayName("--keywords parses to the same statements, and is idempotent") + void keywords(String example) { + String original = Examples.text(example); + String keywords = fmt(original, "--keywords"); + + assertThat(statements(keywords)).isEqualTo(statements(original)); + assertThat(fmt(keywords, "--keywords")).isEqualTo(keywords); + assertThat(comments(keywords)).isEqualTo(comments(original)); + } + + @ParameterizedTest(name = "{0}") + @FieldSource("com.darkcollective.relix.cli.command.FmtTest#EXAMPLES") + @DisplayName("--glyphs turns --keywords' output back into the default's") + void glyphs(String example) { + String original = Examples.text(example); + + assertThat(fmt(fmt(original, "--keywords"), "--glyphs")).isEqualTo(fmt(original)); + } + } + + @Nested + @DisplayName("the layout") + class Layout { + + @Test + @DisplayName("prints each statement canonically, keeping the comments and blank lines around it") + void canonical() { + String text = """ + -- header + namespace demo; -- the namespace + + + N := [| x | + | 2 | + | 10 |]; + /* a view */ + Big := { + σ x > 1 + (N) + }; -- trailing + query Big; + """; + + assertThat(fmt(text)).isEqualTo(""" + -- header + namespace demo; -- the namespace + + N := [ + | x | + |-----| + | 2 | + | 10 | + ]; + /* a view */ + Big := { σ x > 1 (N) }; -- trailing + query Big; + """); + } + + @Test + @DisplayName("keeps a comment at the end of a table's row on that row, its columns aligned") + void rowComment() { + String text = """ + Goals := [ + | id | minute | + |----|--------| + | 1 | 12 | -- the first + | 22 | 5 | + | 3 | 105 | -- and the third + ]; + """; + + assertThat(fmt(text)).isEqualTo(""" + Goals := [ + | id | minute | + |-----|--------| + | 1 | 12 | -- the first + | 22 | 5 | + | 3 | 105 | -- and the third + ]; + """); + } + + @Test + @DisplayName("keeps a blank line between a comment and what follows it") + void blankAfterComment() { + String text = "-- header\n\nnamespace demo;\n-- about N\n\nquery N;\n"; + + assertThat(fmt(text)).isEqualTo(text); + } + + @Test + @DisplayName("formats a statement with a comment inside an expression, keeping the comment") + void innerComment() { + String text = "Big := {\n σ x > 1 -- only the big ones\n (N)\n};\nquery Big ;\n"; + + String formatted = fmt(text); + + assertThat(formatted).doesNotContain(" σ").endsWith("\nquery Big;\n"); + assertThat(comments(formatted)).containsExactly("-- only the big ones"); + assertThat(statements(formatted)).isEqualTo(statements(text)); + assertThat(fmt(formatted)).isEqualTo(formatted); + } + + @Test + @DisplayName("does not take -- in a string, a table cell or a table's rule for a comment") + void notComments() { + String text = "T := [\n| a | b |\n|---|---|\n| 1 | x -- y |\n];\nQ := { σ b = \"--\" (T) };\n"; + + String formatted = fmt(text); + + assertThat(formatted).contains("| 1 | x -- y |").contains("Q := { σ b = \"--\" (T) };"); + assertThat(statements(formatted)).isEqualTo(statements(text)); + } + + @Test + @DisplayName("--keywords writes the ASCII spelling, and leaves glyphs in strings alone") + void keywords() { + String formatted = fmt("Q := { π a (σ a ≠ \"σ\" ∧ a ≥ 1 (T ⋈ U)) };\n", "--keywords"); + + assertThat(formatted).isEqualTo("Q := { PROJECT a (SELECT (a != \"σ\") AND (a >= 1) ((T) JOIN (U))) };\n"); + } + + @Test + @DisplayName("an empty script stays empty, and a comment alone stays") + void empty() { + assertThat(fmt("")).isEmpty(); + assertThat(fmt(" -- just this\n\n")).isEqualTo("-- just this\n"); + } + } + + @Nested + @DisplayName("files") + class Files_ { + + private Path write(String name, String text) throws IOException { + return Files.writeString(dir.resolve(name), text, StandardCharsets.UTF_8); + } + + @Test + @DisplayName("prints each file's formatted text to stdout, changing none") + void prints() throws IOException { + write("a.relix", "query { N } ;\n"); + write("b.relix", "query { M };\n"); + + var result = Cli.in(dir).run("fmt", "a.relix", "b.relix"); + + assertThat(result.status()).isZero(); + assertThat(result.out()).isEqualTo("query { N };\nquery { M };\n"); + assertThat(Files.readString(dir.resolve("a.relix"))).isEqualTo("query { N } ;\n"); + } + + @Test + @DisplayName("-w rewrites the files that change, and only those") + void write() throws IOException { + Path changed = write("a.relix", "query { N } ;\n"); + Path same = write("b.relix", "query { M };\n"); + var before = Files.getLastModifiedTime(same); + + var result = Cli.in(dir).run("fmt", "-w", "a.relix", "b.relix"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEmpty(); + assertThat(Files.readString(changed)).isEqualTo("query { N };\n"); + assertThat(Files.getLastModifiedTime(same)).isEqualTo(before); + } + + @Test + @DisplayName("--check exits 0 when nothing would change") + void checkClean() throws IOException { + write("b.relix", "query { M };\n"); + + var result = Cli.in(dir).run("fmt", "--check", "b.relix"); + + assertThat(result.status()).isZero(); + assertThat(result.out()).isEmpty(); + } + + @Test + @DisplayName("--check names each file that would change, changes none, and exits 1") + void checkDirty() throws IOException { + write("a.relix", "query { N } ;\n"); + write("b.relix", "query { M };\n"); + + var result = Cli.in(dir).run("fmt", "--check", "a.relix", "b.relix"); + + assertThat(result.status()).isEqualTo(1); + assertThat(result.out()).isEqualTo("a.relix\n"); + assertThat(Files.readString(dir.resolve("a.relix"))).isEqualTo("query { N } ;\n"); + } + + @Test + @DisplayName("a file that does not parse is reported at its place, left alone, and exits 3") + void unparsable() throws IOException { + write("bad.relix", "N := [\n| x |\n];\nquery { σ (N };\n"); + write("b.relix", "query { M };\n"); + + var result = Cli.in(dir).run("fmt", "-w", "bad.relix", "b.relix"); + + assertThat(result.status()).isEqualTo(3); + assertThat(result.err()).startsWith("relix: bad.relix:4:"); + assertThat(Files.readString(dir.resolve("b.relix"))).isEqualTo("query { M };\n"); + } + + @Test + @DisplayName("-w with standard input, and -w with --check, are usage errors") + void usage() { + assertThat(Cli.in(dir).stdin("query { N };").run("fmt", "-w").status()).isEqualTo(2); + assertThat(Cli.in(dir).run("fmt", "-w", "--check", "a.relix").status()).isEqualTo(2); + assertThat(Cli.in(dir).run("fmt", "--glyphs", "--keywords", "a.relix").status()).isEqualTo(2); + } + + @Test + @DisplayName("with nothing named and a terminal on stdin, prints usage and exits 2") + void nothing() { + assertThat(Cli.in(dir).run("fmt").status()).isEqualTo(2); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/command/InputTest.java b/src/test/java/com/darkcollective/relix/cli/command/InputTest.java new file mode 100644 index 0000000..ad7143e --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/command/InputTest.java @@ -0,0 +1,344 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.Cli; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Binding data to relation names with {@code -i} (design §3.2), end to end through + * {@code Main}. + */ +@DisplayName("Inputs") +class InputTest { + + /** The same two orders in each format. */ + private static final String CSV = "id,customer\n1,ada\n2,bob\n"; + private static final String TSV = "id\tcustomer\n1\tada\n2\tbob\n"; + private static final String JSON = "[{\"id\":1,\"customer\":\"ada\"},{\"id\":2,\"customer\":\"bob\"}]"; + private static final String NDJSON = "{\"id\":1,\"customer\":\"ada\"}\n{\"id\":2,\"customer\":\"bob\"}\n"; + + @TempDir + Path dir; + + private static String text(String format) { + return switch (format) { + case "csv" -> CSV; + case "tsv" -> TSV; + case "json" -> JSON; + default -> NDJSON; + }; + } + + private void write(String name, String text) throws IOException { + Files.writeString(dir.resolve(name), text, StandardCharsets.UTF_8); + } + + @Nested + @DisplayName("each format") + class Formats { + + @ParameterizedTest(name = "{0} from a file, told by its extension") + @CsvSource({"csv", "tsv", "json", "ndjson"}) + void fromFile(String format) throws IOException { + write("orders." + format, text(format)); + + var result = Cli.in(dir).piped().run("-i", "orders=orders." + format, "-e", "σ id > 1 (orders)"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("id\tcustomer\n2\tbob\n"); + } + + @ParameterizedTest(name = "{0} from standard input") + @CsvSource({"csv", "tsv", "json", "ndjson"}) + void fromStdin(String format) { + var result = Cli.in(dir).piped().stdin(text(format)) + .run("-i", "orders=" + format + ":-", "-e", "σ id > 1 (orders)"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("id\tcustomer\n2\tbob\n"); + } + + @Test + @DisplayName("a named format beats the extension") + void formatBeatsExtension() throws IOException { + write("orders.txt", TSV); + + var result = Cli.in(dir).piped().run("--input=orders=tsv:orders.txt", "-e", "orders"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo(TSV); + } + + @Test + @DisplayName("what relix writes as tsv, relix reads back as it was: tabs, line breaks, backslashes, NULL") + void tsvRoundTrip() { + String awkward = "{\"k\":1,\"s\":\"a\\tb\\nc\\rd\\\\e\"}\n{\"k\":2,\"s\":null}\n"; + var tsv = Cli.in(dir).piped().stdin(awkward).run("-i", "r=ndjson:-", "-e", "r"); + var back = Cli.in(dir).piped().stdin(tsv.out()).run("-i", "r=tsv:-", "-o", "ndjson", "-e", "r"); + + assertThat(tsv.out()).isEqualTo("k\ts\n1\ta\\tb\\nc\\rd\\\\e\n2\t\n"); + assertThat(back.status()).as(back.toString()).isZero(); + assertThat(back.out()).isEqualTo(awkward); + } + + @Test + @DisplayName("inputs join with each other and with the script's own relations") + void join() throws IOException { + write("customers.csv", "customer,city\nada,Leeds\nbob,York\n"); + + var result = Cli.in(dir).piped().stdin(NDJSON).run( + "-i", "orders=ndjson:-", "-i", "customers=customers.csv", + "-e", "Cities := [\n| city | region |\n|---|---|\n| York | north |\n];", + "-e", "π id, region (orders ⋈ customers ⋈ Cities)"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("id\tregion\n2\tnorth\n"); + } + + @Test + @DisplayName("a script file reads a bound input as -e does") + void scriptFile() throws IOException { + write("orders.csv", CSV); + write("count.relix", "query { γ COUNT(*) -> n (orders) };\n"); + + var result = Cli.in(dir).piped().run("-i", "orders=orders.csv", "count.relix"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("n\n2\n"); + } + } + + @Nested + @DisplayName("the heading") + class Heading { + + @Test + @DisplayName("--schema declares it instead of inferring it") + void schema() throws IOException { + write("orders.csv", CSV); + + var inferred = Cli.in(dir).run("-o", "ndjson", "-i", "orders=orders.csv", "-e", "σ id = 1 (orders)"); + var declared = Cli.in(dir).run("-o", "ndjson", "-i", "orders=orders.csv", + "--schema", "orders={ id: STRING, customer: STRING }", "-e", "σ id = '1' (orders)"); + + assertThat(inferred.out()).isEqualTo("{\"id\":1,\"customer\":\"ada\"}\n"); + assertThat(declared.status()).as(declared.toString()).isZero(); + assertThat(declared.out()).isEqualTo("{\"id\":\"1\",\"customer\":\"ada\"}\n"); + } + + @Test + @DisplayName("--infer-rows says how many records the heading is inferred from") + void inferRows() { + String mixed = "{\"v\":1}\n{\"v\":\"x\"}\n"; + + var wide = Cli.in(dir).piped().stdin(mixed).run("-i", "r=ndjson:-", "-e", "r"); + var narrow = Cli.in(dir).piped().stdin(mixed).run("-i", "r=ndjson:-", "--infer-rows=1", "-e", "r"); + + assertThat(wide.status()).as(wide.toString()).isZero(); + assertThat(narrow.status()).as(narrow.toString()).isEqualTo(4); + assertThat(narrow.err()).startsWith("relix: -e: "); + } + + @Test + @DisplayName("a malformed --schema is a usage error") + void badSchema() throws IOException { + write("orders.csv", CSV); + + var result = Cli.in(dir).run("-i", "orders=orders.csv", "--schema", "orders=id NUMBER", "-e", "orders"); + + assertThat(result.status()).isEqualTo(2); + assertThat(result.err()).contains("--schema orders: expected a heading"); + } + + @Test + @DisplayName("text that cannot be read as its format fails as execution does, naming the input") + void unreadable() { + var result = Cli.in(dir).piped().stdin("not json at all\n").run("-i", "r=json:-", "-e", "r"); + + assertThat(result.status()).as(result.toString()).isEqualTo(4); + assertThat(result.err()).startsWith("relix: -e: -i r=-: "); + } + } + + @Nested + @DisplayName("usage errors") + class Usage { + + private void assertUsage(Cli.Result result, String message) { + assertThat(result.status()).as(result.toString()).isEqualTo(2); + assertThat(result.out()).isEmpty(); + assertThat(result.err()).contains(message); + } + + @Test + @DisplayName("standard input with no format") + void stdinNeedsFormat() { + assertUsage(Cli.in(dir).stdin(CSV).run("-i", "r=-", "-e", "r"), + "-i r=-: standard input has no extension to tell its format by"); + } + + @Test + @DisplayName("a file whose extension tells no format") + void unknownExtension() throws IOException { + write("orders.txt", CSV); + + assertUsage(Cli.in(dir).run("-i", "r=orders.txt", "-e", "r"), "cannot tell the format from the extension .txt"); + } + + @Test + @DisplayName("a file that is not there") + void missingFile() { + assertUsage(Cli.in(dir).run("-i", "r=nope.csv", "-e", "r"), "-i r=nope.csv: nope.csv: no such file"); + } + + @Test + @DisplayName("a binding with no name or no location") + void malformed() { + assertUsage(Cli.in(dir).run("-i", "orders.csv", "-e", "r"), "expected NAME=[FORMAT:]LOCATION"); + assertUsage(Cli.in(dir).run("-i", "r=", "-e", "r"), "expected NAME=[FORMAT:]LOCATION"); + assertUsage(Cli.in(dir).run("-i", "r=csv:", "-e", "r"), "expected NAME=[FORMAT:]LOCATION"); + } + + @Test + @DisplayName("a URL without --remote") + void urlNeedsRemote() { + assertUsage(Cli.in(dir).run("-i", "r=https://example.com/orders.csv", "-e", "r"), + "reading a URL needs --remote"); + } + + @Test + @DisplayName("a name bound twice") + void boundTwice() throws IOException { + write("orders.csv", CSV); + + assertUsage(Cli.in(dir).run("-i", "r=orders.csv", "-i", "r=orders.csv", "-e", "r"), "'r' is bound twice"); + } + + @Test + @DisplayName("standard input bound twice") + void stdinTwice() { + assertUsage(Cli.in(dir).stdin(CSV).run("-i", "a=csv:-", "-i", "b=csv:-", "-e", "a"), + "standard input (-) can be bound to one input only"); + } + + @Test + @DisplayName("standard input bound, and the script read from it too") + void stdinIsData() throws IOException { + write("a.relix", "query { r };\n"); + + assertUsage(Cli.in(dir).stdin(CSV).run("-i", "r=csv:-"), "cannot be the script too"); + assertUsage(Cli.in(dir).stdin(CSV).run("-i", "r=csv:-", "-"), "cannot be the script too"); + assertUsage(Cli.in(dir).stdin(CSV).run("-i", "r=csv:-", "a.relix", "a.relix"), + "can be given to one script only"); + } + + @Test + @DisplayName("--schema or --unbounded for a name no -i binds") + void unboundName() throws IOException { + write("orders.csv", CSV); + + assertUsage(Cli.in(dir).run("-i", "r=orders.csv", "--schema", "s={ id: NUMBER }", "-e", "r"), + "--schema s: no input of that name is bound with -i"); + assertUsage(Cli.in(dir).run("-i", "r=orders.csv", "--unbounded=s", "-e", "r"), + "--unbounded s: no input of that name is bound with -i"); + } + + @Test + @DisplayName("an unbounded JSON array, and --infer-rows below 1") + void badModifiers() { + assertUsage(Cli.in(dir).stdin(JSON).run("-i", "r=json:-", "--unbounded=r", "-e", "r"), + "a JSON array is read whole"); + assertUsage(Cli.in(dir).stdin(JSON).run("-i", "r=json:-", "--infer-rows=0", "-e", "r"), + "--infer-rows must be a positive number"); + } + } + + @Nested + @DisplayName("an unbounded input") + class Unbounded { + + @ParameterizedTest(name = "{0}") + @CsvSource(delimiter = ';', value = { + "τ id (r)", + "γ customer, COUNT(id) → n (r)"}) + @DisplayName("a blocking operator over it is refused before it starts") + void blockingRefused(String query) { + var result = Cli.in(dir).piped().stdin(NDJSON) + .run("-i", "r=ndjson:-", "--unbounded=r", "--infer-rows=1", "-e", query); + + assertThat(result.status()).as(result.toString()).isEqualTo(3); + assertThat(result.out()).isEmpty(); + assertThat(result.err()).as(result.toString()).contains("unbounded"); + } + + @Test + @Timeout(60) + @DisplayName("streaming ndjson on standard input emits a row before the input ends") + void streams() throws Exception { + PipedOutputStream feed = new PipedOutputStream(); + PipedInputStream stdin = new PipedInputStream(feed); + CountDownLatch firstRow = new CountDownLatch(1); + ByteArrayOutputStream seen = new ByteArrayOutputStream(); + OutputStream reader = new OutputStream() { + @Override + public synchronized void write(int b) { + seen.write(b); + if (seen.toString(StandardCharsets.UTF_8).contains("503")) { + firstRow.countDown(); + } + } + }; + + CompletableFuture run = CompletableFuture.supplyAsync(() -> + Cli.in(dir).stdin(stdin).stdout(reader).run( + "-i", "log=ndjson:-", "--unbounded=log", "--infer-rows=1", + "-o", "ndjson", "--line-buffered", "-e", "σ status >= 500 (log)")); + + feed.write("{\"status\":200}\n{\"status\":503}\n".getBytes(StandardCharsets.UTF_8)); + feed.flush(); + boolean emitted = firstRow.await(30, TimeUnit.SECONDS); + assertThat(run).as("still waiting for more input").isNotDone(); + feed.write("{\"status\":500}\n".getBytes(StandardCharsets.UTF_8)); + feed.close(); + Cli.Result result = run.get(30, TimeUnit.SECONDS); + + assertThat(emitted).as("a row before the input ended: " + result).isTrue(); + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("{\"status\":503}\n{\"status\":500}\n"); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/command/ManagementCommandsTest.java b/src/test/java/com/darkcollective/relix/cli/command/ManagementCommandsTest.java new file mode 100644 index 0000000..27d996e --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/command/ManagementCommandsTest.java @@ -0,0 +1,328 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.Cli; +import com.darkcollective.relix.docs.RelixDocs; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * The management commands, {@code drivers}, {@code connectors}, {@code doc}, {@code help}, + * {@code version} and {@code completion} (design §3), end to end through {@code Main}. + * + *

    Nothing here downloads: installing a driver or connector that is not installed would + * reach the network, so only the paths that do not are run. + */ +@DisplayName("Management commands") +class ManagementCommandsTest { + + @TempDir + Path dir; + + private Cli cli() { + return Cli.in(dir); + } + + @Nested + @DisplayName("version") + class Version { + + @Test + @DisplayName("prints relix's version and the engine's, as --version does") + void version() { + var result = cli().run("version"); + + assertThat(result.status()).isZero(); + assertThat(result.out()).matches("relix \\S+ \\(engine \\S+\\)\\R").isEqualTo(cli().run("--version").out()); + } + } + + @Nested + @DisplayName("help") + class Help { + + @Test + @DisplayName("prints a command's usage to stdout") + void command() { + var result = cli().run("help", "fmt"); + + assertThat(result.status()).isZero(); + assertThat(result.out()).startsWith("Usage: relix fmt").contains("--keywords"); + } + + @Test + @DisplayName("prints the main usage, listing every command, with no COMMAND") + void main() { + var result = cli().run("help"); + + assertThat(result.status()).isZero(); + assertThat(result.out()).startsWith("Usage: relix"); + for (String command : List.of("run", "check", "explain", "optimize", "trace", "bundle", "ir", + "provenance", "fmt", "catalog", "drivers", "connectors", "doc", "help", "version", "completion")) { + assertThat(result.out()).as(command).containsPattern("(?m)^ " + command + "\\b"); + } + } + } + + @Nested + @DisplayName("drivers") + class Drivers { + + @Test + @DisplayName("ls lists the drivers relix can install, as rows") + void ls() { + var result = cli().piped().run("drivers", "ls"); + + assertThat(result.status()).as(result.toString()).isZero(); + List lines = result.out().lines().toList(); + assertThat(lines.getFirst()).isEqualTo("name\tscheme\tinstalled"); + assertThat(lines).anyMatch(l -> l.startsWith("postgresql\tjdbc:postgresql:\t")) + .anyMatch(l -> l.startsWith("mysql\tjdbc:mysql:\t")); + } + + @Test + @DisplayName("ls writes ndjson with -o") + void ndjson() { + var result = cli().run("drivers", "ls", "-o", "ndjson"); + + assertThat(result.out()).contains("{\"name\":\"postgresql\",\"scheme\":\"jdbc:postgresql:\",\"installed\":"); + } + + @Test + @DisplayName("install of a driver relix does not know is a usage error naming those it does") + void unknown() { + var result = cli().run("drivers", "install", "nope"); + + assertThat(result.status()).isEqualTo(2); + assertThat(result.err()).isEqualTo("relix: no driver named 'nope'; relix can install postgresql, mysql\n"); + } + + @Test + @DisplayName("with no subcommand prints usage and exits 2") + void bare() { + assertThat(cli().run("drivers").status()).isEqualTo(2); + } + } + + @Nested + @DisplayName("connectors") + class Connectors { + + @Test + @DisplayName("ls lists the connectors installed, as rows") + void ls() { + var result = cli().piped().run("connectors", "ls"); + + assertThat(result.status()).isZero(); + assertThat(result.out()).startsWith("type\tinstalled\n").contains("csv\ttrue\n"); + } + + @Test + @DisplayName("install of one already installed says so and downloads nothing") + void installed() { + var result = cli().run("connectors", "install", "csv"); + + assertThat(result.status()).isZero(); + assertThat(result.out()).isEmpty(); + assertThat(result.err()).isEqualTo("relix: the csv connector is already installed\n"); + } + } + + @Nested + @DisplayName("doc") + class Doc { + + @Test + @DisplayName("shows one page for a glyph, its keyword and its name") + void spellings() { + var glyph = cli().run("doc", "σ"); + + assertThat(glyph.status()).isZero(); + assertThat(glyph.out()).contains("Selection (σ / SELECT)"); + assertThat(cli().run("doc", "select").out()).isEqualTo(glyph.out()); + assertThat(cli().run("doc", "Selection").out()).isEqualTo(glyph.out()); + } + + @Test + @DisplayName("--markdown prints the page as the reference holds it") + void markdown() { + var result = cli().run("doc", "--markdown", "σ"); + + assertThat(result.out()).isEqualTo(RelixDocs.referencePage("operators/select.md").orElseThrow()); + } + + @Test + @DisplayName("gives a shared name to the operator, and --function finds the function") + void function() { + assertThat(cli().run("doc", "fix").out()).contains("(FIX)"); + assertThat(cli().run("doc", "--function", "fix").out()).contains("Fix ("); + } + + @Test + @DisplayName("an unknown topic is a usage error") + void unknown() { + var result = cli().run("doc", "no-such-topic"); + + assertThat(result.status()).isEqualTo(2); + assertThat(result.err()).isEqualTo("relix: no reference page for 'no-such-topic'; relix doc lists them\n"); + } + + @Test + @DisplayName("with no topic lists the pages as rows, language pages then functions") + void list() { + var result = cli().piped().run("doc"); + List lines = result.out().lines().toList(); + + assertThat(lines.getFirst()).isEqualTo("topic\tcategory\ttitle\tsummary"); + assertThat(lines).anyMatch(l -> l.startsWith("σ\toperator\tSelection\t")) + .anyMatch(l -> l.startsWith("Fix\tfunction\tFix\t")); + assertThat(lines.size() - 1).isGreaterThan(RelixDocs.referencePages().size()); + } + + @Test + @DisplayName("--function lists only the functions") + void functions() { + var result = cli().piped().run("doc", "--function"); + + assertThat(result.out().lines().skip(1)).isNotEmpty().allMatch(l -> l.split("\t")[1].equals("function")); + } + + @Test + @DisplayName("resolves every key of every language page") + void everyKey() { + List missing = new ArrayList<>(); + for (var page : RelixDocs.referencePages()) { + for (String key : page.keys()) { + var result = cli().run("doc", "--markdown", key); + if (result.status() != 0 || result.out().isEmpty()) { + missing.add(page.path() + ": " + key); + } + } + } + assertThat(missing).isEmpty(); + } + } + + @Nested + @DisplayName("completion") + class Completion_ { + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"bash", "zsh", "fish", "powershell"}) + @DisplayName("prints a script that completes every command") + void script(String shell) { + var result = cli().run("completion", shell); + + assertThat(result.status()).as(result.toString()).isZero(); + for (String command : List.of("run", "check", "fmt", "catalog", "drivers", "doc", "completion")) { + assertThat(result.out()).as(command).contains(command); + } + assertThat(result.out()).contains("keywords").contains("trace"); + } + + @Test + @DisplayName("zsh's is a completion function too, for $fpath, that completes on its first call") + void zshFunction() { + String script = cli().run("completion", "zsh").out(); + + assertThat(script).startsWith("#compdef relix\n"); + assertThat(script).endsWith(""" + if (( ${+compstate} )); then + _bash_complete -F _complete_relix -o default + fi + """); + assertThat(script).contains("complete -F _complete_relix -o default relix"); + } + + @Test + @DisplayName("an unknown shell is a usage error") + void unknown() { + var result = cli().run("completion", "tcsh"); + + assertThat(result.status()).isEqualTo(2); + assertThat(result.err()).contains("'tcsh' is not one of bash, zsh, fish, powershell"); + } + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"bash", "zsh", "fish"}) + @DisabledOnOs(OS.WINDOWS) + @DisplayName("is read without error by its shell, where the shell is installed") + void posixSyntax(String shell) throws Exception { + Path script = write(shell); + List check = shell.equals("fish") + ? List.of("fish", "--no-execute", script.toString()) + : List.of(shell, "-n", script.toString()); + + assertSyntax(shell, check); + } + + @Test + @DisplayName("is read without error by PowerShell, where it is installed") + void powershellSyntax() throws Exception { + Path script = write("powershell"); + String parse = "$errors = $null; [System.Management.Automation.Language.Parser]::ParseFile('" + + script.toString().replace("'", "''") + "', [ref]$null, [ref]$errors) | Out-Null; " + + "if ($errors.Count -gt 0) { $errors | ForEach-Object { Write-Error $_.Message }; exit 1 }"; + + assertSyntax("pwsh", List.of("pwsh", "-NoProfile", "-NonInteractive", "-Command", parse)); + } + + private Path write(String shell) throws IOException { + var result = cli().run("completion", shell); + return Files.writeString(dir.resolve("relix." + shell), result.out(), StandardCharsets.UTF_8); + } + + private void assertSyntax(String shell, List command) throws Exception { + assumeTrue(available(shell), shell + " is not installed"); + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertThat(process.waitFor(60, TimeUnit.SECONDS)).isTrue(); + assertThat(process.exitValue()).as(output).isZero(); + } + + private static boolean available(String shell) { + try { + Process process = new ProcessBuilder(shell, shell.equals("pwsh") ? "-Version" : "--version") + .redirectErrorStream(true).start(); + process.getInputStream().readAllBytes(); + return process.waitFor(30, TimeUnit.SECONDS) && process.exitValue() == 0; + } catch (IOException e) { + return false; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/command/OutputTest.java b/src/test/java/com/darkcollective/relix/cli/command/OutputTest.java new file mode 100644 index 0000000..2e31440 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/command/OutputTest.java @@ -0,0 +1,401 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.Cli; +import com.darkcollective.relix.cli.io.Interruption; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicLong; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Where rows go and in what shape (design §3.3, §3.4, §3.6), end to end through + * {@code Main}. + */ +@DisplayName("Output") +class OutputTest { + + private static final String NUMBERS = """ + N := [ + | x | s | + |---|-------| + | 1 | one | + | 2 | two | + | 3 | NULL | + ]; + """; + + private static final String TWO_QUERIES = NUMBERS + """ + Small := { σ x < 2 (N) }; + Big := { σ x > 2 (N) }; + query Small; + query Big; + """; + + private static final String NATURALS = "source Naturals from generator { name: \"Naturals\" };"; + + @TempDir + Path dir; + + @Nested + @DisplayName("choosing a format") + class Choosing { + + @Test + @DisplayName("a terminal gets a table") + void terminalGetsTable() { + var result = Cli.in(dir).run("-e", NUMBERS, "-e", "π x (N)"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).startsWith("── ").contains("(3 rows)"); + } + + @Test + @DisplayName("a pipe gets tsv") + void pipeGetsTsv() { + var result = Cli.in(dir).piped().run("-e", NUMBERS, "-e", "N"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("x\ts\n1\tone\n2\ttwo\n3\t\n"); + } + + @Test + @DisplayName("-o chooses the format, on a terminal too") + void minusO() { + var result = Cli.in(dir).run("-o", "ndjson", "-e", NUMBERS, "-e", "σ x = 1 (N)"); + + assertThat(result.out()).isEqualTo("{\"x\":1,\"s\":\"one\"}\n"); + } + + @Test + @DisplayName("$RELIX_OUTPUT chooses it when -o does not, and -o beats it") + void environment() { + var fromEnv = Cli.in(dir).env("RELIX_OUTPUT", "csv").run("-e", NUMBERS, "-e", "σ x = 1 (N)"); + var fromOption = Cli.in(dir).env("RELIX_OUTPUT", "csv") + .run("--output=tsv", "-e", NUMBERS, "-e", "σ x = 1 (N)"); + + assertThat(fromEnv.out()).isEqualTo("x,s\n1,one\n"); + assertThat(fromOption.out()).isEqualTo("x\ts\n1\tone\n"); + } + + @Test + @DisplayName("an unknown format is a usage error, naming the ones there are") + void unknownFormat() { + var option = Cli.in(dir).run("-o", "xml", "-e", "N"); + var variable = Cli.in(dir).env("RELIX_OUTPUT", "xml").run("-e", "N"); + + assertThat(option.status()).isEqualTo(2); + assertThat(option.out()).isEmpty(); + assertThat(option.err()).contains("-o: unknown output format: 'xml'").contains("ndjson"); + assertThat(variable.status()).isEqualTo(2); + assertThat(variable.err()).contains("$RELIX_OUTPUT: unknown output format: 'xml'"); + } + + @Test + @DisplayName("relix run takes the same options") + void runCommand() { + var result = Cli.in(dir).run("run", "-o", "csv", "--no-header", "-e", NUMBERS, "-e", "σ x = 1 (N)"); + + assertThat(result.out()).isEqualTo("1,one\n"); + } + } + + @Nested + @DisplayName("modifiers") + class Modifiers { + + @Test + @DisplayName("--no-header leaves out the header row") + void noHeader() { + var result = Cli.in(dir).piped().run("--no-header", "-e", NUMBERS, "-e", "N"); + + assertThat(result.out()).isEqualTo("1\tone\n2\ttwo\n3\t\n"); + } + + @Test + @DisplayName("--null spells NULL, in a pipe and in a table") + void nullSpelling() { + var tsv = Cli.in(dir).piped().run("--null=\\N", "-e", NUMBERS, "-e", "σ x = 3 (N)"); + var table = Cli.in(dir).run("--null=-", "--color=never", "-e", NUMBERS, "-e", "σ x = 3 (N)"); + + assertThat(tsv.out()).isEqualTo("x\ts\n3\t\\N\n"); + assertThat(table.out()).contains(" 3 -").doesNotContain("NULL"); + } + + @Test + @DisplayName("--color=auto colours a table on a terminal, unless $NO_COLOR is set") + void colorAuto() { + var terminal = Cli.in(dir).run("-e", NUMBERS, "-e", "N"); + var noColor = Cli.in(dir).env("NO_COLOR", "1").run("-e", NUMBERS, "-e", "N"); + var pipe = Cli.in(dir).piped().run("-o", "table", "-e", NUMBERS, "-e", "N"); + + assertThat(terminal.out()).contains("\u001b[1m").contains("\u001b[2mNULL"); + assertThat(noColor.out()).doesNotContain("\u001b["); + assertThat(pipe.out()).doesNotContain("\u001b["); + } + + @Test + @DisplayName("--color=always and --color=never decide whatever the terminal") + void colorAlwaysNever() { + var always = Cli.in(dir).piped().env("NO_COLOR", "1") + .run("-o", "table", "--color=always", "-e", NUMBERS, "-e", "N"); + var never = Cli.in(dir).run("--color=never", "-e", NUMBERS, "-e", "N"); + var bad = Cli.in(dir).run("--color=sometimes", "-e", "N"); + + assertThat(always.out()).contains("\u001b[1m"); + assertThat(never.out()).doesNotContain("\u001b["); + assertThat(bad.status()).isEqualTo(2); + assertThat(bad.err()).contains("'sometimes' is not one of auto, always, never"); + } + + @Test + @DisplayName("a machine format is never coloured") + void machineFormatsUncoloured() { + var result = Cli.in(dir).run("-o", "tsv", "--color=always", "-e", NUMBERS, "-e", "N"); + + assertThat(result.out()).doesNotContain("\u001b["); + } + } + + @Nested + @DisplayName("several queries") + class SeveralQueries { + + @Test + @DisplayName("a table prints each, labelled") + void tablePrintsEach() { + var result = Cli.in(dir).run("-e", TWO_QUERIES); + + assertThat(result.out()).contains("── Small ").contains("── Big "); + } + + @Test + @DisplayName("a machine format prints the last") + void machinePrintsLast() { + var result = Cli.in(dir).piped().run("-e", TWO_QUERIES); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("x\ts\n3\t\n"); + } + + @Test + @DisplayName("--query=NAME prints the one named") + void queryByName() { + var result = Cli.in(dir).piped().run("--query=Small", "-e", TWO_QUERIES); + + assertThat(result.out()).isEqualTo("x\ts\n1\tone\n"); + } + + @Test + @DisplayName("--query naming no query is a usage error") + void unknownQuery() { + var result = Cli.in(dir).piped().run("--query=Medium", "-e", TWO_QUERIES); + + assertThat(result.status()).isEqualTo(2); + assertThat(result.out()).isEmpty(); + assertThat(result.err()).isEqualTo("relix: -e: no query named 'Medium'\n"); + } + + @Test + @DisplayName("--all with ndjson prints every query, each row tagged with its name") + void allNdjson() { + var result = Cli.in(dir).run("-o", "ndjson", "--all", "-e", TWO_QUERIES); + + assertThat(result.out()).isEqualTo(""" + {"_query":"Small","x":1,"s":"one"} + {"_query":"Big","x":3,"s":null} + """); + } + + @Test + @DisplayName("--all with another machine format is a usage error") + void allOtherFormat() { + for (String format : new String[] {"tsv", "csv", "json"}) { + var result = Cli.in(dir).run("-o", format, "--all", "-e", TWO_QUERIES); + + assertThat(result.status()).as(format).isEqualTo(2); + assertThat(result.out()).as(format).isEmpty(); + assertThat(result.err()).as(format).contains("--all prints several queries"); + } + } + + @Test + @DisplayName("--all and --query together are a usage error") + void allAndQuery() { + var result = Cli.in(dir).run("-o", "ndjson", "--all", "--query=Big", "-e", TWO_QUERIES); + + assertThat(result.status()).isEqualTo(2); + } + } + + @Nested + @DisplayName("assertions") + class Assertions { + + @Test + @DisplayName("--fail-empty exits 1 on no rows, and 0 on some") + void failEmpty() { + var empty = Cli.in(dir).piped().run("--fail-empty", "-e", NUMBERS, "-e", "σ x > 9 (N)"); + var some = Cli.in(dir).piped().run("--fail-empty", "-e", NUMBERS, "-e", "N"); + + assertThat(empty.status()).isEqualTo(1); + assertThat(empty.out()).as("the header row is still written").isEqualTo("x\ts\n"); + assertThat(empty.err()).isEmpty(); + assertThat(some.status()).isZero(); + } + + @Test + @DisplayName("--fail-rows exits 1 on any row, and 0 on none") + void failRows() { + var some = Cli.in(dir).piped().run("--fail-rows", "-e", NUMBERS, "-e", "σ x = 1 (N)"); + var none = Cli.in(dir).piped().run("--fail-rows", "-e", NUMBERS, "-e", "σ x > 9 (N)"); + + assertThat(some.status()).isEqualTo(1); + assertThat(some.err()).isEmpty(); + assertThat(none.status()).isZero(); + } + + @Test + @DisplayName("an assertion counts only the query printed") + void onlyPrinted() { + var result = Cli.in(dir).piped().run("--fail-rows", "--query=Big", "-e", + NUMBERS + "Small := { σ x < 2 (N) }; Big := { σ x > 9 (N) }; query Small; query Big;"); + + assertThat(result.status()).isZero(); + } + + @Test + @DisplayName("a failed script is reported as itself, not as an assertion") + void failureWins() { + var result = Cli.in(dir).piped().run("--fail-empty", "-e", "σ x > 1 (Nope)"); + + assertThat(result.status()).isEqualTo(3); + } + + @Test + @DisplayName("--fail-empty and --fail-rows together are a usage error") + void both() { + var result = Cli.in(dir).run("--fail-empty", "--fail-rows", "-e", "N"); + + assertThat(result.status()).isEqualTo(2); + } + } + + @Nested + @DisplayName("pipe and signal discipline") + class Discipline { + + @Test + @Timeout(60) + @DisplayName("relix -e 'Naturals' | head ends when head does: exit 141, nothing said") + void head() { + var head = new Head(5); + + var result = Cli.in(dir).stdout(head).run("-e", NATURALS, "-e", "Naturals"); + + assertThat(result.status()).isEqualTo(141); + assertThat(result.err()).isEmpty(); + } + + @Test + @Timeout(60) + @DisplayName("line-buffered, each row reaches the reader as it is made, and the run ends at the first refused") + void headLineBuffered() { + var head = new Head(5); + + var result = Cli.in(dir).stdout(head).run("--line-buffered", "-e", NATURALS, "-e", "Naturals"); + + assertThat(result.status()).isEqualTo(141); + assertThat(result.err()).isEmpty(); + assertThat(result.out()).isEqualTo("n\n0\n1\n2\n3\n"); + } + + @Test + @Timeout(60) + @DisplayName("an endless result in a table is refused rather than collected forever") + void endlessTable() { + var result = Cli.in(dir).run("-e", NATURALS, "-e", "Naturals"); + + assertThat(result.status()).isEqualTo(3); + assertThat(result.out()).isEmpty(); + } + + @Test + @Timeout(60) + @DisplayName("an interrupt stops the query in flight: exit 130, nothing said") + void interrupt() { + Interruption interruption = new Interruption(); + AtomicLong written = new AtomicLong(); + OutputStream reader = new OutputStream() { + @Override + public void write(int b) { + write(new byte[] {(byte) b}, 0, 1); + } + + @Override + public void write(byte[] b, int off, int len) { + // The first rows to arrive set off the interrupt, from another thread, + // as a signal's shutdown hook would. + if (written.getAndAdd(len) == 0) { + Thread.ofPlatform().start(interruption::interrupt); + } + } + }; + + var result = Cli.in(dir).stdout(reader).interruption(interruption) + .run("--line-buffered", "-e", NATURALS, "-e", "Naturals"); + + assertThat(result.status()).isEqualTo(130); + assertThat(result.err()).isEmpty(); + assertThat(interruption.interrupted()).isTrue(); + } + } + + /** {@code head -n}: reads {@code lines} lines, then goes away. */ + static final class Head extends OutputStream { + private int lines; + + Head(int lines) { + this.lines = lines; + } + + @Override + public void write(int b) throws IOException { + if (lines <= 0) { + throw new IOException("Broken pipe"); + } + if (b == '\n') { + lines--; + } + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + for (int i = 0; i < len; i++) { + write(b[off + i]); + } + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/command/QueryParametersTest.java b/src/test/java/com/darkcollective/relix/cli/command/QueryParametersTest.java new file mode 100644 index 0000000..629c7b2 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/command/QueryParametersTest.java @@ -0,0 +1,142 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.Cli; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Query parameters bound with {@code -a NAME=VALUE} (design §9, R3), end to end through + * {@code Main}. + */ +@DisplayName("Query parameters bound with -a") +class QueryParametersTest { + + private static final String CUSTOMERS = """ + Customers := [ + | name | city | + |-------|--------| + | Ada | London | + | Grace | Paris | + ]; + """; + + private static final String ORDERS = """ + Orders := [ + | id | who | + |----|-------| + | 7 | Ada | + | 42 | Grace | + ]; + """; + + private static final String BY_CITY = CUSTOMERS + "query { π name (σ city = $city (Customers)) };\n"; + + @TempDir + Path dir; + + @BeforeEach + void script() throws IOException { + Files.writeString(dir.resolve("by-city.relix"), BY_CITY); + } + + @Test + @DisplayName("-a binds the parameter the script names") + void binds() { + var result = Cli.in(dir).run("-a", "city=Paris", "by-city.relix"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).contains("Grace").doesNotContain("Ada").contains("(1 row)"); + } + + @Test + @DisplayName("--arg is the long form, and it may follow the script") + void longFormAfterScript() { + var result = Cli.in(dir).run("by-city.relix", "--arg", "city=London"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).contains("Ada").doesNotContain("Grace"); + } + + @Test + @DisplayName("each -a binds one parameter") + void several() { + var result = Cli.in(dir).run("-a", "city=London", "-a", "who=Ada", + "-e", CUSTOMERS, "-e", "σ city = $city ∧ name = $who (Customers)"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).contains("Ada").contains("(1 row)"); + } + + @Test + @DisplayName("a value takes the type of what its parameter is compared with") + void typedFromText() { + var result = Cli.in(dir).run("-a", "id=42", "-e", ORDERS, "-e", "σ id = $id (Orders)"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).contains("Grace").contains("(1 row)"); + } + + @Test + @DisplayName("a value that is not of its parameter's type is an error that names both") + void wrongType() { + var result = Cli.in(dir).run("-a", "id=abc", "-e", ORDERS, "-e", "σ id = $id (Orders)"); + + assertThat(result.status()).as(result.toString()).isEqualTo(3); + assertThat(result.err()).contains("$id").contains("NUMBER").contains("'abc'"); + assertThat(result.out()).isEmpty(); + } + + @Test + @DisplayName("a value holding a quote is a value: it cannot change the query") + void quoteCannotInject() { + var result = Cli.in(dir).run("-a", "city=Paris' ∨ '1' = '1", "by-city.relix"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).doesNotContain("Ada").doesNotContain("Grace").contains("(0 rows)"); + } + + @Test + @DisplayName("an unbound parameter is an analysis error that names it") + void unbound() { + var result = Cli.in(dir).run("by-city.relix"); + + assertThat(result.status()).as(result.toString()).isEqualTo(3); + assertThat(result.err()).isEqualTo( + "relix: by-city.relix: parameter $city is not bound; give it a value with -a city=VALUE\n"); + assertThat(result.out()).isEmpty(); + } + + @Test + @DisplayName("fmt, bundle and optimize print $name, never the value") + void reportsPrintTheName() { + for (String command : new String[] {"fmt", "bundle", "optimize"}) { + var result = Cli.in(dir).run(command, "-a", "city=Lisbon", "by-city.relix"); + + assertThat(result.status()).as(command + ": " + result).isZero(); + assertThat(result.out()).as(command).contains("$city").doesNotContain("Lisbon"); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/command/ReportCommandsTest.java b/src/test/java/com/darkcollective/relix/cli/command/ReportCommandsTest.java new file mode 100644 index 0000000..e097d05 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/command/ReportCommandsTest.java @@ -0,0 +1,458 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.command; + +import com.darkcollective.relix.cli.Cli; +import com.darkcollective.relix.cli.Examples; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.FieldSource; +import org.junit.jupiter.params.provider.ValueSource; +import tools.jackson.core.JsonParser; +import tools.jackson.core.JsonToken; +import tools.jackson.core.ObjectReadContext; +import tools.jackson.core.json.JsonFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The report commands, {@code check}, {@code explain}, {@code optimize}, {@code trace}, + * {@code bundle}, {@code ir} and {@code provenance}, and {@code run --trace} (design §3, + * §4), end to end through {@code Main} over the example scripts. + */ +@DisplayName("Report commands") +class ReportCommandsTest { + + static final List EXAMPLES = Examples.NAMES; + + /** A script whose second statement names a relation nothing declares. */ + private static final String BROKEN = """ + N := [ + | x | + |---| + | 1 | + ]; + Bad := { σ x > 1 (Missing) }; + query Bad; + """; + + private static final JsonFactory JSON = new JsonFactory(); + + @TempDir + Path dir; + + @BeforeEach + void examples() throws IOException { + Examples.copyTo(dir); + Files.writeString(dir.resolve("broken.relix"), BROKEN, StandardCharsets.UTF_8); + } + + private Cli cli() { + return Cli.in(dir); + } + + /** Whether {@code text} is one JSON value and nothing else. */ + private static boolean isJson(String text) { + try (JsonParser parser = JSON.createParser(ObjectReadContext.empty(), text)) { + parser.nextToken(); + parser.skipChildren(); + return parser.nextToken() == null; + } catch (RuntimeException e) { + return false; + } + } + + @Nested + @DisplayName("every report command") + class Every { + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"explain", "optimize", "trace", "bundle", "ir", "provenance"}) + @DisplayName("writes its report of each example to stdout, and nothing to stderr") + void overTheExamples(String command) { + for (String example : EXAMPLES) { + var result = cli().piped().run(command, example); + + assertThat(result.status()).as(example + "\n" + result).isZero(); + assertThat(result.out()).as(example).isNotBlank(); + assertThat(result.err()).as(example).isEmpty(); + } + } + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"check", "explain", "optimize", "trace", "ir", "provenance"}) + @DisplayName("prints nothing for a script that does not analyse, reports it, and exits 3") + void broken(String command) { + var result = cli().run(command, "broken.relix"); + + assertThat(result.status()).as(result.toString()).isEqualTo(3); + assertThat(result.out()).isEmpty(); + assertThat(result.err()).contains("broken.relix:6:").contains("Missing"); + } + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"check", "explain", "optimize", "trace", "bundle", "ir", "provenance"}) + @DisplayName("reads -e and standard input as run does") + void sources(String command) { + var fromE = cli().run(command, "-e", "λ 2 (relix.functions)"); + var fromStdin = cli().stdin("query { λ 2 (relix.functions) };").run(command); + + assertThat(fromE.status()).as(fromE.toString()).isZero(); + assertThat(fromStdin.status()).as(fromStdin.toString()).isZero(); + } + + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"explain", "trace", "provenance"}) + @DisplayName("--query names one query, and an unknown one is a usage error") + void query(String command) { + var one = cli().run(command, "--query", "LondonBorrowers", "library.relix"); + var unknown = cli().run(command, "--query", "Nope", "library.relix"); + + assertThat(one.status()).as(one.toString()).isZero(); + assertThat(unknown.status()).isEqualTo(2); + assertThat(unknown.err()).contains("library.relix: no query named 'Nope'"); + } + } + + @Test + @DisplayName("the examples declare nothing that reaches the network") + void offline() { + for (String example : EXAMPLES) { + assertThat(Examples.text(example)).as(example) + .doesNotContainPattern("(?m)^[^-\\n]*\\bfrom\\s+(http|jdbc)\\b") + .doesNotContainPattern("(?m)^\\s*connection\\b"); + } + } + + @Nested + @DisplayName("check") + class Check { + + @Test + @DisplayName("is silent and exits 0 over scripts that analyse") + void clean() { + var result = cli().run("check", "introspection.relix", "league.relix", "library.relix"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEmpty(); + assertThat(result.err()).isEmpty(); + } + + @Test + @DisplayName("as xargs runs it, exits 3 when one file of many is broken, naming only that file") + void xargs() { + var result = cli().run("check", "league.relix", "broken.relix", "library.relix"); + + assertThat(result.status()).isEqualTo(3); + assertThat(result.out()).isEmpty(); + assertThat(result.err()).isEqualTo("broken.relix:6:19: error: Undefined relation: 'Missing'\n"); + } + + @Test + @DisplayName("--json writes every script's diagnostics to stdout as one document, and nothing to stderr") + void json() { + var result = cli().run("check", "--json", "league.relix", "broken.relix"); + + assertThat(result.status()).isEqualTo(3); + assertThat(result.err()).isEmpty(); + assertThat(isJson(result.out())).as(result.out()).isTrue(); + assertThat(result.out()).isEqualTo("{\"diagnostics\":[{\"file\":\"broken.relix\",\"line\":6," + + "\"column\":19,\"severity\":\"error\",\"message\":\"Undefined relation: 'Missing'\"}]}\n"); + } + + @Test + @DisplayName("--json over a clean script is an empty list") + void jsonClean() { + var result = cli().run("check", "--json", "league.relix"); + + assertThat(result.status()).isZero(); + assertThat(result.out()).isEqualTo("{\"diagnostics\":[]}\n"); + } + + @Test + @DisplayName("places a diagnostic in an -e fragment") + void expression() { + var result = cli().run("check", "-e", "σ x > 1 (Missing)"); + + assertThat(result.status()).isEqualTo(3); + assertThat(result.err()).startsWith("-e:1:"); + } + } + + @Nested + @DisplayName("explain") + class Explain { + + @Test + @DisplayName("prints one query's plan as the engine draws it") + void one() { + var result = cli().run("explain", "--query", "LondonBorrowers", "library.relix"); + + assertThat(result.out()).startsWith("Join SEMI/HASH").contains("Scan Members").contains("Scan Loans") + .doesNotContain("-- query"); + } + + @Test + @DisplayName("heads each plan with its query when there are several") + void several() { + var result = cli().run("explain", "library.relix"); + + assertThat(result.out()).contains("-- query OnLoanDetails\n").contains("-- query LondonBorrowers\n"); + } + + @Test + @DisplayName("--json writes one object per query per line") + void json() { + var result = cli().run("explain", "--json", "library.relix"); + + List lines = result.out().lines().toList(); + assertThat(lines).hasSize(7).allSatisfy(line -> assertThat(isJson(line)).as(line).isTrue()); + assertThat(lines.getLast()).startsWith("{\"script\":\"library.relix\",\"query\":\"LondonBorrowers\"," + + "\"plan\":{\"op\":\"Join\""); + } + } + + @Nested + @DisplayName("optimize") + class Optimize { + + @ParameterizedTest(name = "{0}") + @FieldSource("com.darkcollective.relix.cli.command.ReportCommandsTest#EXAMPLES") + @DisplayName("writes a script that returns what the original does") + void sameRows(String example) throws IOException { + var optimized = cli().run("optimize", example); + Files.writeString(dir.resolve("optimized.relix"), optimized.out(), StandardCharsets.UTF_8); + + var before = cli().run("-o", "ndjson", "--all", example); + var after = cli().run("-o", "ndjson", "--all", "optimized.relix"); + + assertThat(optimized.status()).as(optimized.toString()).isZero(); + assertThat(after.status()).as(after.toString()).isZero(); + assertThat(untagged(after.out())).isEqualTo(untagged(before.out())); + } + + /** The rows without the label each is tagged with, which a rewritten query may lose. */ + private static String untagged(String ndjson) { + return ndjson.replaceAll("\\{\"_query\":\"[^\"]*\",", "{"); + } + + @Test + @DisplayName("rewrites the query statements, and keeps every other statement and comment as written") + void keepsTheRest() { + var result = cli().run("optimize", "library.relix"); + String original = Examples.text("library.relix"); + + assertThat(result.out()).contains("query { ρ LondonBorrowers (").doesNotContain("query LondonBorrowers;"); + String declarations = original.substring(0, original.indexOf("query OnLoanDetails;")); + assertThat(result.out()).startsWith(declarations); + } + + @Test + @DisplayName("writes back a rewrite that seeds a closure, as a script that runs") + void seededClosure() throws IOException { + // The selection is pushed into the closure as its seed, which render() writes + // back as a selection over the closure (DarkCollective/relix-core#100). + var result = cli().run("optimize", "introspection.relix"); + Files.writeString(dir.resolve("optimized.relix"), result.out(), StandardCharsets.UTF_8); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.err()).isEmpty(); + assertThat(result.out()).doesNotContain("query { π depends_on (σ dependent = \"Report\" (Lineage)) };"); + assertThat(cli().run("check", "optimized.relix").status()).isZero(); + } + + @Test + @DisplayName("works as a filter on standard input") + void filter() { + var result = cli().stdin("N := [\n| x |\n|---|\n| 1 |\n];\nquery { π x (σ x > 0 (N)) };\n") + .run("optimize"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).startsWith("N := [\n").contains("query { ").endsWith(" };\n"); + } + + @Test + @DisplayName("--report prints the rules that fired and the trees before and after") + void report() { + var result = cli().run("optimize", "--report", "library.relix"); + + assertThat(result.out()).contains("RELIX OPTIMIZER namespace=library") + .contains("INLINE-001").contains("── LondonBorrowers").contains("before:").contains("after:"); + } + } + + @Nested + @DisplayName("trace") + class Trace { + + @Test + @DisplayName("prints each query's events and ends each with the rows it delivered, not the rows") + void feed() { + var result = cli().run("trace", "--query", "LondonBorrowers", "library.relix"); + + assertThat(result.out()).contains("[OPTIMIZE] query[1] INLINE-001 view 'LondonBorrowers' inlined") + .contains("[EXECUTE ] LondonBorrowers ROWS query delivered 3 rows") + .doesNotContain("Alice"); + } + + @Test + @DisplayName("ends the feed of every query of the script") + void everyQuery() { + var result = cli().run("trace", "library.relix"); + + assertThat(result.out().lines().filter(l -> l.contains(" ROWS "))).hasSize(7); + } + } + + @Nested + @DisplayName("run --trace") + class RunTrace { + + @Test + @DisplayName("writes the rows to stdout and the events to stderr in one run") + void stderr() { + var result = cli().piped().run("--trace", "--query", "LondonBorrowers", "library.relix"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).isEqualTo("member_id\tname\tcity\tjoined\n1\tAlice\tLondon\t2019\n" + + "3\tCarol\tLondon\t2021\n5\tEve\tLondon\t2022\n"); + assertThat(result.err()).contains("[PLAN ] LondonBorrowers JOIN") + .contains("ROWS query delivered 3 rows"); + } + + @Test + @DisplayName("traces a table too") + void table() { + var result = cli().run("run", "--trace", "--query", "LondonBorrowers", "library.relix"); + + assertThat(result.out()).contains("Alice").contains("(3 rows)"); + assertThat(result.err()).contains("ROWS query delivered 3 rows"); + } + + @Test + @DisplayName("--trace=FILE writes the events to FILE, and a bare --trace never takes the script for its file") + void file() throws IOException { + var result = cli().piped().run("run", "--trace=events.log", "--query", "LondonBorrowers", + "library.relix"); + + assertThat(result.status()).as(result.toString()).isZero(); + assertThat(result.out()).contains("Alice"); + assertThat(result.err()).isEmpty(); + assertThat(Files.readString(dir.resolve("events.log"))).contains("ROWS query delivered 3 rows"); + assertThat(Files.readString(dir.resolve("library.relix"))).isEqualTo(Examples.text("library.relix")); + } + + @Test + @DisplayName("is run's alone: trace has no --trace") + void runOnly() { + var result = cli().run("explain", "--trace", "library.relix"); + + assertThat(result.status()).isEqualTo(2); + } + } + + @Nested + @DisplayName("bundle") + class Bundle { + + @Test + @DisplayName("is one JSON document per script, with each query's plans") + void document() { + var result = cli().run("bundle", "library.relix"); + + assertThat(isJson(result.out())).isTrue(); + assertThat(result.out()).startsWith("{\"namespace\":\"library\",\"diagnostics\":[],\"queries\":[") + .contains("\"label\":\"LondonBorrowers\"").contains("\"physicalPlan\":{\"op\":"); + } + + @Test + @DisplayName("is still produced for a script that does not analyse, carrying its diagnostics, and exits 3") + void broken() { + var result = cli().run("bundle", "broken.relix"); + + assertThat(result.status()).isEqualTo(3); + assertThat(isJson(result.out())).as(result.out()).isTrue(); + assertThat(result.out()).contains("\"diagnostics\":[{\"severity\":\"error\"," + + "\"message\":\"Undefined relation: 'Missing'\",\"file\":\"broken.relix\",\"line\":6,\"column\":19}]"); + } + } + + @Nested + @DisplayName("ir") + class Ir { + + @Test + @DisplayName("prints the IR report of the whole script") + void report() { + var result = cli().run("ir", "library.relix"); + + assertThat(result.out()).contains("RELIX IR namespace=library").contains("LondonBorrowers"); + } + } + + @Nested + @DisplayName("provenance") + class Provenance { + + @Test + @DisplayName("annotates each row in a [prov] column on a terminal, counting by default") + void table() { + var result = cli().run("provenance", "--query", "LondonBorrowers", "library.relix"); + + assertThat(result.out()).contains("[prov]").contains("(3 tuples; provenance semiring: counting)"); + } + + @Test + @DisplayName("writes JSON in a pipe, with the annotation beside each row, never in it") + void json() { + var result = cli().piped().run("provenance", "--semiring", "lineage", "--query", "LondonBorrowers", + "library.relix"); + + assertThat(isJson(result.out())).as(result.out()).isTrue(); + assertThat(result.out()).contains("\"semiring\": \"lineage\"") + .contains("{\"row\": {\"member_id\": 1, \"name\": \"Alice\"").contains("}, \"provenance\": {"); + } + + @Test + @DisplayName("-o table and -o json choose; another format is a usage error") + void format() { + assertThat(cli().piped().run("provenance", "-o", "table", "--query", "LondonBorrowers", + "library.relix").out()).contains("[prov]"); + assertThat(cli().run("provenance", "-o", "json", "--query", "LondonBorrowers", + "library.relix").out()).startsWith("{"); + assertThat(cli().run("provenance", "-o", "csv", "library.relix").status()).isEqualTo(2); + } + + @Test + @DisplayName("an unknown semiring is a usage error that lists the ones there are") + void unknownSemiring() { + var result = cli().run("provenance", "--semiring", "nope", "library.relix"); + + assertThat(result.status()).isEqualTo(2); + assertThat(result.err()).contains("'nope' is not one of").contains("counting").contains("lineage"); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/config/ProfilesTest.java b/src/test/java/com/darkcollective/relix/cli/config/ProfilesTest.java index 606384a..6d9f8ef 100644 --- a/src/test/java/com/darkcollective/relix/cli/config/ProfilesTest.java +++ b/src/test/java/com/darkcollective/relix/cli/config/ProfilesTest.java @@ -63,8 +63,13 @@ void tree() throws IOException { } } + /** The command in {@code work}, every directory trusted: trust has a test of its own below. */ private Cli cli() { - return Cli.in(work).home(home); + return in(work); + } + + private Cli in(Path dir) { + return Cli.in(dir).home(home).env("RELIX_TRUST_ALL", "1"); } private void profiles(Path dir, String json) throws IOException { @@ -171,8 +176,8 @@ void minusC() throws IOException { profiles(work, "{ \"dev\": { \"DATA\": \"three.csv\" } }"); Files.writeString(project.resolve("one.csv"), "x\n1\n"); - var fromProject = Cli.in(project).home(home).run("-P", "dev", "-e", SCRIPT); - var fromWork = Cli.in(project).home(home).run("-C", "work", "-P", "dev", "-e", SCRIPT); + var fromProject = in(project).run("-P", "dev", "-e", SCRIPT); + var fromWork = in(project).run("-C", "work", "-P", "dev", "-e", SCRIPT); assertThat(fromProject.status()).as("no profile above project").isEqualTo(2); assertThat(fromWork.out()).as(fromWork.toString()).contains("(3 rows)"); @@ -186,11 +191,27 @@ void outsideHome(@TempDir Path elsewhere) throws IOException { profiles(elsewhere.resolve("a"), "{ \"dev\": { \"DATA\": \"one.csv\" } }"); profiles(dir, "{ \"dev\": { \"DATA\": \"two.csv\" } }"); - var result = Cli.in(dir).home(home).run("-P", "dev", "-e", SCRIPT); + var result = in(dir).run("-P", "dev", "-e", SCRIPT); assertThat(result.out()).as(result.toString()).contains("(2 rows)"); assertThat(RelixDirectories.discover(dir, home)).containsExactly(dir.resolve(".relix")); } + + @Test + @DisplayName("a project's file is read only once the project is trusted") + void untrustedProject() throws IOException { + profiles(home, "{ \"dev\": { \"DATA\": \"four.csv\" } }"); + profiles(project, "{ \"dev\": { \"DATA\": \"two.csv\" } }"); + + var untrusted = Cli.in(work).home(home).run("-P", "dev", "-e", SCRIPT); + var trust = Cli.in(work).home(home).run("catalog", "trust", project.toString()); + var trusted = Cli.in(work).home(home).run("-P", "dev", "-e", SCRIPT); + + assertThat(untrusted.out()).as(untrusted.toString()).contains("(4 rows)"); + assertThat(untrusted.err()).contains("relix catalog trust " + project); + assertThat(trust.status()).as(trust.toString()).isZero(); + assertThat(trusted.out()).as(trusted.toString()).contains("(2 rows)"); + } } @Nested diff --git a/src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java b/src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java new file mode 100644 index 0000000..09032fb --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/ExampleRunner.java @@ -0,0 +1,294 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import com.darkcollective.relix.cli.Main; +import com.darkcollective.relix.cli.io.Host; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.PosixFilePermissions; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +/** + * Runs the guide's examples where a reader would: in {@code ~/shop}, a copy of the fixture + * tree under {@code src/test/resources/docs-fixture/home}, which is the home directory. + * + *

    The fixture is the shop the pages describe: a user-level {@code ~/.relix} with a + * table and a profile, the project {@code ~/shop} with its catalog, data files, scripts and + * profiles, and {@code ~/shop/reports}, a sub-project that overlays {@code Orders}. Setting + * it up does what a fresh copy cannot carry: {@code @HOME@} in a profile becomes the home + * directory's path, every {@code profiles.json} is made private, and the H2 database the + * {@code warehouse} connection opens is written, with user {@code shop} and the password + * {@code ~/.relix/profiles.json} holds. + * + *

    An example that is a plain {@code relix} command ({@link ShellCommand}) runs in this + * JVM, with standard output a terminal and standard input a terminal unless the example + * redirects it, as a reader typing it would have them. Anything else runs through + * {@code bash}, with a {@code relix} on its {@code PATH} that starts the command in a JVM + * of its own; there standard output is a pipe, as it is for a reader's pipeline. Both get + * the same environment: the fixture's {@code HOME}, {@code NO_COLOR=1}, {@code TZ=UTC}, + * and nothing of the user running the build. + */ +final class ExampleRunner implements AutoCloseable { + + private static final String FIXTURE = "/docs-fixture/home"; + + /** The H2 database's credentials, as the fixture's profiles give them. */ + private static final String WAREHOUSE_USER = "shop"; + private static final String WAREHOUSE_PASSWORD = "s3cret"; + + private final Path home; + private final Path shop; + private final Path bin; + private final Map environment; + private final Path stdin; + private final Path stdout; + + /** + * A fresh copy of the fixture. + * + * @param root an empty directory to put it in + * @param trusted whether {@code ~/shop} and {@code ~/shop/reports} start trusted + */ + ExampleRunner(Path root, boolean trusted) { + try { + this.home = Files.createDirectories(root.resolve("home")).toRealPath(); + copy(fixture(), home); + this.shop = home.resolve("shop"); + for (Path profiles : List.of(home.resolve(".relix/profiles.json"), shop.resolve(".relix/profiles.json"))) { + Files.writeString(profiles, Files.readString(profiles).replace("@HOME@", home.toString())); + Files.setPosixFilePermissions(profiles, PosixFilePermissions.fromString("rw-------")); + } + warehouse(shop.resolve("data/warehouse")); + this.bin = Files.createDirectories(root.resolve("bin")); + this.environment = environment(home, bin); + this.stdin = root.resolve("stdin.txt"); + this.stdout = root.resolve("stdout.txt"); + shim(bin.resolve("relix"), home); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + if (trusted) { + for (Path project : List.of(shop, shop.resolve("reports"))) { + Output trust = runInProcess(new ShellCommand(Map.of(), List.of("catalog", "trust", project.toString()), + null, null, false, false)); + if (trust.status() != 0) { + throw new IllegalStateException("cannot trust " + project + ": " + trust); + } + } + } + } + + /** {@return the fixture's home directory} */ + Path home() { + return home; + } + + /** + * What an example did. + * + * @param status its exit status + * @param out what it wrote to standard output, with the home directory written {@code ~} + * @param err what it wrote to standard error + * @param shell whether it ran through {@code bash} + */ + record Output(int status, String out, String err, boolean shell) { + @Override + public String toString() { + return (shell ? "bash" : "in-process") + ", exit " + status + "\n── stdout ──\n" + out + + "── stderr ──\n" + err; + } + } + + /** + * Runs an example. + * + * @param command the example's text + * @return what it did + */ + Output run(String command) { + return ShellCommand.parse(command).map(this::runInProcess).orElseGet(() -> runInBash(command)); + } + + private Output runInProcess(ShellCommand command) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteArrayOutputStream err = new ByteArrayOutputStream(); + OutputStream errTarget = command.mergeStderr() ? out : err; + InputStream in; + try { + in = command.heredoc() != null + ? new ByteArrayInputStream(command.heredoc().getBytes(StandardCharsets.UTF_8)) + : command.stdinFile() != null ? Files.newInputStream(shop.resolve(command.stdinFile())) + : new ByteArrayInputStream(new byte[0]); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + Map env = new HashMap<>(environment); + env.putAll(command.environment()); + boolean stdinIsTerminal = command.heredoc() == null && command.stdinFile() == null; + Host host = new Host(in, out, new PrintStream(errTarget, true, StandardCharsets.UTF_8), env, shop, home, + stdinIsTerminal, true); + // The engine finds its driver directory and file cache under the JVM's user.home, + // not $HOME, so the fixture is made that too for the run, as the bash shim does. + String userHome = System.getProperty("user.home"); + int status; + try { + System.setProperty("user.home", home.toString()); + status = Main.run(host, command.arguments().toArray(String[]::new)); + } finally { + System.setProperty("user.home", userHome); + } + String stdout = out.toString(StandardCharsets.UTF_8); + if (command.echoStatus()) { + stdout += "exit " + status + "\n"; + status = 0; + } + return new Output(status, tilde(stdout), err.toString(StandardCharsets.UTF_8), false); + } + + private Output runInBash(String command) { + // Descriptors 8 and 9 are the example's own standard input and output, which the + // shim compares a stage's streams with to tell where a reader's terminal would be. + ProcessBuilder builder = new ProcessBuilder("bash", "-c", "exec 8<&0 9>&1\n" + command) + .directory(shop.toFile()); + builder.environment().clear(); + builder.environment().putAll(environment); + try { + Files.write(stdin, new byte[0]); + Files.write(stdout, new byte[0]); + Path err = Files.createTempFile(bin.getParent(), "stderr", ".txt"); + Process process = builder.redirectInput(stdin.toFile()).redirectOutput(stdout.toFile()) + .redirectError(err.toFile()).start(); + if (!process.waitFor(2, TimeUnit.MINUTES)) { + process.destroyForcibly(); + throw new IllegalStateException("timed out: " + command); + } + return new Output(process.exitValue(), tilde(Files.readString(stdout)), Files.readString(err), true); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + + /** The home directory's path, which differs on every run, written as a reader's shell abbreviates it. */ + private String tilde(String text) { + return text.replace(home.toString(), "~"); + } + + @Override + public void close() { + // Nothing stays open: each example's JVM or session ends with it. + } + + private static Map environment(Path home, Path bin) { + Map env = new HashMap<>(); + // The relix under test first; then where jq, find and xargs are on a build machine. + env.put("PATH", bin + ":/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin"); + env.put("HOME", home.toString()); + env.put("NO_COLOR", "1"); + env.put("TZ", "UTC"); + // A UTF-8 locale, as a reader's terminal has: on Linux the JVM decodes its + // arguments by the locale, and `relix doc σ` would otherwise not say σ. + env.put("LC_ALL", System.getProperty("os.name").startsWith("Mac") ? "en_US.UTF-8" : "C.UTF-8"); + env.put("PAGER", "cat"); + return env; + } + + /** + * A {@code relix} for {@code bash}: this JVM's class path and {@link TerminalMain}, the + * fixture as the home directory a JDBC driver's {@code ~} means too, and standard input + * and output taken for terminals when they are the example's own — descriptors 8 and + * 9 — as they would be a reader's. + */ + private static void shim(Path file, Path home) throws IOException { + String java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + String script = "#!/bin/sh\n" + + "in=false; [ /dev/fd/0 -ef /dev/fd/8 ] && in=true\n" + + "out=false; [ /dev/fd/1 -ef /dev/fd/9 ] && out=true\n" + + "exec " + quote(java) + " -XX:TieredStopAtLevel=1 -Xshare:auto" + // The launcher's streams are UTF-8 whatever the locale, so a glyph in a + // message reads as it does on a reader's terminal. + + " -Dstdout.encoding=UTF-8 -Dstderr.encoding=UTF-8" + + " -Duser.home=" + quote(home.toString()) + + " -Drelix.docs.stdinIsTerminal=$in -Drelix.docs.stdoutIsTerminal=$out" + + " -cp " + quote(System.getProperty("java.class.path")) + + " " + TerminalMain.class.getName() + " \"$@\"\n"; + Files.writeString(file, script); + Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rwxr-xr-x")); + } + + private static String quote(String word) { + return "'" + word.replace("'", "'\\''") + "'"; + } + + private static Path fixture() { + URL url = ExampleRunner.class.getResource(FIXTURE); + if (url == null) { + throw new IllegalStateException("no " + FIXTURE + " on the test class path"); + } + try { + return Path.of(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalStateException(e); + } + } + + private static void copy(Path from, Path to) throws IOException { + try (Stream walk = Files.walk(from)) { + for (Path source : walk.toList()) { + Path target = to.resolve(from.relativize(source).toString()); + if (Files.isDirectory(source)) { + Files.createDirectories(target); + } else { + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + } + } + } + } + + /** The stock database the fixture's {@code warehouse} connection opens. */ + private static void warehouse(Path file) { + try (Connection connection = DriverManager.getConnection("jdbc:h2:" + file, WAREHOUSE_USER, WAREHOUSE_PASSWORD); + Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE STOCK (SKU VARCHAR(16) PRIMARY KEY, ON_HAND INT)"); + statement.execute("INSERT INTO STOCK VALUES ('mug', 12), ('tee', 0), ('tote', 31)"); + } catch (SQLException e) { + throw new IllegalStateException("cannot write the warehouse database: " + e.getMessage(), e); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/GuideExamplesTest.java b/src/test/java/com/darkcollective/relix/cli/docs/GuideExamplesTest.java new file mode 100644 index 0000000..0b43af0 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/GuideExamplesTest.java @@ -0,0 +1,134 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Runs every example in the command-line section of the programming guide and compares + * what it writes to standard output, and its exit status, with what the page shows + * (design §7.4), so that no page can be published with an example the command does not + * honour. + * + *

    An example is a {@code shell} block; the plain block on the line after it is its + * standard output, and an example with none must print nothing. Its exit status must be + * 0, unless it ends {@code ; echo "exit $?"}, which prints the status for the page to + * show. Standard error is not compared, except where an example sends it to standard + * output with {@code 2>&1}. + * + *

    Each page runs in a fresh copy of the fixture ({@link ExampleRunner}), its examples + * in order, so that one may build on what an earlier one did. The project starts trusted, + * except on the pages listed in {@link #UNTRUSTED}, which show trusting it. + * + *

    Comparison forgives trailing whitespace on a line, which a table pads with and an + * editor strips, and the path of the fixture's home directory, written {@code ~}. Nothing + * else. + * + *

    The examples run where {@code bash} is a POSIX shell, so not on Windows; the pages + * are written for one. Every other platform the build runs on runs them all. + */ +@DisabledOnOs(OS.WINDOWS) +@DisplayName("The command-line guide's examples print what the pages show") +class GuideExamplesTest { + + /** Pages whose project starts untrusted, because they show trusting it. */ + private static final Set UNTRUSTED = Set.of("catalog.md"); + + @TempDir + Path root; + + @TestFactory + List everyExample() { + Path directory = GuidePages.directory(); + List pages = new ArrayList<>(); + for (String page : GuidePages.files(directory)) { + List examples = GuidePages.examples(GuidePages.lines(directory.resolve(page))); + if (examples.isEmpty()) { + continue; + } + AtomicReference runner = new AtomicReference<>(); + List tests = new ArrayList<>(); + for (GuidePages.Example example : examples) { + tests.add(DynamicTest.dynamicTest(page + ":" + example.line(), () -> { + if (runner.get() == null) { + Path fixture = Files.createDirectories(root.resolve(page.replace('/', '_'))); + runner.set(new ExampleRunner(fixture, !UNTRUSTED.contains(page))); + } + check(page, example, runner.get().run(example.command())); + })); + } + pages.add(DynamicContainer.dynamicContainer(page, tests)); + } + assertThat(pages).as("pages with examples").isNotEmpty(); + return pages; + } + + private static void check(String page, GuidePages.Example example, ExampleRunner.Output output) { + String where = page + ":" + example.line() + "\n" + example.command() + "\n"; + assertThat(output.status()) + .as(where + "exits 0 (end it with ; echo \"exit $?\" to show another status)\n" + output) + .isZero(); + assertThat(normalise(output.out())) + .as(where + output) + .isEqualTo(example.expected() == null ? "" : normalise(example.expected())); + } + + private static String normalise(String text) { + return text.lines().map(String::stripTrailing).reduce((a, b) -> a + "\n" + b).orElse("").stripTrailing(); + } + + @Test + @DisplayName("every command a page shows a reader is a shell example, which runs") + void noUnrunCommandBlocks() { + Path directory = GuidePages.directory(); + List found = new ArrayList<>(); + for (String page : GuidePages.files(directory)) { + for (GuidePages.Block block : GuidePages.blocks(GuidePages.lines(directory.resolve(page)))) { + if (Set.of("bash", "sh", "console", "zsh", "shell-session").contains(block.lang())) { + found.add(page + ":" + block.line() + " (" + block.lang() + ")"); + } + } + } + assertThat(found).as("blocks the examples test would not run; write them as ```shell").isEmpty(); + } + + static String read(Path file) { + try { + return Files.readString(file); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/GuidePages.java b/src/test/java/com/darkcollective/relix/cli/docs/GuidePages.java new file mode 100644 index 0000000..46cfcf6 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/GuidePages.java @@ -0,0 +1,219 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +/** + * The command-line section of the programming guide, {@code docs/guide/command-line}: its + * pages, and the fenced blocks and links in each. + * + *

    The pages are Markdown in the dialect the site and the engine's manuals share. A + * fenced block is an example when its language is {@code shell}; the plain fence + * immediately after it, if there is one, is what the example writes to standard output. + */ +final class GuidePages { + + /** The section's directory, relative to the repository. */ + static final Path DIRECTORY = Path.of("docs/guide/command-line"); + + /** The language of a block the examples test runs. */ + static final String EXAMPLE = "shell"; + + private static final Pattern FENCE = Pattern.compile("^```\\s*([A-Za-z0-9_-]*)\\s*$"); + private static final Pattern LINK = Pattern.compile("\\[([^\\]]*)]\\(([^)\\s]+)\\)"); + private static final Pattern HEADING = Pattern.compile("^(#{1,6})\\s+(.+?)\\s*$"); + + private GuidePages() { + } + + /** + * A fenced block. + * + * @param lang its language, empty for a plain fence + * @param body its lines, joined + * @param line the line of its opening fence, from 1 + */ + record Block(String lang, String body, int line) { + } + + /** + * An example and what it prints. + * + * @param line the line of its opening fence + * @param command the block's text + * @param expected the plain block after it, or {@code null} when it prints nothing + */ + record Example(int line, String command, String expected) { + } + + /** + * A link written in a page. + * + * @param line the line it is on + * @param href its target, as written + */ + record Link(int line, String href) { + } + + /** {@return the section's directory: {@code relix.cli.guide}, or the repository's} */ + static Path directory() { + String property = System.getProperty("relix.cli.guide"); + return property != null ? Path.of(property) : DIRECTORY; + } + + /** + * Every Markdown file of the section, as paths relative to its directory. + * + * @param directory the section's directory + * @return the files, in path order + */ + static List files(Path directory) { + try (Stream walk = Files.walk(directory)) { + return walk.filter(p -> p.toString().endsWith(".md")) + .map(p -> directory.relativize(p).toString().replace('\\', '/')) + .sorted() + .toList(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + static List lines(Path file) { + try { + return Files.readAllLines(file, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * The fenced blocks of a page. + * + * @param lines the page + * @return its blocks, in order + */ + static List blocks(List lines) { + List blocks = new ArrayList<>(); + for (int i = 0; i < lines.size(); i++) { + Matcher open = FENCE.matcher(lines.get(i)); + if (!open.matches()) { + continue; + } + int start = i; + List body = new ArrayList<>(); + for (i++; i < lines.size() && !lines.get(i).strip().equals("```"); i++) { + body.add(lines.get(i)); + } + blocks.add(new Block(open.group(1), String.join("\n", body), start + 1)); + } + return blocks; + } + + /** + * The examples of a page: each {@code shell} block, paired with the plain block that + * follows on the very next line. + * + * @param lines the page + * @return its examples, in order + */ + static List examples(List lines) { + List blocks = blocks(lines); + List examples = new ArrayList<>(); + for (int i = 0; i < blocks.size(); i++) { + Block block = blocks.get(i); + if (!block.lang().equals(EXAMPLE)) { + continue; + } + String expected = null; + if (i + 1 < blocks.size()) { + Block next = blocks.get(i + 1); + int closing = block.line() + (int) block.body().lines().count() + 1; + if (next.lang().isEmpty() && next.line() == closing + 1) { + expected = next.body(); + i++; + } + } + examples.add(new Example(block.line(), block.body(), expected)); + } + return examples; + } + + /** + * The links of a page, outside its fenced blocks and code spans. + * + * @param lines the page + * @return its links, in order + */ + static List links(List lines) { + List links = new ArrayList<>(); + boolean fenced = false; + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i); + if (FENCE.matcher(line).matches()) { + fenced = !fenced; + continue; + } + if (fenced) { + continue; + } + Matcher link = LINK.matcher(line.replaceAll("`[^`]*`", "")); + while (link.find()) { + links.add(new Link(i + 1, link.group(2))); + } + } + return links; + } + + /** + * The anchors a page's headings get on the site: the heading's text, lower-cased, with + * each run of anything but a letter or digit made a hyphen. + * + * @param lines the page + * @return its anchors + */ + static List anchors(List lines) { + List anchors = new ArrayList<>(); + boolean fenced = false; + for (String line : lines) { + if (FENCE.matcher(line).matches()) { + fenced = !fenced; + continue; + } + Matcher heading = HEADING.matcher(line); + if (!fenced && heading.matches()) { + anchors.add(slug(heading.group(2).replace("`", ""))); + } + } + return anchors; + } + + static String slug(String text) { + String slug = text.toLowerCase(java.util.Locale.ROOT) + .replaceAll("[^a-z0-9]+", "-") + .replaceAll("(^-|-$)", ""); + return slug.isEmpty() ? "section" : slug; + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/GuideStructureTest.java b/src/test/java/com/darkcollective/relix/cli/docs/GuideStructureTest.java new file mode 100644 index 0000000..0e9a0d9 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/GuideStructureTest.java @@ -0,0 +1,169 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import com.darkcollective.relix.cli.packaging.CommandPages; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Holds the command-line section of the programming guide together: every command has a + * page and the pages are what the command model makes of it now, the index lists every + * page, and no link between pages, or into the engine's manuals, is broken. + * + *

    The section is rendered by the site inside the engine's programming guide, at + * {@code guide/command-line/}, so a link out of it is written as if it sat in the + * engine's {@code docs/guide/command-line}: {@code ../secrets.md} is a guide page and + * {@code ../../reference/README.md} the language reference. Such a link is resolved here + * against the pinned engine's manuals, which the build unpacks. + */ +@DisplayName("The command-line guide") +class GuideStructureTest { + + private static final Pattern INDEX_ROW_LINK = Pattern.compile("^\\|\\s*\\[[^\\]]+]\\(([^)#]+\\.md)\\)"); + + private final Path guide = GuidePages.directory(); + + @Test + @DisplayName("has the command pages and index the command model makes now") + void commandPagesAreCurrent() { + Map generated = CommandPages.generate(examples(), guide); + List stale = new ArrayList<>(); + generated.forEach((path, text) -> { + Path file = guide.resolve(path); + if (!Files.isRegularFile(file) || !GuideExamplesTest.read(file).equals(text)) { + stale.add(path); + } + }); + for (String page : GuidePages.files(guide)) { + if (page.startsWith(CommandPages.COMMANDS + "/") && !generated.containsKey(page)) { + stale.add(page + " (no such command)"); + } + } + assertThat(stale).as("stale command pages; run ./gradlew commandPages").isEmpty(); + } + + @Test + @DisplayName("gives every hand-written part of a command page a command") + void everyExamplesFileHasACommand() { + Set commands = new LinkedHashSet<>(); + for (String path : CommandPages.generate(examples(), guide).keySet()) { + if (path.startsWith(CommandPages.COMMANDS + "/")) { + commands.add(path.substring(CommandPages.COMMANDS.length() + 1)); + } + } + List orphans = new ArrayList<>(); + for (String file : GuidePages.files(examples())) { + if (!commands.contains(file)) { + orphans.add(file); + } + } + assertThat(orphans).as("files in docs/command-examples naming no command").isEmpty(); + } + + @Test + @DisplayName("lists every page in its index, and links nothing that is not there") + void indexListsEveryPage() { + Set listed = new LinkedHashSet<>(); + for (String line : GuidePages.lines(guide.resolve("README.md"))) { + Matcher row = INDEX_ROW_LINK.matcher(line); + if (row.find()) { + listed.add(row.group(1)); + } + } + Set pages = new LinkedHashSet<>(GuidePages.files(guide)); + pages.remove("README.md"); + assertThat(listed).as("pages the index lists").containsExactlyInAnyOrderElementsOf(pages); + } + + @Test + @DisplayName("has no broken link, to a page or to a heading") + void noBrokenLinks() { + Path engine = engineDocs(); + List broken = new ArrayList<>(); + for (String page : GuidePages.files(guide)) { + List lines = GuidePages.lines(guide.resolve(page)); + for (GuidePages.Link link : GuidePages.links(lines)) { + String problem = problem(page, link.href(), engine); + if (problem != null) { + broken.add(page + ":" + link.line() + ": " + link.href() + ": " + problem); + } + } + } + assertThat(broken).isEmpty(); + } + + /** What is wrong with a link written on {@code page}, or {@code null} when nothing is. */ + private String problem(String page, String href, Path engine) { + if (href.startsWith("https://") || href.startsWith("http://") || href.startsWith("mailto:")) { + return null; + } + int hash = href.indexOf('#'); + String target = hash < 0 ? href : href.substring(0, hash); + String anchor = hash < 0 ? null : href.substring(hash + 1); + Path file; + if (target.isEmpty()) { + file = guide.resolve(page); + } else { + if (!target.endsWith(".md")) { + return "not a page"; + } + // The section as it sits in the engine's tree: docs/guide/command-line/. + Path virtual = Path.of("docs/guide/command-line").resolve(page).resolveSibling(target).normalize(); + String path = virtual.toString().replace('\\', '/'); + if (path.equals("docs/guide/command-line/README.md")) { + return "the index is not a page of its own on the site; link a page"; + } + if (path.startsWith("docs/guide/command-line/")) { + file = guide.resolve(path.substring("docs/guide/command-line/".length())); + } else if (path.startsWith("docs/")) { + file = engine.resolve(path); + } else { + return "outside the manuals"; + } + } + if (!Files.isRegularFile(file)) { + return "no such page"; + } + if (anchor != null && !GuidePages.anchors(GuidePages.lines(file)).contains(anchor)) { + return "no such heading"; + } + return null; + } + + private static Path examples() { + String property = System.getProperty("relix.cli.commandExamples"); + return Path.of(property != null ? property : CommandPages.EXAMPLES); + } + + private static Path engineDocs() { + String property = System.getProperty("relix.engine.docs"); + assertThat(property).as("relix.engine.docs, the unpacked engine manuals").isNotBlank(); + return Path.of(property); + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/ShellCommand.java b/src/test/java/com/darkcollective/relix/cli/docs/ShellCommand.java new file mode 100644 index 0000000..544ff67 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/ShellCommand.java @@ -0,0 +1,223 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * An example that is one {@code relix} command and nothing a shell would have to do for + * it, so that it can run inside the test's JVM rather than through {@code bash}. + * + *

    The forms recognised are the ones the guide's pages write: + * + *

    + * [NAME=VALUE ...] relix ARG ... [< FILE | <<'EOF' ... EOF] [2>&1 | 2>/dev/null] [; echo "exit $?"]
    + * 
    + * + *

    An argument may be quoted with single or double quotes or escaped with a backslash, + * and a line may continue with a trailing backslash. Anything else a shell would act on + * — a pipe, an expansion, a glob, a {@code ~}, a second command — makes the example not + * one of these, and it runs through {@code bash}. The test errs that way: an example + * this class wrongly took for a plain command would run differently from how a reader's + * shell runs it, while one it wrongly passes to {@code bash} is only slower. + * + * @param environment the {@code NAME=VALUE} words before {@code relix} + * @param arguments the words after it + * @param stdinFile the file standard input is redirected from, or {@code null} + * @param heredoc the here-document standard input holds, or {@code null} + * @param mergeStderr whether {@code 2>&1} sends standard error to standard output + * @param echoStatus whether the command is followed by {@code echo "exit $?"} + */ +record ShellCommand(Map environment, List arguments, String stdinFile, + String heredoc, boolean mergeStderr, boolean echoStatus) { + + /** What prints the exit status after an example, so the page can show it. */ + static final String ECHO_STATUS = "; echo \"exit $?\""; + + private static final Pattern ASSIGNMENT = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*=.*", Pattern.DOTALL); + private static final Pattern HEREDOC = Pattern.compile("<<-?'([A-Za-z_]+)'"); + + /** + * Reads an example as a plain {@code relix} command, if it is one. + * + * @param text the example, as the page's {@code shell} block holds it + * @return the command, or empty when a shell has to run it + */ + static Optional parse(String text) { + String body = text.strip(); + String heredoc = null; + int newline = body.indexOf('\n'); + Matcher here = HEREDOC.matcher(newline < 0 ? body : body.substring(0, newline)); + if (here.find()) { + // relix ... <<'EOF' on the first line, the document after it, EOF alone last. + String delimiter = here.group(1); + List lines = body.lines().toList(); + if (lines.size() < 2 || !lines.getLast().equals(delimiter)) { + return Optional.empty(); + } + heredoc = String.join("\n", lines.subList(1, lines.size() - 1)) + "\n"; + body = lines.getFirst().substring(0, here.start()) + lines.getFirst().substring(here.end()); + } else { + body = body.replace("\\\n", " "); + if (body.contains("\n")) { + return Optional.empty(); + } + } + + boolean echoStatus = false; + if (body.endsWith(ECHO_STATUS)) { + echoStatus = true; + body = body.substring(0, body.length() - ECHO_STATUS.length()); + } + + List words = words(body); + if (words == null) { + return Optional.empty(); + } + Map environment = new LinkedHashMap<>(); + int i = 0; + while (i < words.size() && ASSIGNMENT.matcher(words.get(i)).matches()) { + String word = words.get(i++); + int equals = word.indexOf('='); + environment.put(word.substring(0, equals), word.substring(equals + 1)); + } + if (i >= words.size() || !words.get(i).equals("relix")) { + return Optional.empty(); + } + i++; + + List arguments = new ArrayList<>(); + String stdinFile = null; + boolean mergeStderr = false; + for (; i < words.size(); i++) { + String word = words.get(i); + switch (word) { + case "\u0000<" -> { + if (i + 1 >= words.size() || stdinFile != null || heredoc != null) { + return Optional.empty(); + } + stdinFile = words.get(++i); + } + case "\u00002>&1" -> mergeStderr = true; + case "\u00002>/dev/null" -> { + // Standard error is not compared, so discarding it changes nothing. + } + default -> { + if (word.startsWith("\u0000")) { + return Optional.empty(); + } + arguments.add(word); + } + } + } + return Optional.of(new ShellCommand(environment, arguments, stdinFile, heredoc, mergeStderr, echoStatus)); + } + + /** + * Splits a line into words as a shell would, or returns {@code null} when the line + * holds anything a shell would do more with than quote. The redirections this class + * knows come back as words of their own, marked with a leading NUL that no word typed + * on a page can hold. + */ + private static List words(String line) { + List words = new ArrayList<>(); + StringBuilder word = null; + int i = 0; + while (i < line.length()) { + char c = line.charAt(i); + if (c == ' ' || c == '\t') { + if (word != null) { + words.add(word.toString()); + word = null; + } + i++; + continue; + } + if (word == null) { + if (line.startsWith("2>&1", i) || line.startsWith("2>/dev/null", i)) { + String redirect = line.startsWith("2>&1", i) ? "2>&1" : "2>/dev/null"; + int end = i + redirect.length(); + if (end == line.length() || line.charAt(end) == ' ') { + words.add("\u0000" + redirect); + i = end; + continue; + } + } + if (c == '<') { + words.add("\u0000<"); + i++; + continue; + } + if (c == '~' || c == '#') { + return null; + } + word = new StringBuilder(); + } + switch (c) { + case '\'' -> { + int close = line.indexOf('\'', i + 1); + if (close < 0) { + return null; + } + word.append(line, i + 1, close); + i = close + 1; + } + case '"' -> { + i++; + while (i < line.length() && line.charAt(i) != '"') { + char d = line.charAt(i); + if (d == '$' || d == '`' || d == '!') { + return null; + } + if (d == '\\' && i + 1 < line.length() && "\"\\".indexOf(line.charAt(i + 1)) >= 0) { + d = line.charAt(++i); + } + word.append(d); + i++; + } + if (i >= line.length()) { + return null; + } + i++; + } + case '\\' -> { + if (i + 1 >= line.length()) { + return null; + } + word.append(line.charAt(i + 1)); + i += 2; + } + default -> { + if ("|&;<>()$`*?[]{}!".indexOf(c) >= 0) { + return null; + } + word.append(c); + i++; + } + } + } + if (word != null) { + words.add(word.toString()); + } + return words; + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/docs/TerminalMain.java b/src/test/java/com/darkcollective/relix/cli/docs/TerminalMain.java new file mode 100644 index 0000000..a722193 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/docs/TerminalMain.java @@ -0,0 +1,54 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.docs; + +import com.darkcollective.relix.cli.Main; +import com.darkcollective.relix.cli.io.Host; +import com.darkcollective.relix.cli.io.Interruption; + +import java.io.FileDescriptor; +import java.io.FileOutputStream; +import java.nio.file.Path; + +/** + * The {@code relix} an example's {@code bash} runs: the command's own entry point, over a + * host whose standard streams are terminals where a reader's would be. + * + *

    A reader typing {@code relix -e … | jq} has a terminal on {@code relix}'s standard + * input, and one on {@code jq}'s standard output; the example's run has files there, which + * {@link ExampleRunner} gives the shim. The shim compares each stream with them and says + * which is "the terminal" in two system properties, so that the command chooses the + * format, and whether to read a script from standard input, as it would for the reader. + */ +public final class TerminalMain { + + private TerminalMain() { + } + + /** + * Runs the command and exits with its status. + * + * @param args the command line + */ + public static void main(String[] args) { + Interruption interruption = new Interruption(); + Runtime.getRuntime().addShutdownHook(new Thread(interruption::interrupt, "relix-interrupt")); + Host host = new Host(System.in, new FileOutputStream(FileDescriptor.out), System.err, System.getenv(), + Path.of(System.getProperty("user.dir")), Path.of(System.getProperty("user.home")), + Boolean.getBoolean("relix.docs.stdinIsTerminal"), Boolean.getBoolean("relix.docs.stdoutIsTerminal")); + System.exit(Main.run(host, interruption, args)); + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/io/PagerTest.java b/src/test/java/com/darkcollective/relix/cli/io/PagerTest.java new file mode 100644 index 0000000..9617d52 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/io/PagerTest.java @@ -0,0 +1,53 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.io; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Which pager a long page goes through. */ +@DisplayName("The pager") +class PagerTest { + + @Test + @DisplayName("is $PAGER, run by the shell") + void pager() { + assertThat(Pager.command(Map.of("PAGER", "most -s"), false)) + .contains(List.of("sh", "-c", "most -s")); + assertThat(Pager.command(Map.of("PAGER", "more"), true)) + .contains(List.of("cmd", "/c", "more")); + } + + @Test + @DisplayName("is less when $PAGER is not set, except on Windows") + void defaults() { + assertThat(Pager.command(Map.of(), false)).contains(List.of("sh", "-c", "less")); + assertThat(Pager.command(Map.of(), true)).isEmpty(); + } + + @Test + @DisplayName("is none when $PAGER is empty or cat") + void none() { + assertThat(Pager.command(Map.of("PAGER", ""), false)).isEqualTo(Optional.empty()); + assertThat(Pager.command(Map.of("PAGER", "cat"), false)).isEmpty(); + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/io/RowSinkTest.java b/src/test/java/com/darkcollective/relix/cli/io/RowSinkTest.java new file mode 100644 index 0000000..6891d25 --- /dev/null +++ b/src/test/java/com/darkcollective/relix/cli/io/RowSinkTest.java @@ -0,0 +1,146 @@ +/* + * Copyright 2026 Darkcollective, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.darkcollective.relix.cli.io; + +import com.darkcollective.relix.cli.CommandFailure; +import com.darkcollective.relix.cli.ExitCode; +import com.darkcollective.relix.cli.render.OutputFormat; +import com.darkcollective.relix.processor.Row; +import com.darkcollective.relix.symbol.ColumnDefinition; +import com.darkcollective.relix.symbol.ScalarType; +import com.darkcollective.relix.symbol.Schema; +import com.darkcollective.relix.value.NumberValue; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * The row sink's end-of-pipe discipline (design §3.6, §7.5): counts, not times. + */ +@DisplayName("RowSink") +class RowSinkTest { + + private static final Schema SCHEMA = new Schema(List.of(new ColumnDefinition("n", ScalarType.NUMBER))); + + private final AtomicLong pulled = new AtomicLong(); + private final AtomicBoolean closed = new AtomicBoolean(); + + /** An endless row stream that counts what is pulled from it and notes its closing. */ + private Stream naturals() { + return Stream.iterate(0L, n -> n + 1) + .map(n -> { + pulled.incrementAndGet(); + return Row.of(SCHEMA, NumberValue.of(Long.toString(n))); + }) + .onClose(() -> closed.set(true)); + } + + private static OutputFormat.Encoder ndjson() { + return OutputFormat.NDJSON.encoder("R", SCHEMA, OutputFormat.Options.DEFAULTS); + } + + @ParameterizedTest(name = "fails after {0} rows") + @ValueSource(ints = {0, 1, 7, 100}) + @DisplayName("a sink that fails after N rows closes the stream, having pulled at most N + 1") + void failsAfterNRows(int n) { + RowSink sink = new RowSink(new FailsAfterLines(n), true); + + assertThatThrownBy(() -> sink.write(naturals(), ndjson())) + .isInstanceOfSatisfying(CommandFailure.class, f -> { + assertThat(f.exitCode()).isEqualTo(ExitCode.PIPE_CLOSED); + assertThat(f.getMessage()).isNull(); + }); + assertThat(closed).as("the stream was closed").isTrue(); + assertThat(pulled.get()).as("rows pulled").isLessThanOrEqualTo(n + 1L); + } + + @Test + @DisplayName("buffered, a closed pipe is noticed when the buffer is written, and the stream is closed") + void bufferedFailure() { + RowSink sink = new RowSink(new FailsAfterLines(3), false); + + assertThatThrownBy(() -> sink.write(naturals(), ndjson())).isInstanceOf(CommandFailure.class); + assertThat(closed).isTrue(); + } + + @Test + @DisplayName("line-buffered, each row reaches the reader before the next is pulled") + void lineBuffered() { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + RowSink sink = new RowSink(out, true); + Stream rows = Stream.iterate(0, i -> i < 3, i -> i + 1).map(i -> { + assertThat(out.toString(StandardCharsets.UTF_8).lines()).as("before row " + i).hasSize(i); + return Row.of(SCHEMA, NumberValue.of(Integer.toString(i))); + }); + + assertThat(sink.write(rows, ndjson())).isEqualTo(3); + assertThat(out.toString(StandardCharsets.UTF_8)).isEqualTo("{\"n\":0}\n{\"n\":1}\n{\"n\":2}\n"); + } + + @Test + @DisplayName("buffered, a result set reaches the reader at its end") + void flushedPerResultSet() { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + RowSink sink = new RowSink(out, false); + Stream rows = Stream.iterate(0, i -> i < 3, i -> i + 1).map(i -> { + assertThat(out.size()).as("nothing yet, before row " + i).isZero(); + return Row.of(SCHEMA, NumberValue.of(Integer.toString(i))); + }); + + sink.write(rows, ndjson()); + + assertThat(out.toString(StandardCharsets.UTF_8).lines()).hasSize(3); + } + + /** A reader that takes {@code lines} lines, then refuses every write. */ + private static final class FailsAfterLines extends OutputStream { + private int lines; + + FailsAfterLines(int lines) { + this.lines = lines; + } + + @Override + public void write(int b) throws IOException { + if (lines <= 0) { + throw new IOException("Broken pipe"); + } + if (b == '\n') { + lines--; + } + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + for (int i = 0; i < len; i++) { + write(b[off + i]); + } + } + } +} diff --git a/src/test/java/com/darkcollective/relix/cli/render/OutputFormatTest.java b/src/test/java/com/darkcollective/relix/cli/render/OutputFormatTest.java index 8653236..04a234d 100644 --- a/src/test/java/com/darkcollective/relix/cli/render/OutputFormatTest.java +++ b/src/test/java/com/darkcollective/relix/cli/render/OutputFormatTest.java @@ -18,10 +18,14 @@ import com.darkcollective.relix.processor.Row; import com.darkcollective.relix.value.ArrayValue; import com.darkcollective.relix.value.BooleanValue; +import com.darkcollective.relix.value.DateValue; +import com.darkcollective.relix.value.DurationValue; import com.darkcollective.relix.value.NullValue; import com.darkcollective.relix.value.NumberValue; import com.darkcollective.relix.value.StringValue; import com.darkcollective.relix.value.StructValue; +import com.darkcollective.relix.value.TimeValue; +import com.darkcollective.relix.value.TimestampValue; import com.darkcollective.relix.value.Value; import com.darkcollective.relix.symbol.ColumnDefinition; import com.darkcollective.relix.symbol.ScalarType; @@ -32,22 +36,64 @@ import java.io.PrintWriter; import java.io.StringWriter; +import java.math.BigDecimal; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.util.LinkedHashMap; import java.util.List; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -@DisplayName("OutputFormat — streaming CSV / JSON / Markdown writers") +@DisplayName("OutputFormat — streaming TSV / CSV / NDJSON / JSON / Markdown writers") final class OutputFormatTest { private static final Schema SCHEMA = new Schema(List.of( new ColumnDefinition("id", ScalarType.NUMBER), new ColumnDefinition("name", ScalarType.STRING))); + /** A heading with one column of every type, and a row holding one value of each. */ + private static final Schema EVERY_TYPE = new Schema(List.of( + new ColumnDefinition("n", ScalarType.NUMBER), + new ColumnDefinition("s", ScalarType.STRING), + new ColumnDefinition("b", ScalarType.BOOLEAN), + new ColumnDefinition("d", ScalarType.DATE), + new ColumnDefinition("t", ScalarType.TIME), + new ColumnDefinition("ts", ScalarType.TIMESTAMP), + new ColumnDefinition("dur", ScalarType.DURATION), + new ColumnDefinition("doc", ScalarType.ANY), + new ColumnDefinition("arr", ScalarType.ANY), + new ColumnDefinition("none", ScalarType.STRING))); + + private static Row everyType() { + LinkedHashMap fields = new LinkedHashMap<>(); + fields.put("name", new StringValue("Alice")); + fields.put("tags", new ArrayValue(List.of(new StringValue("x"), NumberValue.of("1")))); + return row(EVERY_TYPE, + new NumberValue(new BigDecimal("-12.50")), + new StringValue("plain"), + BooleanValue.of(true), + new DateValue(LocalDate.of(2026, 10, 8)), + new TimeValue(LocalTime.of(9, 5, 30)), + new TimestampValue(Instant.parse("2026-10-08T09:05:30Z")), + new DurationValue(Duration.ofMinutes(90)), + new StructValue(fields), + new ArrayValue(List.of(NumberValue.of("1"), NullValue.INSTANCE)), + NullValue.INSTANCE); + } + private static String render(OutputFormat format, Schema schema, List rows) { + return render(format, schema, rows, OutputFormat.Options.DEFAULTS); + } + + private static String render(OutputFormat format, Schema schema, List rows, + OutputFormat.Options options) { StringWriter sw = new StringWriter(); try (PrintWriter pw = new PrintWriter(sw)) { - format.write(pw, "R", schema, rows.stream()); + format.write(pw, "R", schema, rows.stream(), options); } return sw.toString(); } @@ -69,6 +115,8 @@ void caseInsensitive() { assertThat(OutputFormat.fromString("JSON")).isEqualTo(OutputFormat.JSON); assertThat(OutputFormat.fromString("Markdown")).isEqualTo(OutputFormat.MARKDOWN); assertThat(OutputFormat.fromString("table")).isEqualTo(OutputFormat.TABLE); + assertThat(OutputFormat.fromString("TSV")).isEqualTo(OutputFormat.TSV); + assertThat(OutputFormat.fromString("ndjson")).isEqualTo(OutputFormat.NDJSON); } @Test @@ -77,7 +125,181 @@ void unknownThrows() { assertThatThrownBy(() -> OutputFormat.fromString("xml")) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("xml") - .hasMessageContaining("csv"); + .hasMessageContaining("table, tsv, csv, ndjson, json, markdown"); + } + } + + // ── TSV ───────────────────────────────────────────────────────────────── + + @Nested + @DisplayName("TSV") + class Tsv { + + @Test + @DisplayName("writes a header row then one tab-separated line per row") + void headerAndRows() { + String out = render(OutputFormat.TSV, SCHEMA, List.of( + row(SCHEMA, NumberValue.of("1"), new StringValue("Alice")), + row(SCHEMA, NumberValue.of("2"), new StringValue("Bob, Jr. \"B\"")))); + + assertThat(out).isEqualTo("id\tname\n1\tAlice\n2\tBob, Jr. \"B\"\n"); + } + + @Test + @DisplayName("writes every value type as its display text") + void everyValueType() { + assertThat(render(OutputFormat.TSV, EVERY_TYPE, List.of(everyType()))).isEqualTo( + "n\ts\tb\td\tt\tts\tdur\tdoc\tarr\tnone\n" + + "-12.5\tplain\ttrue\t2026-10-08\t09:05:30\t2026-10-08T09:05:30Z\tPT1H30M" + + "\t{name: Alice, tags: [x, 1]}\t[1, NULL]\t\n"); + } + + @Test + @DisplayName("escapes a tab, a line feed, a carriage return and a backslash, so a row is one line") + void escaping() { + String out = render(OutputFormat.TSV, SCHEMA, List.of( + row(SCHEMA, NumberValue.of("1"), new StringValue("a\tb\nc\rd\\e")))); + + assertThat(out).isEqualTo("id\tname\n1\ta\\tb\\nc\\rd\\\\e\n"); + assertThat(out.lines()).hasSize(2); + } + + @Test + @DisplayName("escapes the header's names as it escapes values") + void escapesHeader() { + Schema schema = new Schema(List.of(new ColumnDefinition("a\tb", ScalarType.STRING))); + assertThat(render(OutputFormat.TSV, schema, List.of())).isEqualTo("a\\tb\n"); + } + + @Test + @DisplayName("writes NULL as an empty field, or as the spelling given, unescaped") + void nulls() { + List rows = List.of(row(SCHEMA, NumberValue.of("1"), NullValue.INSTANCE)); + + assertThat(render(OutputFormat.TSV, SCHEMA, rows)).isEqualTo("id\tname\n1\t\n"); + assertThat(render(OutputFormat.TSV, SCHEMA, rows, new OutputFormat.Options(true, "\\N", false, false))) + .isEqualTo("id\tname\n1\t\\N\n"); + } + + @Test + @DisplayName("leaves the header row out when asked") + void noHeader() { + String out = render(OutputFormat.TSV, SCHEMA, + List.of(row(SCHEMA, NumberValue.of("1"), new StringValue("Alice"))), + new OutputFormat.Options(false, null, false, false)); + + assertThat(out).isEqualTo("1\tAlice\n"); + } + + @Test + @DisplayName("an empty result is its header row alone") + void empty() { + assertThat(render(OutputFormat.TSV, SCHEMA, List.of())).isEqualTo("id\tname\n"); + } + } + + // ── NDJSON ────────────────────────────────────────────────────────────── + + @Nested + @DisplayName("NDJSON") + class Ndjson { + + @Test + @DisplayName("writes one object per line, with no header") + void objectPerLine() { + String out = render(OutputFormat.NDJSON, SCHEMA, List.of( + row(SCHEMA, NumberValue.of("1"), new StringValue("Alice")), + row(SCHEMA, NumberValue.of("2"), new StringValue("Bob")))); + + assertThat(out).isEqualTo(""" + {"id":1,"name":"Alice"} + {"id":2,"name":"Bob"} + """); + } + + @Test + @DisplayName("encodes every value type as the json format does, nested values natively") + void everyValueType() { + String out = render(OutputFormat.NDJSON, EVERY_TYPE, List.of(everyType())); + + assertThat(out).isEqualTo("{\"n\":-12.5,\"s\":\"plain\",\"b\":true,\"d\":\"2026-10-08\"," + + "\"t\":\"09:05:30\",\"ts\":\"2026-10-08T09:05:30Z\",\"dur\":\"PT1H30M\"," + + "\"doc\":{\"name\":\"Alice\",\"tags\":[\"x\",1]},\"arr\":[1,null],\"none\":null}\n"); + assertThat(render(OutputFormat.JSON, EVERY_TYPE, List.of(everyType()))) + .isEqualTo("[\n " + out.strip() + "\n]\n"); + } + + @Test + @DisplayName("escapes a line break inside a string, so an object is one line") + void escaping() { + String out = render(OutputFormat.NDJSON, SCHEMA, List.of( + row(SCHEMA, NumberValue.of("1"), new StringValue("a\"b\n\tc\\")))); + + assertThat(out).isEqualTo("{\"id\":1,\"name\":\"a\\\"b\\n\\tc\\\\\"}\n"); + assertThat(out.lines()).hasSize(1); + } + + @Test + @DisplayName("writes NULL as JSON null, whatever spelling the delimited formats were given") + void nulls() { + List rows = List.of(row(SCHEMA, NumberValue.of("1"), NullValue.INSTANCE)); + + assertThat(render(OutputFormat.NDJSON, SCHEMA, rows, new OutputFormat.Options(false, "\\N", false, false))) + .isEqualTo("{\"id\":1,\"name\":null}\n"); + } + + @Test + @DisplayName("an empty result writes nothing") + void empty() { + assertThat(render(OutputFormat.NDJSON, SCHEMA, List.of())).isEmpty(); + } + + @Test + @DisplayName("tags each object with its result set's label when asked") + void tagged() { + String out = render(OutputFormat.NDJSON, SCHEMA, + List.of(row(SCHEMA, NumberValue.of("1"), new StringValue("Alice"))), + new OutputFormat.Options(true, null, false, true)); + + assertThat(out).isEqualTo("{\"_query\":\"R\",\"id\":1,\"name\":\"Alice\"}\n"); + } + } + + // ── Streaming ─────────────────────────────────────────────────────────── + + @Nested + @DisplayName("Streaming") + class Streaming { + + @Test + @DisplayName("tsv and ndjson write the first row before the input ends") + void firstRowBeforeTheEnd() { + for (OutputFormat format : List.of(OutputFormat.TSV, OutputFormat.NDJSON)) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + String[] seenAtSecondPull = {null}; + Stream rows = Stream.iterate(1, i -> i <= 2, i -> i + 1).map(i -> { + if (i == 2) { + // Pulled only after the first row was handed over: what has been + // written by now is what a reader of the pipe could already see. + pw.flush(); + seenAtSecondPull[0] = sw.toString(); + } + return row(SCHEMA, NumberValue.of(Integer.toString(i)), new StringValue("r" + i)); + }); + + format.write(pw, "R", SCHEMA, rows); + + assertThat(seenAtSecondPull[0]).as(format.name()).contains("r1").doesNotContain("r2"); + } + } + + @Test + @DisplayName("only the table holds its rows back") + void onlyTheTableBuffers() { + for (OutputFormat format : OutputFormat.values()) { + assertThat(format.streams()).as(format.name()).isEqualTo(format != OutputFormat.TABLE); + } } } diff --git a/src/test/resources/docs-fixture/home/.relix/catalog/00-regions.relix b/src/test/resources/docs-fixture/home/.relix/catalog/00-regions.relix new file mode 100644 index 0000000..b22f3d6 --- /dev/null +++ b/src/test/resources/docs-fixture/home/.relix/catalog/00-regions.relix @@ -0,0 +1,8 @@ +-- Your own tables: every directory under your home sees them. +Regions := [ +| city | region | +|--------|------------| +| London | South East | +| Leeds | North | +| York | North | +]; diff --git a/src/test/resources/docs-fixture/home/.relix/profiles.json b/src/test/resources/docs-fixture/home/.relix/profiles.json new file mode 100644 index 0000000..c289ff4 --- /dev/null +++ b/src/test/resources/docs-fixture/home/.relix/profiles.json @@ -0,0 +1,3 @@ +{ + "dev": { "WAREHOUSE_PASSWORD": "s3cret" } +} diff --git a/src/test/resources/docs-fixture/home/shop/.relix/catalog/00-sources.relix b/src/test/resources/docs-fixture/home/shop/.relix/catalog/00-sources.relix new file mode 100644 index 0000000..bc6ca4f --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/.relix/catalog/00-sources.relix @@ -0,0 +1,10 @@ +-- The shop's data. A relative path resolves against this file's directory. +source Customers from csv("../../data/customers.csv") { + header: true, + schema: { customer_id: NUMBER, name: STRING, city: STRING } +}; + +source Orders from csv("../../data/orders.csv") { + header: true, + schema: { order_id: NUMBER, customer_id: NUMBER, amount: NUMBER, status: STRING, placed: DATE } +}; diff --git a/src/test/resources/docs-fixture/home/shop/.relix/catalog/10-views.relix b/src/test/resources/docs-fixture/home/shop/.relix/catalog/10-views.relix new file mode 100644 index 0000000..8a7cd7f --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/.relix/catalog/10-views.relix @@ -0,0 +1,5 @@ +Completed := { SELECT status = 'completed' (Orders) }; + +Spend := { GROUP customer_id, SUM(amount) -> total (Completed) }; + +BigSpenders := { PROJECT name, total (Customers JOIN (SELECT total >= 200 (Spend))) }; diff --git a/src/test/resources/docs-fixture/home/shop/.relix/catalog/20-warehouse.relix b/src/test/resources/docs-fixture/home/shop/.relix/catalog/20-warehouse.relix new file mode 100644 index 0000000..cb4e96c --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/.relix/catalog/20-warehouse.relix @@ -0,0 +1,11 @@ +-- The stock database. Its address and credentials come from placeholders. +connection warehouse from database { + url: "${WAREHOUSE_URL}", + user: "${WAREHOUSE_USER}", + password: "${WAREHOUSE_PASSWORD}" +}; + +source Stock from warehouse { + table: "STOCK", + schema: { sku: STRING, on_hand: NUMBER } +}; diff --git a/src/test/resources/docs-fixture/home/shop/.relix/profiles.json b/src/test/resources/docs-fixture/home/shop/.relix/profiles.json new file mode 100644 index 0000000..3ee415f --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/.relix/profiles.json @@ -0,0 +1,6 @@ +{ + "dev": { + "WAREHOUSE_URL": "jdbc:h2:@HOME@/shop/data/warehouse", + "WAREHOUSE_USER": "shop" + } +} diff --git a/src/test/resources/docs-fixture/home/shop/data/customers.csv b/src/test/resources/docs-fixture/home/shop/data/customers.csv new file mode 100644 index 0000000..1124ade --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/data/customers.csv @@ -0,0 +1,5 @@ +customer_id,name,city +1,Alice,London +2,Bob,Leeds +3,Carol,London +4,Dan,York diff --git a/src/test/resources/docs-fixture/home/shop/data/orders.csv b/src/test/resources/docs-fixture/home/shop/data/orders.csv new file mode 100644 index 0000000..aa5f79c --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/data/orders.csv @@ -0,0 +1,5 @@ +order_id,customer_id,amount,status,placed +1,1,120,completed,2026-09-02 +2,1,80,completed,2026-09-14 +3,2,50,pending,2026-09-20 +4,3,200,completed,2026-10-01 diff --git a/src/test/resources/docs-fixture/home/shop/data/returns.ndjson b/src/test/resources/docs-fixture/home/shop/data/returns.ndjson new file mode 100644 index 0000000..1131b81 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/data/returns.ndjson @@ -0,0 +1,2 @@ +{"order_id": 2, "reason": "damaged", "refund": {"amount": 80, "currency": "GBP"}} +{"order_id": 4, "reason": "late", "refund": {"amount": 20, "currency": "GBP"}} diff --git a/src/test/resources/docs-fixture/home/shop/reports/.relix/catalog/10-sample.relix b/src/test/resources/docs-fixture/home/shop/reports/.relix/catalog/10-sample.relix new file mode 100644 index 0000000..aa0a3b4 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/reports/.relix/catalog/10-sample.relix @@ -0,0 +1,5 @@ +-- Reports are written against a fixed sample rather than the live orders. +source Orders from csv("../../sample-orders.csv") { + header: true, + schema: { order_id: NUMBER, customer_id: NUMBER, amount: NUMBER, status: STRING, placed: DATE } +}; diff --git a/src/test/resources/docs-fixture/home/shop/reports/sample-orders.csv b/src/test/resources/docs-fixture/home/shop/reports/sample-orders.csv new file mode 100644 index 0000000..5ce152b --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/reports/sample-orders.csv @@ -0,0 +1,3 @@ +order_id,customer_id,amount,status,placed +101,1,500,completed,2026-01-05 +102,4,250,completed,2026-01-06 diff --git a/src/test/resources/docs-fixture/home/shop/scripts/broken.relix b/src/test/resources/docs-fixture/home/shop/scripts/broken.relix new file mode 100644 index 0000000..995f15d --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/scripts/broken.relix @@ -0,0 +1 @@ +query { SELECT amount > 100 (Ordrs) }; diff --git a/src/test/resources/docs-fixture/home/shop/scripts/by-city.relix b/src/test/resources/docs-fixture/home/shop/scripts/by-city.relix new file mode 100644 index 0000000..bfb5427 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/scripts/by-city.relix @@ -0,0 +1,2 @@ +-- The customers of one city, named when the script runs: relix -a city=London +query { PROJECT name (SELECT city = $city (Customers)) }; diff --git a/src/test/resources/docs-fixture/home/shop/scripts/messy.relix b/src/test/resources/docs-fixture/home/shop/scripts/messy.relix new file mode 100644 index 0000000..a533d40 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/scripts/messy.relix @@ -0,0 +1,3 @@ +-- Orders worth a look +Large:={select amount>100(Orders)}; +query {project order_id,amount (Large)}; diff --git a/src/test/resources/docs-fixture/home/shop/scripts/top-customers.relix b/src/test/resources/docs-fixture/home/shop/scripts/top-customers.relix new file mode 100644 index 0000000..a2fff29 --- /dev/null +++ b/src/test/resources/docs-fixture/home/shop/scripts/top-customers.relix @@ -0,0 +1,2 @@ +-- The customers who have spent the most, best first. +query { SORT total DESC (PROJECT name, total (Customers JOIN Spend)) }; diff --git a/src/test/resources/examples/introspection.relix b/src/test/resources/examples/introspection.relix new file mode 100644 index 0000000..3593147 --- /dev/null +++ b/src/test/resources/examples/introspection.relix @@ -0,0 +1,64 @@ +-- ─── Introspection ─────────────────────────────────────────────────────────── +-- relix describes itself. The reserved `relix` namespace exposes read-only +-- catalog relations you can query like any other relation (no new syntax): +-- +-- relix.relations every relation (name, kind, namespace, materialization) +-- relix.columns every column (relation, column, type, ordinal) +-- relix.functions every function (name, category, arity, return_type, …) +-- relix.dependencies lineage edges (dependent, depends_on) +-- relix.plan the logical expression tree, one row per IR node +-- (query, node_id, parent_id, ordinal, op, label, …) +-- +-- Run: relix --exec examples/introspection.relix +-- See: relix help catalog + +namespace shop; + +-- ─── A small schema to introspect ───────────────────────────────────────────── + +Customers := [| customer_id | name | city | + | 1 | Alice | London | + | 2 | Bob | Leeds | + | 3 | Carol | London | + ]; + +Orders := [| order_id | customer_id | amount | status | + | 1 | 1 | 120 | completed | + | 2 | 1 | 80 | completed | + | 3 | 2 | 50 | pending | + | 4 | 3 | 200 | completed | + ]; + +-- A little chain of views, so lineage has something to walk. +Completed := { σ status = "completed" (Orders) }; +PerCustomer := { γ customer_id, SUM(amount) → spent (Completed) }; +TopSpenders := { τ spent DESC (PerCustomer) }; +Report := { π name, spent (TopSpenders ⋈ Customers) }; + +-- ─── Introspection queries ──────────────────────────────────────────────────── + +-- 1. The catalogue of relations in this script. +query { τ kind, name (relix.relations) }; + +-- 2. Every column of every relation — the information_schema.columns analogue. +query { τ relation, ordinal (relix.columns) }; + +-- 3. Which functions are safe to fold / cache (pure AND deterministic)? +query { τ name (σ pure = "true" ∧ deterministic = "true" (relix.functions)) }; + +-- 4. Direct lineage edges: which relation reads which. +query { relix.dependencies }; + +-- 5. Transitive lineage — everything `Report` (transitively) reads from. +-- relix.dependencies holds the direct edges; CLOSURE makes them transitive +-- (the recursive-CTE problem, in one line). +Lineage := { CLOSURE dependent, depends_on (relix.dependencies) }; +query { π depends_on (σ dependent = "Report" (Lineage)) }; + +-- 6. IR size per view — which queries are the most complex? +-- relix.plan turns each view's expression tree into queryable rows. +query { γ query, COUNT(node_id) → nodes (relix.plan) }; + +-- 7. The expression tree of `Report`, in node order — the queryable form of the +-- ASCII `:tree`. parent_id wires each node to its parent (NULL at the root). +query { τ node_id (π node_id, parent_id, op, label (σ query = "Report" (relix.plan))) }; diff --git a/src/test/resources/examples/league.relix b/src/test/resources/examples/league.relix new file mode 100644 index 0000000..2d80940 --- /dev/null +++ b/src/test/resources/examples/league.relix @@ -0,0 +1,158 @@ +-- ─── Five-a-Side League ────────────────────────────────────────────────────── +-- A fictional five-a-side football league, three rounds played so far. +-- Run: relix --exec examples/league.relix + +namespace league; + +-- ─── Master data ───────────────────────────────────────────────────────────── + +Teams := [| team_id | name | city | founded | + | 1 | Red Rockets | London | 2010 | + | 2 | Blue Bolts | Manchester | 2008 | + | 3 | Green Giants | Birmingham | 2015 | + | 4 | Yellow Yellows | Leeds | 2012 | + | 5 | Purple Pythons | Bristol | 2018 | + ]; + +Players := [| player_id | name | team_id | position | age | + | 1 | Sam Shaw | 1 | striker | 28 | + | 2 | Tom Turner | 1 | keeper | 31 | + | 3 | Lena Lopez | 2 | striker | 25 | + | 4 | Kai Kim | 2 | mid | 27 | + | 5 | Ravi Patel | 3 | striker | 24 | + | 6 | Mia Moore | 3 | mid | 29 | + | 7 | Joe Jones | 4 | striker | 33 | + | 8 | Ana Anand | 4 | keeper | 26 | + | 9 | Ben Baker | 5 | striker | 22 | + | 10 | Zoe Zhang | 5 | mid | 30 | + ]; + +-- Each match records home and away teams, goals scored, and the round number +Matches := [| match_id | home_id | away_id | home_goals | away_goals | round | + | 1 | 1 | 2 | 3 | 1 | 1 | + | 2 | 3 | 4 | 0 | 2 | 1 | + | 3 | 5 | 1 | 1 | 1 | 1 | + | 4 | 2 | 3 | 2 | 2 | 2 | + | 5 | 4 | 5 | 3 | 0 | 2 | + | 6 | 1 | 3 | 2 | 0 | 2 | + | 7 | 2 | 4 | 1 | 1 | 3 | + | 8 | 5 | 3 | 2 | 3 | 3 | + | 9 | 1 | 4 | 4 | 2 | 3 | + ]; + +Goals := [| goal_id | match_id | player_id | minute | + | 1 | 1 | 1 | 12 | + | 2 | 1 | 1 | 34 | + | 3 | 1 | 4 | 45 | + | 4 | 1 | 1 | 67 | + | 5 | 2 | 7 | 23 | + | 6 | 2 | 7 | 55 | + | 7 | 3 | 9 | 30 | + | 8 | 3 | 1 | 72 | + | 9 | 4 | 3 | 18 | + | 10 | 4 | 3 | 60 | + | 11 | 4 | 5 | 40 | + | 12 | 4 | 6 | 80 | + | 13 | 5 | 7 | 15 | + | 14 | 5 | 7 | 44 | + | 15 | 5 | 7 | 70 | + | 16 | 6 | 1 | 25 | + | 17 | 6 | 2 | 58 | -- keepers get on the scoresheet too + | 18 | 7 | 3 | 37 | + | 19 | 7 | 7 | 62 | + | 20 | 8 | 9 | 10 | + | 21 | 8 | 5 | 33 | + | 22 | 8 | 6 | 50 | + | 23 | 8 | 9 | 62 | + | 24 | 8 | 5 | 88 | + | 25 | 9 | 1 | 8 | + | 26 | 9 | 1 | 29 | + | 27 | 9 | 7 | 41 | + | 28 | 9 | 1 | 63 | + | 29 | 9 | 1 | 77 | + | 30 | 9 | 8 | 55 | -- and so does the other one + ]; + +-- ─── Derived relations ─────────────────────────────────────────────────────── + +-- Rename match columns so we can distinguish home vs away in joins +HomeView := { ρ HomeView(match_id, team_id, goals_for, goals_against, round) ( + π match_id, home_id, home_goals, away_goals, round (Matches) +) }; + +AwayView := { ρ AwayView(match_id, team_id, goals_for, goals_against, round) ( + π match_id, away_id, away_goals, home_goals, round (Matches) +) }; + +-- All results from each team's perspective +AllResults := { HomeView ∪ AwayView }; + +-- ─── Queries ───────────────────────────────────────────────────────────────── + +-- Top scorers (players with at least 2 goals), sorted descending +GoalTally := { γ player_id, COUNT(goal_id) → goals (Goals) }; +TopScorers := { + τ goals DESC ( + π name, goals ( + σ goals >= 2 (GoalTally ⋈ Players) + ) + ) +}; + +-- Rename GoalTally so scorer_id ≠ player_id; this lets the semi/anti-join +-- predicate resolve the two sides unambiguously (qualifier stripping is safe +-- only when the column names differ between left and right). +ScoredBy := { ρ ScoredBy(scorer_id, goals) (GoalTally) }; + +-- Strikers who have scored at least one goal (semi-join) +StrikersScoredAGoal := { + σ position = "striker" (Players) + ⋉ Players.player_id = ScoredBy.scorer_id + ScoredBy +}; + +-- Players who have NOT scored yet (anti-join) +NotYetScored := { + Players ▷ Players.player_id = ScoredBy.scorer_id ScoredBy +}; + +-- Goals per team (Goals → Players gives team_id; join Teams for names) +-- Rename Teams.name → team_name to avoid clashing with Players.name in the join +TeamInfo := { ρ TeamInfo(team_id, team_name, city, founded) (Teams) }; + +TeamGoals := { + γ team_id, COUNT(goal_id) → team_goals ( + Goals ⋈ Players + ) +}; +TeamGoalsNamed := { π team_name, team_goals (TeamGoals ⋈ TeamInfo) }; + +-- League standings: union home + away perspectives, then aggregate +-- (HomeView ∪ AwayView already unified above as AllResults) +TeamRecord := { + γ team_id, COUNT(match_id) → played, + SUM(goals_for) → scored, SUM(goals_against) → conceded + (AllResults) +}; +Standings := { + τ scored DESC ( + π team_name, played, scored, conceded (TeamInfo ⋈ TeamRecord) + ) +}; + +-- Teams that have played in round 3 (semi-join; home_id ≠ team_id — no clash) +Round3Teams := { π home_id (σ round = 3 (Matches)) }; +PlayedRound3 := { + Teams ⋉ Teams.team_id = Round3Teams.home_id Round3Teams +}; + +-- Distinct cities represented in the league +LeagueCities := { δ (π city (Teams)) }; + +query TopScorers; +query StrikersScoredAGoal; +query NotYetScored; +query TeamGoalsNamed; +query Standings; +query PlayedRound3; +query LeagueCities; diff --git a/src/test/resources/examples/library.relix b/src/test/resources/examples/library.relix new file mode 100644 index 0000000..1c894db --- /dev/null +++ b/src/test/resources/examples/library.relix @@ -0,0 +1,99 @@ +-- ─── Library ──────────────────────────────────────────────────────────────── +-- A small book-lending library. All data is inline — no external sources. +-- Run: relix --exec examples/library.relix + +namespace library; + +-- ─── Catalogue ─────────────────────────────────────────────────────────────── + +Books := [| book_id | title | author | genre | year | pages | + | 1 | Dune | Frank Herbert | sci-fi | 1965 | 688 | + | 2 | Foundation | Isaac Asimov | sci-fi | 1951 | 244 | + | 3 | Neuromancer | William Gibson | sci-fi | 1984 | 271 | + | 4 | Shogun | James Clavell | fiction | 1975 | 1210 | + | 5 | The Name of the Rose | Umberto Eco | mystery | 1980 | 502 | + | 6 | Hyperion | Dan Simmons | sci-fi | 1989 | 482 | + | 7 | Foucault's Pendulum | Umberto Eco | fiction | 1988 | 641 | + | 8 | The Left Hand of Darkness| Ursula K. Le Guin | sci-fi | 1969 | 286 | + | 9 | Snow Crash | Neal Stephenson | sci-fi | 1992 | 440 | + ]; + +Members := [| member_id | name | city | joined | + | 1 | Alice | London | 2019 | + | 2 | Bob | Manchester | 2020 | + | 3 | Carol | London | 2021 | + | 4 | Dave | Bristol | 2020 | + | 5 | Eve | London | 2022 | + ]; + +-- Each loan records who borrowed which book; status: "out" or "returned" +Loans := [| loan_id | book_id | member_id | status | + | 1 | 1 | 1 | returned | + | 2 | 2 | 1 | out | + | 3 | 3 | 2 | returned | + | 4 | 1 | 3 | out | + | 5 | 4 | 2 | returned | + | 6 | 5 | 4 | returned | + | 7 | 6 | 3 | returned | + | 8 | 2 | 5 | out | + | 9 | 7 | 1 | returned | + | 10 | 8 | 4 | out | + ]; + +-- ─── Queries ───────────────────────────────────────────────────────────────── + +-- Books currently on loan (not yet returned) +OnLoan := { σ status = "out" (Loans) }; + +-- Full details of overdue books with borrower names +OnLoanDetails := { + π title, name, city ( + (OnLoan ⋈ Books) ⋈ Members + ) +}; + +-- Books that have never been borrowed (anti-join). +-- Book 9 is in the catalogue but in no loan, so it is the one row this returns — +-- an anti-join that always came back empty would demonstrate nothing. +NeverBorrowed := { Books ▷ Books.book_id = Loans.book_id Loans }; + +-- Prolific readers: members who have borrowed more than once +LoanCounts := { γ member_id, COUNT(loan_id) → times_borrowed (Loans) }; +ProlificReaders := { + π name, times_borrowed ( + σ times_borrowed > 1 (LoanCounts ⋈ Members) + ) +}; + +-- Sci-fi books sorted by year, most recent first +SciFiByYear := { + τ year DESC ( + σ genre = "sci-fi" (Books) + ) +}; + +-- Pages per genre — genres with their total and average page counts +GenreStats := { + γ genre, COUNT(book_id) → book_count, SUM(pages) → total_pages (Books) +}; + +-- Authors with multiple books in the catalogue +AuthorTally := { γ author, COUNT(book_id) → books_in_catalogue (Books) }; +ProlificAuthors := { σ books_in_catalogue > 1 (AuthorTally) }; + +-- London members who currently have a book out (semi-join). +-- The condition names the two sides of *this* join — the left is the filtered +-- Members, not the Loans the right side was derived from. +LondonBorrowers := { + σ city = "London" (Members) ⋉ + Members.member_id = OnLoan.member_id + OnLoan +}; + +query OnLoanDetails; +query NeverBorrowed; +query ProlificReaders; +query SciFiByYear; +query GenreStats; +query ProlificAuthors; +query LondonBorrowers;