diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 732b828..09d36c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -213,10 +213,81 @@ jobs: dotnet-version: | 8.0.x 10.0.x - - name: Run independent .NET adapter against Core + - name: Run independent .NET adapter and SDK against Core run: | dotnet test test/JacquardHost.Tests -f net8.0 dotnet test test/JacquardHost.Tests -f net10.0 working-directory: dotnet env: JACQUARD_REQUIRE_LIVE: "1" + - name: Run the four example projects through the .NET SDK + working-directory: dotnet + run: | + set -euo pipefail + # The SDK refuses a program cache under a directory another user + # could write; check $RUNNER_TEMP's ancestry here by the same rule, + # so a runner set up otherwise fails clearly rather than inside the + # SDK. A group-writable level passes only when its group is the + # user's private group: the user's primary group, named as the user, + # no other listed member, and no access ACL; and getent passwd + # succeeds, prints at most 16 MiB ending in a newline, lists at most + # 100,000 well-formed entries with no name twice, lists the user + # with that primary group, and lists no account of another uid + # with it. + me=$(id -un); uid=$(id -u); gid=$(id -g) + private=no + listing=$(mktemp) + trap 'rm -f "$listing"' EXIT + passwd_status=0 + # At most one byte over the cap is kept; a longer listing fails (getent is stopped) and is refused. + getent passwd | head -c 16777217 > "$listing" || passwd_status=$? + size=$(wc -c < "$listing") + group_status=0 + group_entry=$(getent group "$gid") || group_status=$? + if [ "$passwd_status" = 0 ] && [ "$group_status" = 0 ] \ + && [ "$size" -gt 0 ] && [ "$size" -le 16777216 ] \ + && [ "$(tail -c 1 "$listing" | od -An -tx1 | tr -d ' \n')" = 0a ] \ + && awk -F: -v n="$me" -v u="$uid" -v g="$gid" ' + NF != 7 || $1 == "" || $3 !~ /^(0|[1-9][0-9]*)$/ || $4 !~ /^(0|[1-9][0-9]*)$/ || seen[$1]++ { bad = 1; exit } + $1 == n { if ($3 != u || $4 != g) { bad = 1; exit }; listed = 1; next } + $4 == g && $3 != u { bad = 1; exit } + END { exit (bad || !listed || NR > 100000) }' "$listing" \ + && [ "$(printf '%s\n' "$group_entry" | cut -d: -f1)" = "$me" ] \ + && [ -z "$(printf '%s\n' "$group_entry" | cut -d: -f4 | tr , '\n' | grep -vx "$me")" ]; then + private=yes + fi + rm -f "$listing" + d=$(realpath "$RUNNER_TEMP") + while :; do + mode=$(stat -c %a "$d"); owner=$(stat -c %u "$d"); group=$(stat -c %g "$d") + bad= + if [ "$owner" != "$uid" ] && [ "$owner" != 0 ]; then + bad="owned by another user" + elif [ $((0$mode & 01000)) = 0 ] && [ $((0$mode & 02)) != 0 ]; then + bad="writable by others" + elif [ $((0$mode & 01000)) = 0 ] && [ $((0$mode & 020)) != 0 ]; then + if [ "$group" != "$gid" ] || [ "$private" != yes ]; then + bad="writable by its group, which is not your private group" + elif ls -ld "$d" | cut -c11 | grep -q '+'; then + bad="writable by its group, with an access control list" + fi + fi + if [ -n "$bad" ]; then + echo "::error::$d (mode $mode, owner $owner, group $group) is $bad; the .NET SDK refuses a cache under it" >&2 + exit 1 + fi + [ "$d" = / ] && break + d=$(dirname "$d") + done + dotnet build examples/JacquardExamples + run() { dotnet examples/JacquardExamples/bin/Debug/net8.0/JacquardExamples.dll --workdir "$RUNNER_TEMP/dotnet-examples" "$@"; } + run quickstart + run planner | grep -q '"score": 16' + run triage | grep -q '"urgency": "WATCH"' + run fulfillment | grep -q '"$tag": "Fulfilled"' + run fulfillment | grep -q '"$tag": "DuplicateOrder"' + status=0 + run --workdir "$RUNNER_TEMP/dotnet-recovery" --fault publish-failed fulfillment || status=$? + test "$status" = 1 + run --workdir "$RUNNER_TEMP/dotnet-recovery" --deliver-pending fulfillment | grep -q '{"delivered":1}' + run --workdir "$RUNNER_TEMP/dotnet-recovery" --deliver-pending fulfillment | grep -q '{"delivered":0}' diff --git a/CHANGELOG.md b/CHANGELOG.md index 3391481..7a44550 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,8 +78,7 @@ codecs that parse integers exactly from their text, the session state machine, a bounded process carrier for `jacquard host worker`, and a `Host` with a closed operation registry. It carries the pinned kit as embedded - resources and replays all 21 recordings. It is protocol-level only, with no - SDK layer. .NET cannot start the worker in its own process group or + resources and replays all 21 recordings. .NET cannot start the worker in its own process group or interrupt a pipe read or write, so the carrier never calls one that can block: it waits on the C library's `poll(2)` and runs workers on Linux and macOS only. It kills the worker's process tree with `Process.Kill(true)`, @@ -88,6 +87,48 @@ `scripts/core_pin.py` now generates `dotnet/src/JacquardHost/Compat.cs` and keeps the .NET kit copy in sync, and CI runs the offline tests on .NET 8 and 10 and the live tests against the pinned Core. +- .NET SDK (`JacquardLang.Host.Sdk`): prepares and caches checked programs + with Core's public commands, maps .NET values through explicit `Codec` + objects (records through field readers and a constructor function, + enumerations, type hierarchies, `Option`, `Unit`, lazy codecs for + recursive types, and `Dynamic` for untyped code) without reflection over + members (enumeration codecs read `Enum` names and values, and hierarchy + codecs check case types with `Type`), binds + handlers to exact operation identities with the `HandlerContext` and its + `CancellationToken`, checks arguments against the checked signature before a + worker starts, and keeps each invocation's raw evidence. `MapBatches` runs + explicit batches. There is no `decimal` mapping: money crosses as integer + minor units, and `decimal`, `ulong`, `nuint`, `char`, `float`, and the + other numeric types with no Jacquard type do not compile as arguments. + Caller names over 4,096 chars are refused before they are looked up. + Core's commands run on the caller's thread through `poll(2)`, bounded in + time and output, and on Linux through `setsid`, whose process group is + killed when a command exits or is stopped; without `setsid`, descendants + are recorded from `/proc` within entry and byte caps, and on a stop Core + is frozen with `SIGSTOP` before the last record and the cleanup ends + within 750 ms. Only the cache's owner may write it: the root, each entry, + and each file must be owned by the current user and not group- or + world-writable, every directory above the root must be owned by the user + or root and not group- or world-writable unless it is sticky or the group + is the user's private group (Debian OpenSSH's rule, which looks for + another account with that primary group in the password database, listed + here by `getent passwd` in a separate process, with it and the `setsid` + it runs through root's alone, at most once per preparation or save, + limited to 5 seconds (then stopped within 750 ms) and 16 MiB, stopped by + the token, and refused on any failure or malformed entry; with stricter conditions, so a clone made with a `umask` of 002 + under user-private groups is accepted), a refusal names + every such directory and creates nothing, and the root's device and inode + are checked again before each use, each worker a prepared program starts + included. Replacement by the same user is not covered. + Every binding is derived again from Core's cached output on load, and a + preparation during which the Core executable or prelude changed is + refused. `Invocation.Save` refuses a target directory another user could + write, by the same directory rule, writes a new owner-only file through one + exclusively created handle, flushes it, checks the file at its path is the + one written, and renames it over the target. + The four example projects run through a `JacquardExamples` command in + `dotnet/examples/`, and CI runs them and the SDK's live tests against the + pinned Core. ## 0.2.0 — 2026-10-25 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ce606f1..d0a9217 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -146,12 +146,15 @@ fails if they differ). ## .NET -The independent .NET adapter lives in `dotnet/` as package `JacquardLang.Host` -(`dotnet/src/JacquardHost`). It targets .NET 8, has no runtime dependencies -beyond the .NET base class library, and does not call Python, Node.js, Go, -Rust, or Java; it calls the C library's `poll(2)` and runs workers on Linux -and macOS only. NUnit, its adapter, and the test SDK are its only (test) -dependencies. The .NET 8 or 10 SDK is required. +The independent .NET adapter and SDK live in `dotnet/` as package +`JacquardLang.Host` (`dotnet/src/JacquardHost`, the SDK in its `Sdk/` +directory). It targets .NET 8, has no runtime dependencies beyond the .NET base +class library, and does not call Python, Node.js, Go, Rust, or Java; it calls +the C library (`poll(2)`, and for the SDK `kill(2)`, `geteuid(2)`, `mkdir(2)`, +`statx(2)` or `stat(2)`, `getpwuid_r(3)`, `getgrgid_r(3)`, and `lgetxattr(2)`; it also runs `getent passwd`) and runs on Linux and macOS only. On Linux the +SDK runs Core's commands through util-linux `setsid` when one is installed. +NUnit, its adapter, and the test SDK are its only (test) dependencies. The +.NET 8 or 10 SDK is required. ```sh cd dotnet @@ -159,17 +162,38 @@ dotnet test test/JacquardHost.Tests -f net8.0 dotnet test test/JacquardHost.Tests -f net10.0 # with a .NET 10 SDK # With the pinned Core configured (eval "$(python3 ../scripts/core_pin.py install)"): JACQUARD_REQUIRE_LIVE=1 dotnet test test/JacquardHost.Tests -f net8.0 +dotnet build examples/JacquardExamples +dotnet examples/JacquardExamples/bin/Debug/net8.0/JacquardExamples.dll quickstart ``` -CI runs the offline tests on .NET 8 and 10, and the live tests on both in the -shared pinned-Core job. The build treats warnings as errors, with nullable +The example command's program cache is under `dotnet/.scratch/`, and the SDK +refuses it when another user could write a directory above it: one writable +by others, or by its group unless that group is your private group (named +after you, listing no other member, and no other account in the password +database having it as its primary group; a clone made with a `umask` of 002 +under user-private groups passes). The refusal names every such directory; +run `chmod go-w` on each, or pass `--workdir` under directories only you or +root may write. + +The SDK tests make their scratch directories under `TMPDIR` when neither a +group nor another user may write it or any directory above it (unless +sticky), and otherwise under `/tmp` when that holds, so the refusals they +expect name only the levels they make; when neither does, they fail asking +for such a `TMPDIR`. + +CI runs the offline tests on .NET 8 and 10, and the live tests on both and the +four example projects in the shared pinned-Core job. The build treats warnings as errors, with nullable reference types and the `8.0-recommended` analyzers. `dotnet/src/JacquardHost/Compat.cs` is generated and `dotnet/src/JacquardHost/kit/` is a byte copy of the pinned kit. `scripts/core_pin.py` maintains both; do not edit them by hand. The `JacquardHost.StandIn` program under `dotnet/test/` is a fixture for the -offline tests. Copy the repository `NOTICE` to `dotnet/NOTICE` when it changes -(a test fails if they differ). +offline tests. The `JacquardExamples` command under `dotnet/examples/` reads the +rules and inputs from `examples/` directly, so there is no copy to keep in +sync. The README's C# examples are compiled test sources under +`dotnet/test/JacquardHost.Tests/Readme/` (a test fails if they differ). Copy +the repository `NOTICE` to `dotnet/NOTICE` when it changes (a test fails if +they differ). ## Releasing diff --git a/MANIFEST.in b/MANIFEST.in index 3808d03..19294ab 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -25,3 +25,5 @@ prune dotnet/test/JacquardHost.Tests/bin prune dotnet/test/JacquardHost.Tests/obj prune dotnet/test/JacquardHost.StandIn/bin prune dotnet/test/JacquardHost.StandIn/obj +prune dotnet/examples/JacquardExamples/bin +prune dotnet/examples/JacquardExamples/obj diff --git a/README.md b/README.md index c72341d..86a018a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,22 @@ # Jacquard Host -Call checked Jacquard functions from Python, TypeScript, Go, Rust, or Java and implement +Call checked Jacquard functions from Python, TypeScript, Go, Rust, Java, or .NET and implement their declared host operations with ordinary services. Jacquard owns the decision logic; the host owns persistence, I/O, and application lifetimes. The SDK prepares and caches checked programs, converts native Python values, binds services to exact operation identities, and retains the raw evidence for -every invocation. Python 3.11+, Node.js 22+, Go 1.24+, Rust 1.88+, or Java 21+, -with no third-party runtime dependencies. The [TypeScript SDK](typescript/README.md), -the [Go SDK](go/README.md), the [Rust SDK](rust/README.md), and the -[Java SDK](java/README.md) implement the pinned protocol independently. The +every invocation. Python 3.11+, Node.js 22+, Go 1.24+, Rust 1.88+, Java 21+, or +.NET 8+, with no third-party runtime dependencies. The +[TypeScript SDK](typescript/README.md), the [Go SDK](go/README.md), the +[Rust SDK](rust/README.md), the [Java SDK](java/README.md), and the +[.NET SDK](dotnet/README.md) implement the pinned protocol independently. The TypeScript SDK supports async service methods, the Go SDK maps native Go types and context cancellation, the Rust SDK maps native Rust types through `record!` -and `variants!` declarations, and the Java SDK maps records, enums, and sealed -interfaces through explicit codecs. A protocol-level [.NET](dotnet/README.md) -(.NET 8+) adapter also implements the pinned protocol independently. It has a -closed operation registry and kit replay, but no SDK layer yet. +and `variants!` declarations, the Java SDK maps records, enums, and sealed +interfaces through explicit codecs, and the .NET SDK maps records, enumerations, +and type hierarchies through explicit codecs without reflection over members, +with `CancellationToken` cancellation. **Experimental:** the SDK targets Core's pinned `jacquard-host-v0` protocol. Core is installed separately. Each invocation runs one fresh, serial worker; @@ -25,7 +26,8 @@ the host and operating system are trusted. See [limits](docs/LIMITS.md). For Node.js, follow the [TypeScript walkthrough](typescript/README.md); for Go, the [Go walkthrough](go/README.md); for Rust, the [Rust walkthrough](rust/README.md); -for Java, the [Java walkthrough](java/README.md). For Python: +for Java, the [Java walkthrough](java/README.md); for .NET, the +[.NET walkthrough](dotnet/README.md). For Python: From a clone of this repository: diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md index a7f73fd..ba66d25 100644 --- a/docs/EVIDENCE.md +++ b/docs/EVIDENCE.md @@ -143,6 +143,28 @@ dependencies. | Pin consistency | `dotnet/src/JacquardHost/Compat.cs` is generated, and the embedded kit copy matches the pinned digests and the Python copy (`scripts/core_pin.py check`); `tests/test_core_pin.py` covers drift in both, and the offline Python suite passes | | Distribution | `scripts/check_distribution.py` passes; the source archive carries the .NET sources, `dotnet/NOTICE`, and the kit copy | +## .NET SDK and projects (2026-10-29) + +Recorded against the same `jacquard-core-0.3.0` release asset (linux-x86_64), +on Linux 7.0 with the .NET 8.0.427 and 10.0.403 SDKs (runtimes 8.0.33 and +10.0.14). The library still has no runtime dependencies beyond the base class +library; the SDK calls the C library's `kill(2)`, `geteuid(2)`, `mkdir(2)`, +`access(2)`, `realpath(3)`, `getpwuid_r(3)`, `getgrgid_r(3)`, `lgetxattr(2)`, and `statx(2)` (`stat(2)` and `fstat(2)` on macOS, compiled but not run), and on Linux runs +Core's commands and `getent passwd` (glibc's, here `files systemd sss`) through util-linux `setsid`, here 2.41.3. The test +dependencies are unchanged. + +| Lane | Result | +|---|---| +| .NET SDK offline (`dotnet test test/JacquardHost.Tests -f net8.0` with the .NET 8 SDK pinned as the CI 8.0 job pins it, and `-f net10.0`, without Core) | 224 tests pass on each runtime and 28 live tests are ignored: the adapter's 96 offline tests and 128 SDK ones. Core commands (22, Linux): exit status, output, and added environment, with stdin at end of file; output over its limits read and counted but not kept; a timeout and a cancellation each kill the command and its child; under `setsid` the command keeps its pid and its group and session ids equal it (with `setsid --fork` the test fails); `sh -c 'sleep 300 & echo $!'` succeeds and its child dies, and survives when the exit kill is disabled; a child that closed its output is killed too; a reaped leader's pid held by an unrelated group leader is not killed; a process outside the group holding the command's stdout is neither killed nor waited for; a stop kills a child and a grandchild; without groups a recorded descendant is killed at the exit (and survives with recording off) and a stop kills the tree; against a fake `/proc` (a seam, pids above Linux's largest), one record reads exactly its entry cap (20 of 50) and its byte cap (5 of 50 entries at 5 KiB), and 1,024 bytes of each 64 KiB `stat` file, and a 1,025-byte cap reads 1,024 bytes and then 1; a cancellation after the third read ends the record there, then Core is sent SIGSTOP and is seen stopped (state `T`) before the last record's first read, the last record ends at its 250 ms budget (exactly three reads on a test clock advancing 100 ms per read), and only then is Core sent SIGKILL; with 300 recorded fake descendants and a test clock that stands still except for 2 ms per read after the stop, the last record reads at 0 to 248 ms (125 reads) and Core is killed at 250 ms, the kills read at 250 to 498 ms (125 reads) and end at 500 ms, and the exit wait ends at 750 ms, so 250 of the 600 reads happen; a cancellation during the second of ten exit kills still checks the other eight; a cancellation during the exit cleanup is reported; a missing executable, and one `setsid` cannot exec, are start failures; an escaped flooder holds neither the caller nor more than the kept output; 40 commands leave the descriptor count unchanged, counted at once. Preparation against a fake Core (59, Linux): option and path validation (empty, NUL, and over-long paths refused with excerpts before the file system sees them, a relative path measured absolute); one version check across four threads and two runtimes; prelude discovery once per Core and prelude; output over 8 MiB; the 30-second bound (shortened by a seam) and cancellation killing the command's child; a waiting preparation observing its own cancellation; FIFO and directory sources refused without blocking; sources over 16 MiB refused from their sizes before reading; cancellation between sources, during source validation, during fingerprinting (one 64 KiB piece), between empty prelude files, during the failure cleanup, and as a warm load finishes; prelude walks over 256 MiB, with a FIFO, over the entry cap (lowered by a seam), and 1,500 directories deep; a changed `program.json` binding, a changed Core output, a rewritten digest, and a `program.json` replaced by a link refused; group-writable roots, entries, files, and directories refused and new entries owner-only; a foreign-owned root refused; an executable over 512 MiB refused from its size; source lists over 1,024 refused from their count with no element read; the cleanup stopping at the cap; a symbolic link at the cache root followed to its directory; directories the preparation did not make, and one made at the vacated staging path, left alone; the cache root renamed away and replaced (a seam) before staging, publishing, and a cold or warm load, and after a load read the entry, each refused; the executable replaced by a new file after `--version`, `check`, `hash`, and `run`, a prelude file rewritten after `check`, and a file added to the prelude after `run`, each refused with nothing published; the version check run again for the same executable bytes with a new modification time; an executable only its group and others may run refused by `access(2)` before it runs; a bindings dictionary read without the program's lock held; function, effect, binding, and operation names over 4,096 chars refused and names of exactly 4,096 looked up; a parent of the cache root writable by others, or by a group that is not the user's private group, and a grandparent writable by others, each refused naming it, a sticky world-writable parent and a 0755 one accepted, and a path through a link to the parent resolved to the real directory; with the password and group answers from a seam, a 0770 parent whose group is the user's private group (no members, or only the user) accepted, and refused when others may write it, when the group is not the user's primary group, is named otherwise, lists another member (alone or beside the user), or is unknown, or the user is unknown; the same 0770 parent, its group listing no members, refused when the password database (a seam) lists another account ("attacker", another uid) with that primary group, and accepted when the only other account has another primary group; the password database listing (`getent passwd`'s output and exit status from a seam) accepting only the user, exactly 100,000 entries, another name with the user's id, and id 0, and refusing another uid with the group, 100,001 entries, a nonzero exit, a timeout, a start failure, output over the cap, output cut within a line or before its last newline, no output, an empty line, six or eight fields, an empty name, a user or group id that is empty, signed, zero-padded, not decimal, padded, or over 32 bits, invalid UTF-8, the user or another name listed twice, the user missing, and the user listed with another group or uid, and a cancellation reported as a cancelled preparation with the caller's token passed through; the real runner with `/bin/sh` in `getent`'s place refusing a valid listing with exit status 1 and a command still running at a 100 ms limit, and a cancellation once it started stopping it as cancelled, neither command left running; `getent` refused when missing, when the first path present is a file this user owns (without falling through to `/usr/bin/getent`), and, with statuses altered through a seam, when its file is another user's, writable by group or others, or not a regular file, or its directory or `/` is not root's alone; a preparation whose token is cancelled while the listing runs failing as cancelled with nothing made, the listing having received that token; the SDK's verdict on the real `getent passwd` listing equal to the test's own (the test runs `getent passwd` itself and applies the rule; here it accepts this user), and on that listing with another account of this user's primary group added by a `/bin/sh` stand-in equal to it too (refused), while a full `setpwent`/`getpwent`/`endpwent` enumeration runs in the test process during each listing, and `sync`, whose primary group 65534 other uids share, refused; one preparation that checks a 0770 private-group parent twice (the cache root missing) listing the password database once, and a later one listing it again; one preparation whose account lookup answers "alice" for the first check and "alice-renamed" (same uid and group, both listed) for the second listing it once and refusing the parent at the second check; with `/bin/sh` as `getent` printing a valid listing, accepted, and refused when, with statuses altered through a seam, `setsid`'s file is another user's, writable by group or others, or not a regular file, or its directory is writable by its group (altered for `setsid`'s check only, since the stand-in `getent`'s real path, `/usr/bin/dash`, shares that directory), and accepted with such a `setsid` when commands run without groups; names that are not valid UTF-8 (an account, a group, or a member name) read as lookup errors; the account records read only within their buffer, with crafted pointers to another valid array refused for the name, a group name, a member name, and the member array, a member array whose terminating slot would lie past the end refused, and a name without its NUL in the buffer refused; the tests' scratch directories made under `/tmp`, owner-only, when `TMPDIR` is a 0775 directory; a private-group parent with an access ACL (`setfacl -m u:nobody:rwx`, mode 0770 from its mask) refused (the test is ignored where `/usr/bin/setfacl` is missing); the real `getpwuid_r` and `getgrgid_r` answering for this user; one refusal naming every unsafe level nearest first with its reason (a group-writable one and an other-writable one above it), and 16 of 18 with the other 2 counted; a cache under missing levels below an unsafe parent refused with nothing made; the cache root renamed away and back after preparation still used, and renamed away and replaced by a new owner-only directory refused at the next call of a function bound before and one bound after, with no worker started; the missing levels a preparation makes for a cache root each 0700; a source list and a bindings dictionary whose count, element, or enumerator throws wrapped in a `Preparation` or `Binding` exception keeping the cause. Data (26): signatures, hash groups, and display names; round trips and type conformance; the full Int range, Real bits, and `int` narrowing; records, enumerations, unions, nested `Option`, nullable codecs, tuples, custom codecs, and `Dynamic`; `ulong` (2^53 + 1), `nuint`, `char`, `float`, `Half`, `decimal`, `Int128`, `UInt128`, and `BigInteger` conversions and `Dynamic.Of` overloads obsolete as errors and, reached by reflection, refused, while the smaller integer types and `uint` still convert to `Int`; caller names over 4,096 chars refused before lookup (public, made, dynamic, record, and operation names); a lazy codec whose factory threw twice made on the third use and never again across 16 threads; two 64-level shared tuple graphs (2^64 paths) compared equal and unequal at once, and two lists of 1,000 tuples with no shared sub-value compared with no pair set made (a seam counts them); recursive types through a lazy codec made once across eight threads, a chain of a million lazy codecs, and one standing for itself refused; the node, text, frame, depth, and count bounds (a list whose items throw is refused from its count, tuples and records before an item is read, a constructor of 20 million fields before it is copied); a raw value 100,000 levels deep refused by its depth; a `Dynamic` over the hard limit refused before it is copied, an endless enumeration after one item past the limit; deep `Dynamic` equality and text without recursion; 64-char excerpts and the 8,192-char message cap, caller exceptions kept as inner exceptions; 100,000 layers of nested unions flattened to one case and round-tripped; a recursive union through a lazy case from eight threads; signature depth 64 accepted, 65 and 5,000 parentheses refused, 100,000 refused by the token cap. Batches and operations (19): argument counts refused before encoding; a throwing codec an invalid argument; cancellation before, during, and after batch validation, each an exact encode count with no worker; every batch validated and the first encoded again; no field or return value that could keep an encoding; a shared graph from a custom codec measured before its type is checked; rows that shrink from 10 to 3 during validation refused after one batch's four encodings; `Save` over a 0666 file leaving a 0600 file, and over a symbolic link replacing the link and leaving its target unchanged; `Save` into a 0777 directory refused with nothing written there, and into a sticky 0777 one saved; a temporary file replaced (a seam) after it was written and before its rename detected, the replacement neither renamed over the target nor removed, and nothing saved; `Save` into a 0770 private-group directory refused, with nothing written, when the password database (a seam) lists another account with that primary group, and saved when only the user has it; a failing status read of the temporary file's handle (a seam) raised, with the temporary file removed; handlers' decoding, results, failures, context, and token, and no handler starting once the call stopped during decoding. Examples (2): usage errors exit 2; the store removes only the temporary file it created | +| .NET SDK live (`JACQUARD_REQUIRE_LIVE=1`, `JACQUARD_BIN` and `JACQUARD_PRELUDE` set) | 20 tests pass on .NET 8 and on .NET 10: quickstart calls, `ToDynamic`, `Render`, and an owner-only saved inspection; the README examples; the 64-row triage oracle in batches of 8, with a codec failing on the second batch keeping the first and its exception; local refusals before any worker (a field over the Int range named by path, an over-deep call, a wrong type, a wrong count); missing handlers, a misspelled operation, and a 100-name service refused at binding with no handler run; unknown completion once with raw evidence; no handler starting once a real invocation stopped during decoding; a handler's token cancelled when the caller cancels; the 25-case planner oracle; cache reuse without Core, a byte-exact new entry, and corruption refused; a 2,500-`Option` signature left unbound with `safe` working cold and warm; a redirected name in `program.json` refused; the full Int range, Real bits, and nested options; spawn failure and cancellation; output overflow at batch 0; unit results kept after a later batch fails (E1602); too many bindings refused from the count; a throwing result codec keeping the result and its exception, a null mapping message, and an over-long save path; prelude contents keying the cache, and two concurrent preparations publishing one entry | +| .NET projects live | 3 further tests pass on both runtimes: all four `JacquardExamples` commands, including duplicate fulfillment, `--inspect` (owner-only), explicit delivery, a failed delivery printing no count, and usage errors; the command in a fresh process exiting 1 with E1613; and the three fulfillment faults recovered explicitly across restart, each receipt delivered once. The CI step's four projects and recovery commands also pass locally, as do the `dotnet/README.md` quickstart and its "Run the projects" and recovery commands with their default work directories, run from `dotnet/` in a checkout whose directories are 0775 (a `umask` of 002, and the user's private group), including an `--inspect` file saved into a 0775 directory and refused, with nothing written, in a 0777 one. The CI step's check of `$RUNNER_TEMP`'s ancestry, run locally with gawk and with mawk, passes a 0700, a 0775 private-group, and a sticky 1777 directory and fails a 0775 directory of the group `users`, a 0777 one, and a 0770 one with an access ACL; with a `getent` stand-in it fails the private group when the listing's last line lacks a newline (also when that line is the user's) or the listing is 18,208,113 bytes of well-formed entries, and passes one of 16,184,113 bytes; with a `getent` stand-in, a 0770 private-group directory passes with the real listing and with exactly 100,000 entries, and fails when `getent passwd` exits 2, omits the user, lists the user with another group, lists another uid with the user's group, lists a name twice, prints a malformed line, more than 100,000 entries, or nothing, or when `getent group` fails | +| Totals | 252 tests on each runtime with Core required, none ignored; 224 pass and 28 are ignored without Core (the .NET 8 lane built with the 8.0.427 SDK through a temporary `global.json`, as CI pins it) | +| Regressions checked by mutation (.NET 8) | Each guard was removed in turn and the named tests run; all 42 mutations failed them: the command's exit group kill, its stop kill, `setsid` without forking, the pid-reuse check, the recorded-descendant kill, the stop check after the exit cleanup, and closing the output streams; the source kind check (the FIFO test failed after its one-minute wait), source count, source validation stop, source ceiling, tree entry and byte caps, the executable's size check and per-piece stop, the derived-binding check, the owner, mode, and owner-only checks, the cleanup's stop and cap, leaving the vacated staging path, and the final stop check; the encoder's node budget, the decoder's measurement, the text charge and length check, the list, record, and constructor counts, measuring before the type check, counting arguments before encoding, the batch stop checks before and after validation, the handler's stop check (offline and live), the `Dynamic` limit, excerpts, the message cap, wrapping a field reader's and a record constructor's exceptions, the lazy codec's cycle check (the test then never returned and was stopped), and the store's exclusive temporary file. Separately, checking the cancellation in an exception filter, which .NET runs before the preparation's cleanup, failed the failure-cleanup test until the check moved into the handler | +| Regressions checked against the code before each fix (.NET 8) | Each fix was undone in turn and its tests run; all 16 undoings failed them: `Invocation.cs`, `Builtins.cs`, and `Function.cs` as they were before the fixes (the old save left the existing file at 0666; the old lazy codec never ran its factory again; the old batch count made a batch of size -1, reported as an `OverflowException` from the codec), `Dynamic.cs` without the pair set (the equality test timed out after its one-minute wait), and guards turned off: the cache root recheck, the executable and prelude recheck, the identity in the version key, the record's entry cap, byte cap, and per-file read cap, the stop check in a record and in the exit kills, the name limit, the `ulong` refusal, `access(2)` (back to any execute bit), and reading the bindings outside the lock | +| Regressions checked against the code before each fix, continued (.NET 8) | Each test below fails on the code without its fix (the file as it was before the fix, with only the test seams declared, or the guard turned off): the 1,025-byte cap test (the second read takes 1,024 bytes, not 1); the SIGSTOP-order test (the last record reads all 40 entries with no signal); the cleanup-budget test with the last record's, the kills', or the exit wait's budget turned off in turn (300 record reads, not 125; 300 kill reads, not 125; the exit wait not ending 750 ms after the stop); the finished-exit-kills test (0 kills after the stop, not 8); the parent test (a 0777 parent accepted) and the next-call root test (a call through a replaced root runs a worker); the owner-only levels test (a level made with the umask, 0775 here, refused as a group-writable parent); the private-group test (a 0770 parent of the user's private group refused), the other-group test with the member check turned off (a group listing another member accepted), and the access-ACL test with the ACL check turned off (accepted); the every-level test with the walk stopped at the first unsafe level (one named, not two); the nothing-made test with the early check turned off (the missing level `x` made before the refusal); the save-directory test (a 0777 directory accepted) and the replaced-temporary test with the identity check turned off (the replacement renamed over the target); the pair-set test (2 sets for unshared values); and the wrapping test (the `InvalidOperationException` escapes). The private-group primary-account tests for a cache parent and a save directory (a 0770 directory accepted, with "attacker" holding its group as primary group), the buffer test (a name outside the buffer read), the handle-status test (the temporary file left behind), and the scratch test (the scratch directory made under the 0775 `TMPDIR`) each fail on the code without the scan, the range checks, the cleanup, or the scratch placement (with only the seams and entry points declared, keeping that code's behavior); with the member-slot or the pointer range check turned off, the buffer test fails; with the password database not consulted, both primary-account tests fail; and with the old scratch placement and a 0775 `TMPDIR`, six other tests fail besides the scratch test, five of them expecting exact refusals. The earlier two-pass `getpwent_r` scan accepted an account hidden by another enumeration in the process: run against the real glibc database, a test-side `getpwent` consuming `sync`, `_apt`, `dnsmasq`, and `sshd` just before the scan reached them, in both passes, made the passes agree and the scan accept `nobody` as the only account with primary group 65534. The `getent` listing shares no position with the process, so that schedule cannot arise; the in-process-enumeration test above shows `sync` still refused. Against the code before the shared verdict, the at-most-once test fails (two listings, not one); against the code that listed again for another account, user id, or group, the account-change test fails (the preparation accepted); with the `setsid` check removed and the test's file cases skipped, the `setsid` test fails at its group-writable directory case, which passed when the seam altered that directory for `getent`'s check as well; against the code before the `setsid` check, the `setsid` test fails (a `setsid` another user owns accepted); the earlier form of the machine listing test, which required this user to be accepted, fails when the listing adds another account with the user's primary group, and the current form passes; with the other-uid guard turned off, the current form fails on that listing, and with the verdict inverted, on the real one; and the CI step's check, before it captured the listing to a file, accepted a listing whose last line lacked a newline and one of 18,208,113 bytes. Against the code before the listing, the UTF-8 test fails (a name with byte 0xFF read as `\uFFFD` text), and the CI step's check, run locally with a `getent` stand-in, accepted the private group when `getent passwd` exited 2, omitted the user, listed the user with another group, listed a name twice, printed a malformed line or more than 100,000 entries, or printed nothing. With each guard of the listing turned off in turn (the exit status, the output cap, strict UTF-8, the repeated name, the field count, the entry cap, the final newline, the user-listed check, another uid with the group, the user's own ids, strict and unpadded ids, the token passed to the runner, the cancellation mapping, the timeout refusal, the file's and directories' root-only checks, the regular-file check, the root-only rule, the preparation's token, and the strict name decoding), a listing test fails at that case. Not tested: a parent owned by another user, which an unprivileged test cannot make | +| Build | `TreatWarningsAsErrors`, nullable reference types, and the `8.0-recommended` analyzers are clean with both SDKs | +| Unchanged lanes | The adapter's 96 offline and 5 live tests (the README-example check now covers all four examples), the 21-recording kit replay, `scripts/core_pin.py check`, `tests/test_core_pin.py`, and `scripts/check_distribution.py` (the source archive now carries the SDK and the example command) pass | + ## Outcome frame node budget (2026-10-25) Core's `outcome` encodes the evidence's interface types and the result value @@ -232,7 +254,7 @@ reaches persistence has request ordinal 1 and one accepted `ok` observation. | Linux, Go 1.24 and stable | CI lanes | Offline Go tests with the race detector on both; live Go tests in the shared pinned-Core job | | Linux, Rust 1.88 and stable | CI lanes | fmt, clippy, and offline Rust tests on both; live Rust tests in the shared pinned-Core job | | Linux, Java 21 and 25 | CI lanes | Offline Java adapter and SDK tests on both; live Java tests and the four Java projects on Java 21 in the shared pinned-Core job; offline and live suites also run locally on Temurin 21.0.13 and 25.0.5 | -| Linux, .NET 8 and 10 | CI lanes | Offline .NET tests on both; live .NET tests on both in the shared pinned-Core job; offline and live suites also run locally on .NET 8.0.33 and 10.0.14 | +| Linux, .NET 8 and 10 | CI lanes | Offline .NET adapter and SDK tests on both; live .NET tests on both and the four .NET projects in the shared pinned-Core job; offline and live suites also run locally on .NET 8.0.33 and 10.0.14 | | Linux, Node.js 26.7 | exercised | TypeScript offline/live suites and tarball installation exercised locally; no broader runtime or OS claim | | macOS | development only | no CI; the descriptor-count assertions are skipped without `/proc` (the socket host tests skip, the kit and adapter tests skip only the count); SQLite WAL mode is recorded, not assumed | | Windows and others | no claim | see LIMITS.md | diff --git a/docs/LIMITS.md b/docs/LIMITS.md index 469a02f..2b01917 100644 --- a/docs/LIMITS.md +++ b/docs/LIMITS.md @@ -29,7 +29,14 @@ also updates this document. | .NET worker process tree, not group | .NET cannot start the worker in its own process group, so the .NET host kills a process tree. At expiry, cancellation, or disposal of a running worker, it calls `Process.Kill(true)`, which stops the worker, lists its children from the process table, kills it, and does the same for each child. A descendant that was already reparented keeps running: one whose parent exited, or one that daemonized. So does one started after its parent was listed. Descendants are found and signalled by pid, with nothing pinning their identity: one that exits, and is reaped once the worker is killed, between the listing and the signal can have its pid reused by an unrelated process, which is then signalled (the Java host signals process handles, and the Rust and Go hosts a process group). Listing the process table takes time proportional to the number of processes on the machine. The runtime reaps the worker as soon as it exits. A descendant still holding stdout or stderr after the exit is detected and, after a one-second grace, reported as a carrier loss, as the Go and Rust hosts report it; it is not stopped. .NET reports a signal as exit status 128 + its number, so only the host's own SIGKILL is reported as a signal. | An OS-level containment mechanism (for example a cgroup or job object) with its own evidence, or a .NET process-group API. | | .NET pipe I/O goes through `poll(2)` | .NET's pipe streams have no read or write that a deadline or a cancellation can interrupt, so the .NET host never calls one that can block: the caller's thread asks the C library's `poll(2)` whether a pipe is ready, with a poll timeout of at most 5 ms, rechecking the deadline, the `CancellationToken`, and disposal between polls, and then reads only bytes already in the pipe or writes at most `PIPE_BUF` bytes (4096 on Linux, 512 on macOS), which a writable pipe accepts at once. No thread is left blocked on a worker pipe, even while an escaped descendant holds stdin without reading or floods stdout or stderr. The `pollfd` layout and event bits are the same on Linux and macOS; elsewhere, including Windows, the host does not start a worker and reports a spawn failure. .NET opens its own descriptors close-on-exec, but passes the worker any descriptor the host process itself inherited without that flag. | A .NET API for interruptible or non-blocking process pipes, and CI on each added platform. | | .NET handler cancellation is cooperative | Each .NET handler runs on its own background thread. At the deadline or on cancellation the host kills the worker's process tree, closes its input, and sends no late response. `HandlerContext.IsCancelled` turns true (it reads the deadline exactly), and stays true once the invocation ends. `HandlerContext.CancellationToken` is cancelled when the call stops or ends, never earlier (a lifetime of any length does not cancel it before its deadline). While a handler runs, the caller's thread rechecks the deadline and the caller's token between poll waits of at most 5 ms; that is the requested poll timeout, not a bound on the interval between checks, which reading pending stderr (up to 1 MiB per pass) and thread scheduling can lengthen. Whichever check sees the stop, the token's cancellation starts before the worker is killed; otherwise it starts shortly after the invocation ends. The cancellation runs on a short-lived thread the host starts for it, one per invocation that gave a handler a token, so it does not wait for the shared thread pool. If no thread can be created or started, the host queues it as a pool work item, which runs only when a pool thread is free, so a starved pool delays it. If the work item cannot be queued either, the host gives up: the token is never cancelled (`HandlerContext.IsCancelled` still turns true), `InvocationResult.HandlerTokenNotCancelled` is set, and a handler waiting only on the token keeps waiting; `Invoke` still returns its result and disposes the worker. Callbacks a handler registers on that token run on that thread, which the host never waits for; a callback that blocks holds that thread, and anything a callback throws is discarded. A callback registered after the token is cancelled runs at once on the handler's own thread and throws there, as with any `CancellationToken`. A handler that ignores both keeps running on its thread until it returns. No outside action is stopped or rolled back. A stack overflow in a handler ends the process, as it does any .NET code. | A separately enforced operation lifetime with explicit completion and reconciliation evidence. | -| .NET adapter is protocol-level only | .NET callers pass exact identities, boundary types, and protocol values to a prepared store; there is no .NET SDK for program preparation, native type mapping, or caching, and `Invoke` is synchronous. Linux runs in CI on .NET 8 and 10. macOS has no CI; the offline descendant tests need `sh`, `sleep`, and `yes`, and the descriptor tests need `/proc`. Windows is not supported. | A .NET SDK with typed-binding and preparation evidence comparable to the other SDKs, and CI on each added platform. | +| .NET SDK Core command descendants | Each Core `--version`, `check`, `hash`, and `run` command must exit within 30 seconds and keeps at most 8 MiB of stdout and 4 KiB of stderr; cancelling the token stops it. Its output is read only after `poll(2)` reports it ready, on the caller's thread. On Linux with a `setsid` at `/usr/bin/setsid` or `/bin/setsid` (util-linux's, or a compatible one such as BusyBox's; any executable there is used), the command runs as `setsid -- `, which execs without forking, so Core keeps its pid and leads its own process group; when `setsid` cannot exec Core (exit status 126 or 127 with its own `setsid: ` message) the command is reported as a start failure. When the command exits, and on a timeout or cancellation, the host sends `kill(-pgid, SIGKILL)` to that group. Once .NET has reaped Core, a live process holding Core's pid means the id was reused and nothing is killed; a process that took Core's pid after a full pid wraparound, made its own group, and forked between that check and the kill could still have that group killed, the same window as the Rust SDK's `kill(-pgid)`. A descendant that leaves the group (calls `setsid`) is neither detected nor killed; the host does not wait for the end of the output after the exit, and closes its ends, so that descendant's next write fails. With `setsid`, the wait for a killed Core to exit after a stop ends 750 ms after the stop. Without `setsid` (Linux without one at those paths), descendants are recorded from `/proc` about every 10 ms by pid and start time, and every recorded descendant still alive with that start time is killed when the command returns (the start time is read just before the kill, a short window in which a reused pid could be signalled, as with the JDK's process handles). Each record reads at most 16,384 `/proc` entries and 4 MiB of `stat` files in all, each read limited to 1 KiB and to the bytes left, at most 16,384 descendants are kept, and a record ends at the deadline or a cancellation, checked before each entry. The kills when Core exits by itself run to the end; a stop that lands during them lets them go on for at most 250 ms more. On a timeout or cancellation, Core is first sent `SIGSTOP` (only while its start time still matches the one read when it started), so it starts no more children and its children stay its children; then a last record is made, within the same caps, until 250 ms after the stop; then Core is killed, and every recorded descendant until 500 ms after the stop; then the host waits for Core to exit until 750 ms after the stop. Each budget is checked before each entry or descendant, so the cleanup returns within 750 ms of the stop plus one `stat` read and one signal. Only Core is frozen: its descendants run until they are killed, so one whose parent exits before the last record reaches it is reparented and missed. Descendants past the caps or the budgets, and fast fork-and-exit chains between records, are missed too. .NET's `Process.Kill(true)` is not used on Linux. macOS has no `/proc`, so there only `Process.Kill(true)` on a stop applies. The .NET worker does not run in its own group (see the worker row above). | An OS-level containment mechanism, or a .NET process-group API, with its own evidence. | +| .NET SDK cache and source reads | A cache entry is verified against the file digests in its `program.json`, and every binding is derived again from Core's cached raw output, so an accidental or partial change is refused. No record inside the cache resists someone who can rewrite it: an output, its recorded digest, and the bindings changed together load. So the cache's owner is trusted and only the owner may write it: preparation refuses a cache root, an entry, or any file or directory in one that the current user does not own or that its group or others may write (read with `statx(2)` on Linux and `stat(2)` on macOS, without following links), on create and on load, and new entries are owner-only. Every symbolic link in the cache root's path is resolved once (`realpath(3)`), and the real directory is checked and used from then on; nothing inside it is followed. Every directory above the root, up to `/`, must be owned by the current user or root and must not be writable by others, nor by its group, unless it has the sticky bit (as OpenSSH checks the directories above its configuration) or the group is the user's private group by Debian OpenSSH's rule (no account but the user has the group as its primary group, and the group lists no member but the user) with stricter conditions: the directory's group is the user's primary group, the group's name is the user's name, and the group lists no members (`gr_mem`) or only the user (`getpwuid_r(3)` and `getgrgid_r(3)`, each within 64 KiB; an entry that needs more, or a pointer libc returns outside the buffer it was given, counts as not private); on Linux the password database, as `getent passwd` lists it (`gr_mem` omits accounts whose primary group it is), lists the user's account with its user id and that primary group and no account of another user id with that primary group; and on Linux the directory has no access ACL (`lgetxattr(2)`). Elsewhere a group-writable directory is refused. `getent` runs as a separate process, so it shares no enumeration position (`getpwent`'s is process-global) with code in this process. It is the first of `/usr/bin/getent` and `/bin/getent` present, and it and every directory above its real path must be owned by root and not writable by group or others. It runs through the runner Core's commands use, in its own process group where `setsid` is installed, and then that `setsid` (the first of `/usr/bin/setsid` and `/bin/setsid` present) must be owned by root and not writable by group or others too, as must every directory above its real path, or the level is refused; each runs by its checked real path. The listing is limited to 5 seconds, after which it is stopped within the runner's 750 ms stop budget, with its stdout capped at 16 MiB (so the names kept are too) and its stderr at 4 KiB. It runs at most once per `Runtime.Prepare` call (whose two checks, of the existing directories before a missing level is made and of the cache root's ancestors, share its verdict, and a check whose account, user id, or group differs from the first one's counts as not private without listing again) and at most once per `Invocation.Save` call, so each spends at most about 5.75 seconds in it; its verdict is not kept after the call. Its output is parsed strictly: no such `getent` (or not Linux), a timeout, a nonzero exit status, stdout over the cap, output that is not valid UTF-8 or does not end with a newline, more than 100,000 entries, a line that is not seven colon-separated fields with a nonempty name and decimal user and group ids, a name listed twice, or the user's account missing or listed with other ids counts as not private. A name listed by two sources (for example an account in both `/etc/passwd` and an enumerating LDAP or sssd source) is a name listed twice, so on such a system every group-writable level is refused. Like Debian OpenSSH, the rule sees only accounts the password database enumerates: an account from a name service that does not enumerate (LDAP or sssd with enumeration off) whose primary group is the user's private group is not seen, so a directory it can write is accepted. Likewise a source that fails partway through its listing can end `getent`'s output early: if `getent` then exits nonzero the level is refused, and if it does not, the accounts that source did not list are not seen. The user and group lookups (`getpwuid_r`, `getgrgid_r`) and the ACL read run on the caller's thread and are not stopped by the cancellation token or the deadline; with a network name service (LDAP, sssd, NIS) they can block for its timeout. The password database listing can block a preparation or a save for at most its 5 seconds plus the 750 ms stop budget, once per call, and is stopped by the preparation's cancellation token (the preparation then fails as cancelled); `Invocation.Save` takes no token, so its listing is bounded by those limits alone. These lookups run only when a directory above the cache or a save target is writable by its group. Otherwise preparation is refused with a message naming every such directory, nearest first (at most 16, the rest counted). So no other user, apart from the accounts the rule cannot see (above), can rename the root away and put another directory at its path. The existing directories are checked before a missing level is made, so a refused preparation creates nothing, and each level it makes is owner-only, whatever the `umask`. The check reads mode bits only: an ACL that grants another user write without showing in the mode bits is not seen (macOS ACLs, `chmod +a`; on Linux a POSIX ACL's mask shows in the group bits, so such a directory is refused, and one with an access ACL never passes as the private group's), and on WSL drive mounts (`/mnt/c`, DrvFs without the `metadata` option) every directory shows as 0777 and `chmod go-w` has no effect, so the cache must live in the Linux file system there. The root is used by its path, also by every worker a prepared program starts (on `//store`): its device and inode, read at the owner check, are compared again before the staging directory is made, before publishing, before and after each load, and before each worker starts, and a changed root is refused (a call then fails with a `Preparation` exception and starts no worker). Not covered: replacement by the same user or by root, whom the owner check trusts; a root renamed away and back between two comparisons, which keeps its inode (once the parents are checked, only its owner or root can rename it); and a directory above the root made writable by others after preparation, which the next preparation refuses but a prepared program does not notice (its comparisons still refuse a replaced root, except in the few system calls between the last one and a worker's start). The Core executable and every prelude file and directory are fingerprinted by path, and Core opens them by path later: each one's identity (device, inode, size, mode, owner, and modification and status-change times) is read as it is fingerprinted and compared after Core's commands finish (inside the version check and prelude discovery, before a new entry is published, and before an existing one is loaded), and any change refuses the preparation. Not detected: a file swapped and then restored with the same inode and times, which an unprivileged user can do only within the file system's timestamp granularity, since writing or renaming a file updates its status-change time (on Linux from a coarse clock, a few milliseconds); files Core loads besides the executable and prelude (its shared libraries and loader); and any change after the last comparison, since a prepared program's workers later run the executable and read the prelude (`JACQUARD_PRELUDE`) by path without checking them again. The macOS status calls are compiled but not exercised by any test. A failed preparation's cleanup removes at most 100,000 entries of its staging directory, created exclusively, and ends at a cancellation, so a stopped preparation leaves an unpublished `prepare-*` directory in the cache root; it is never loaded, and nothing removes it later (a name and an age cannot show that the SDK made a directory or that no preparation is using it), so remove leftovers yourself when no preparation is running. At most 1,024 sources are taken, refused from the list's count before any element is read, and they must be regular files totalling at most 16 MiB, the Core executable at most 512 MiB, and the prelude and cache walks read at most 100,000 entries and 256 MiB of regular files under one root, each file checked from its size before it is read and again while it is read; a source or cache file replaced by a FIFO (or, in the cache, by a link) between that check and its open can still block or redirect the open, which a cancellation does not end. | An open that cannot block on a FIFO or follow a link; directory-relative (`openat`) access pinned to the checked root; Core reading descriptors the host opened and verified. | +| .NET SDK cancellation is by token only | Preparation, Core's commands, `Call`, `Invoke`, and `MapBatches` observe the `CancellationToken` they take. `Thread.Interrupt` is not a stop signal: the SDK's own waits are `poll(2)` sleeps, which neither end at an interrupt nor consume it. The API is synchronous; there are no `Task`-returning variants. | `Task`-returning variants with the same bounds, if callers need them. | +| .NET SDK signature depth | A checked function whose signature nests types deeper than 64 levels (`max_json_depth`, the protocol's value-depth limit), names more than 4,096 types, or has more than 16,384 tokens is left unbound, like a polymorphic one: preparation succeeds, the other functions bind, and binding it fails with a `Binding` exception naming the limit. The SDK parses no operation signatures (Core's `check --print-sigs` prints none). | None needed for depth, since a deeper value cannot cross the protocol; Core's structured host interface exporter for the rest. | +| .NET SDK record fields are declared, not exported | A record codec's fields, in the order given, are a constructor's fields; the pinned CLI exports no field types, so a wrong order is not detected locally, and Core refuses it only where the swapped fields' types differ. | Core's structured host interface exporter, so bindings can be generated or checked. | +| .NET SDK saved inspections | `Invocation.Save` resolves the target's directory once and refuses it unless it and every directory above it pass the cache's directory rule (owned by the user or root; not writable by others, nor by its group unless sticky or the user's private group), since another user who could write it could replace the file before or after the rename. It writes the record through one handle to a new file it created exclusively (`O_CREAT|O_EXCL`, so no link is followed), owner-only and flushed to disk, checks that the file at the temporary path has that handle's device and inode, and only then renames it over the target; a replacement found there is refused and left in place, and nothing is saved. If the status of the new file's handle cannot be read, the file is removed by its path while that is still an empty regular file the user owns, as the one just created is, and the error is raised. Not covered: a replacement by the same user or by root between that check and the rename (no other user can, given the directory rule); the directory itself is not synced, so the rename may not survive a power loss; and the mode-bit limits of the cache's rule (ACLs not shown in the mode bits, WSL drive mounts). | A rename relative to a directory descriptor pinned to the checked directory (`renameat(2)`, `RENAME_NOREPLACE` where the target is new), and an `fsync(2)` of that directory. | +| .NET fulfillment is a single-process snapshot example | Atomic JSON replacement demonstrates booking/outbox recovery and deduplication, without cross-process locking or power-loss durability evidence. | A concrete production storage contract and crash/concurrent-writer evidence. | +| .NET platforms | Linux runs in CI on .NET 8 and 10. macOS has no CI; the offline descendant tests need `sh`, `sleep`, and `yes`, and the descriptor, Core command, and preparation tests need `/proc` and run on Linux only. Windows is not supported: the worker reports a spawn failure and `Prepare` refuses. | CI, lifecycle, and cleanup evidence on each added platform. | | Node handler cancellation is cooperative | The Node adapter can expire while awaiting a Promise, signal the handler, and kill the worker; it cannot stop an outside action or preempt JavaScript that blocks the event loop. No late response or automatic retry follows expiration. | A separately enforced operation lifetime with explicit completion and reconciliation evidence. | | Socket host closes without a response on its own failures | A ceiling violation, early disconnect, or failed invocation ends the connection silently; clients see a reset, not an HTTP error. | An owner decision that the host may emit fixed minimal status lines, recorded here with the exact bytes and the rule that they carry no application meaning. | | Request bytes cross as Latin-1 `Text` | v0 has no bytes value; each byte is one codepoint 0..255 and a response codepoint above 255 is refused by the host. | A frozen bytes value in a later protocol version. | @@ -41,7 +48,7 @@ also updates this document. | No outbound network in the first application | The synthetic trial cannot call APIs, analytics, or email services. | A concrete application need and reviewed host/port authority with refusal evidence. | | Host-owned SQLite through one named operation | Jacquard receives neither raw SQL nor generic filesystem authority; `study.save-response` is given a synthetic session token by application contract (the host cannot verify what a token denotes), and its receipt is a content hash with no timestamp. | Repeated concrete operations justify a reusable storage contract with migration, recovery, and authority evidence. | | No automatic side-effect retry | An ambiguous completion fails rather than risking duplicate external work. | Operation-specific idempotency keys, authenticated receipts, and crash/recovery evidence. | -| SDK uses explicit application data mappings | The pinned CLI does not export structured constructor layouts. Python dataclasses, TypeScript record codecs, Go records, and Java record components must follow Jacquard positional field order; ambiguous names across declaration groups are refused. Core validates wire types. | Core's versioned host-format interface exporter and independent typed-binding conformance. | +| SDK uses explicit application data mappings | The pinned CLI does not export structured constructor layouts. Python dataclasses, TypeScript record codecs, Go records, Java record components, and .NET record codecs' fields must follow Jacquard positional field order; ambiguous names across declaration groups are refused. Core validates wire types. | Core's versioned host-format interface exporter and independent typed-binding conformance. | | Go fulfillment is a single-process snapshot example | Atomic JSON replacement demonstrates booking/outbox recovery and deduplication, without cross-process locking or power-loss durability evidence. | A concrete production storage contract and crash/concurrent-writer evidence. | | JavaScript numbers have a smaller exact integer range | Native safe integers and the TypeScript `int` codec use numbers; full 63-bit Int values require bigint (or the explicit bigint codec). Unsafe numeric input is refused. | An explicit application choice of bigint; no silent precision loss is allowed. | | TypeScript fulfillment is a single-process snapshot example | Atomic JSON replacement demonstrates booking/outbox recovery and deduplication, without cross-process locking or power-loss durability evidence. It is not the Python SQLite implementation. | A concrete production storage contract and crash/concurrent-writer evidence. | diff --git a/dotnet/.editorconfig b/dotnet/.editorconfig index f9fd46f..b08abcc 100644 --- a/dotnet/.editorconfig +++ b/dotnet/.editorconfig @@ -6,3 +6,7 @@ dotnet_diagnostic.CA1036.severity = none # JsonObject is named for JSON's object, not as a collection. dotnet_diagnostic.CA1710.severity = none +# The SDK's types are named for Jacquard's concepts, as in the other SDKs: +# Function, Option, and Int are Jacquard names, not .NET keywords or types. +dotnet_diagnostic.CA1716.severity = none +dotnet_diagnostic.CA1720.severity = none diff --git a/dotnet/JacquardHost.sln b/dotnet/JacquardHost.sln index bc797ee..67350d1 100644 --- a/dotnet/JacquardHost.sln +++ b/dotnet/JacquardHost.sln @@ -13,6 +13,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JacquardHost.StandIn", "tes EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JacquardHost.Tests", "test\JacquardHost.Tests\JacquardHost.Tests.csproj", "{3ED29B8E-500E-4D0D-B6AD-F987037225AD}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "examples", "examples", "{B36A84DF-456D-A817-6EDD-3EC3E7F6E11F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JacquardExamples", "examples\JacquardExamples\JacquardExamples.csproj", "{79FD70B9-B8BC-4217-9720-9553BA48F2BD}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -59,6 +63,18 @@ Global {3ED29B8E-500E-4D0D-B6AD-F987037225AD}.Release|x64.Build.0 = Release|Any CPU {3ED29B8E-500E-4D0D-B6AD-F987037225AD}.Release|x86.ActiveCfg = Release|Any CPU {3ED29B8E-500E-4D0D-B6AD-F987037225AD}.Release|x86.Build.0 = Release|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Debug|x64.ActiveCfg = Debug|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Debug|x64.Build.0 = Debug|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Debug|x86.ActiveCfg = Debug|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Debug|x86.Build.0 = Debug|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Release|Any CPU.Build.0 = Release|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Release|x64.ActiveCfg = Release|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Release|x64.Build.0 = Release|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Release|x86.ActiveCfg = Release|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -67,5 +83,6 @@ Global {3C269BF6-7938-464F-B385-6997B38B63B9} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {CB695A56-2929-47FB-B93C-3E74883663E2} = {0C88DD14-F956-CE84-757C-A364CCF449FC} {3ED29B8E-500E-4D0D-B6AD-F987037225AD} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {79FD70B9-B8BC-4217-9720-9553BA48F2BD} = {B36A84DF-456D-A817-6EDD-3EC3E7F6E11F} EndGlobalSection EndGlobal diff --git a/dotnet/README.md b/dotnet/README.md index 6edbe11..150106b 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -1,32 +1,590 @@ # Jacquard Host for .NET -An independent .NET adapter for Jacquard Core's experimental `jacquard-host-v0` -protocol. It implements the pinned protocol on its own, without calling -Python, Node.js, Go, Rust, or Java, and has no runtime dependencies beyond -the .NET base class library. It targets .NET 8 and runs on .NET 8 and 10. It -runs workers on Linux and macOS: the carrier calls the C library's `poll(2)` -(see "Process carrier" below). - -This is the protocol-level adapter, comparable to the Go `host` package, the -Rust `host` module, and the Java adapter. It includes strict codecs, the host -session state machine, and a bounded process carrier for `jacquard host -worker`. It also provides a `Host` with a closed operation registry and a -replayer for Core's pinned conformance kit. There is no SDK layer yet. It does -not compile programs, map native types, or cache stores. You prepare a store -with the Core CLI and pass exact identities and boundary types. - -**Experimental:** the protocol is Core's pinned `jacquard-host-v0`, and Core is -installed separately. Each invocation runs one fresh, serial worker. The host -and operating system are trusted. Linux is exercised; macOS should work but has -no CI; Windows is not supported (the worker reports a spawn failure). See +Call checked Jacquard functions from .NET, bind .NET handlers to their +declared operations, and inspect the exact protocol evidence. This is an +independent .NET implementation of the pinned `jacquard-host-v0` boundary. It +has no runtime dependencies beyond the .NET base class library and runs Core +directly, without Python, Node.js, Go, Rust, or Java. It targets .NET 8 and +runs on .NET 8 and 10. + +The assembly has two layers: +- **The SDK** (`JacquardLang.Host.Sdk`) prepares and caches checked programs, + maps .NET values through explicit codecs, and binds handlers to exact + operation identities. It is comparable to the Python, TypeScript, Go, Rust, + and Java SDKs. +- **The protocol-level host** (`JacquardLang.Host`) takes exact identities, + boundary types, and protocol values for a store you prepare with the Core + CLI. It is comparable to the Go `host` package, the Rust `host` module, and + the Java adapter. + +Linux is exercised. macOS should work but has no CI: the carrier and the SDK +call the C library (`poll(2)`, and for the SDK `kill(2)`, `geteuid(2)`, +`mkdir(2)`, `stat(2)`, `getpwuid_r(3)`, `getgrgid_r(3)`, and on Linux +`lgetxattr(2)` and the `getent passwd` command; see [process carrier](#process-carrier) and +[Core's commands](#cores-commands)), and the macOS status calls are compiled +but not run by any test. Windows is not supported: the worker reports a spawn +failure and `Prepare` refuses. **Experimental:** the protocol is Core's pinned +`jacquard-host-v0`, and Core is installed separately. Each invocation runs one +fresh, serial worker. The host and operating system are trusted. See [limits](../docs/LIMITS.md). -## Use +## Install The package is not published to a registry. Build it from a clone (`dotnet pack src/JacquardHost -c Release` in this directory) and reference -the `JacquardLang.Host` package or the project. The example below invokes a -checked function whose single operation is served by a .NET handler: +the `JacquardLang.Host` package or the project. + +From a clone, install the pinned Core release (see [setup](../docs/SETUP.md)), +then run the quickstart: + +```sh +eval "$(python3 scripts/core_pin.py install)" +cd dotnet +dotnet build examples/JacquardExamples +dotnet examples/JacquardExamples/bin/Debug/net8.0/JacquardExamples.dll quickstart +``` + +Expected output: + +```text +Order total 35: shipping 5 +Order total 75: shipping 0 +``` + +The command keeps its program cache under `.scratch/dotnet-examples/` in the +working directory, and the SDK refuses a cache when another user could write +any directory above it, up to `/`. A clone made with a `umask` of 002 passes +when its directories belong to your private group (a group named after you +that lists no other member and that no other account in the password +database has as its primary group, as Debian, Ubuntu, and Fedora create for +each user). +If preparation is refused, the message names every such directory: run +`chmod go-w` on each, or pass `--workdir` under directories only you or root +may write (see [call a checked function](#call-a-checked-function)). + +The same command runs the other example projects; see +[run the projects](#run-the-projects). + +## Call a checked function + +```csharp +using JacquardLang.Host.Sdk; + +public static class Shipping +{ + public static long Cost(string rules, string cacheDir, string inspection) + { + // Core is $JACQUARD_BIN and $JACQUARD_PRELUDE unless RuntimeOptions say otherwise. + var runtime = new Runtime(); + PreparedProgram program = runtime.Prepare(new[] { rules }, new PrepareOptions { CacheDir = cacheDir }); + Function shipping = program.Function("shipping-cost"); + + // A long converts to an Int argument; Call decodes the result. + long free = shipping.Call(Codecs.Int, 75L); + + // Invoke keeps the raw frames and evidence, saved only on request. + Invocation invocation = shipping.Invoke(35L); + invocation.Save(inspection); + return invocation.Decode(Codecs.Int) + free; + } +} +``` + +The shipping rule is the [quickstart source](../examples/quickstart/rules.jac). +`Prepare` takes `.jac` paths (or `.jqd` paths) in dependency order. It uses +Core's public `check`, `hash`, and `run` commands, and keeps a verified cache +keyed by: +- the source bytes; +- the Core executable bytes; +- every prelude file; +- the protocol and preparation format. + +At most 1,024 sources are taken (`Runtime.SourceFiles`); a longer list is +refused from its `Count` before any element is read, and the token is +observed before each element is checked. A path that is empty, holds a NUL, +or is longer than 4,096 chars as given or made absolute (a source, the cache +directory, the prelude, the executable, or a save path) is refused with an +excerpt before the file system sees it. Each source must be a regular file (a +symbolic link to one is followed), and all of them together at most 16 MiB +(`Runtime.SourceLimit`); both are checked from the files' status before a +source is read, so a FIFO or device is refused rather than opened, and no read +goes past the ceiling. The Core executable is fingerprinted in 64 KiB pieces +up to 512 MiB: a larger file is refused from its size before it is read, the +ceiling is enforced again while it is read, and the token is observed before +each piece. The prelude and cache walks stream each directory on an explicit +stack (no directory depth can exhaust the call stack) and read regular files +only (no symbolic links, FIFOs, or devices), at most 100,000 entries (empty +files and directories count) and 256 MiB under one root; each file's size is +checked before it is read and the ceiling again while it is read, and the +token is observed before each entry and after each piece. A source replaced by a +FIFO between that check and its open can still block the open. + +A preparation stages its entry in a `prepare-*` directory of the cache root, +created exclusively (`mkdir(2)`), so it never adopts a directory it did not +make. One that fails removes that directory in a streamed walk of at most +100,000 entries that ends at the first cancellation, so a stopped preparation +leaves its staging directory behind. Such a directory is never loaded: +entries are named by their identity digest. Nothing removes it later, because +a name and an age cannot show that the SDK made a directory or that no +preparation is using it; remove leftover `prepare-*` directories yourself when +no preparation is running. Once an entry is published, its vacated staging +path is not touched again. A cancellation that arrives as a preparation +finishes, a warm load included, is reported as the stop, not as a program. + +A function whose checked signature nests types deeper than 64 levels (the +protocol's value-depth limit, `max_json_depth`, so no such value could be +passed), names more than 4,096 types, or has more than 16,384 tokens is +refused before the parser recurses past that bound. Like a polymorphic +function it is left unbound: preparation succeeds, the other functions bind, +and `PreparedProgram.Function` on it fails with a `Binding` exception naming +the limit. `check --print-sigs` prints no operation signatures, and the SDK +parses none; an operation's arguments are checked by the protocol's limits +when they arrive. + +The cache defaults to `.scratch/jacquard-host-dotnet` under the working +directory; `PrepareOptions.CacheDir` changes it. Every symbolic link in the +cache root's path is resolved once (`realpath(3)`), and the real directory is +checked and used; nothing inside it is followed. A cache entry keeps Core's raw +`check --print-sigs` output and the `hash` output for the source and each +prelude file beside the store, and `program.json` records every file's digest. +Every binding (names, constructors, signatures, and declaration groups) is +derived from those outputs, on the first preparation and again on each load. +A load refuses an entry whose files do not match the recorded digests, or +whose `program.json` bindings differ from what the outputs give, rather than +repairing it. So an accidental or partial change is refused. No record inside +an entry can resist someone who can rewrite it (an output, its recorded +digest, and the bindings changed together load), so the cache's owner is +trusted and only the owner may write it: preparation refuses a cache root, an +entry, or any file or directory in one that the current user does not own or +that its group or others may write, both when it creates an entry and when it +loads one, and it makes each new entry's directories and files owner-only +(Core's store included). The owner and mode come from `statx(2)` on Linux and +`stat(2)` on macOS, without following links. Every directory above the +root, up to `/`, must be owned by you or root and must not be writable by +others, nor by its group, unless it has the sticky bit (as `/tmp` has), as +OpenSSH checks the directories above its configuration. A directory writable +by its group is accepted when that group is your private group, by Debian +OpenSSH's rule (no account but you has the group as its primary group, and +the group lists no member but you) with stricter conditions. All of these +must hold: the directory's group is your primary group; the group's name is +your user name; the group lists no members or only you (read with +`getpwuid_r(3)` and `getgrgid_r(3)`); on Linux, the password database, as +`getent passwd` lists it in a separate process (so no enumeration position +is shared with code in yours), lists your account with your user id and that +primary group and no account of another user id with it; and, on Linux, the +directory has no access ACL, whose mask is what its group bits show. +Elsewhere a group-writable directory is refused. `getent` must be +`/usr/bin/getent` or, if that is missing, `/bin/getent`, owned by root and not +writable by group or others, as must every directory above it, and so must +the `setsid` it runs through where `setsid` is installed. It runs like Core's +commands, limited to 5 seconds (then stopped within 750 ms), with at most +16 MiB of output, and the preparation's token stops it (`Invocation.Save` +takes no token, so there only those limits bound it). It runs at most once +per `Prepare` call and once per `Save` call, for the account, user id, and +group of the first check that needs it (a later check in the same call that +finds another refuses the directory without listing again), so each spends +at most about 5.75 seconds in it, and every such call lists the database +again. A timeout, a nonzero exit status, more output, output +that is not UTF-8 or not complete lines, more than 100,000 entries, a line +that is not seven colon-separated fields with decimal ids, a name listed twice +(also when two name services list one account), or your account missing or +listed otherwise refuses the directory. So a clone made with a `umask` of 002 +under user-private groups is accepted, and no other account the password +database lists can rename the root away and put another directory at its +path. An account that a name service does not enumerate (LDAP or sssd with +enumeration off), or does not list because it failed partway without +`getent` reporting it, cannot be seen; see [limits](../docs/LIMITS.md). +Otherwise preparation is refused +with a message naming every such directory, nearest first, with its reason +(at most 16 named, the rest counted): run `chmod go-w` on each, or choose a +cache under directories only you or root may write. The existing directories +are checked before any missing level is made, so a refused preparation +creates nothing, and the levels it makes are each owner-only, whatever the +`umask`. Since the listing runs only for a group-writable private-group +level, a caller that prepares per request avoids starting `setsid` and +`getent` on every `Prepare` (cache hits included) by making the cache's +parent directories writable only by their owner (`chmod go-w DIR`). The root is used by its path, also by every +worker a prepared program starts, so its device and inode, read at the owner +check, are compared again before the staging directory is made, before an +entry is published, before and after each load, and before each worker +starts; a changed root is refused (a call fails with a `Preparation` +exception and starts no worker). The owner check trusts you and root: a +replacement by the same user is not detected, nor is a root renamed away and +back between two comparisons (it keeps its inode). + +The executable and every prelude file and directory are fingerprinted by +path, and Core opens them by path afterwards. Each one's identity (device, +inode, size, mode, owner, and modification and status-change times) is read +as it is fingerprinted and must be unchanged once Core's commands have +finished: inside the version check and prelude discovery, before a new entry +is published, and before an existing one is loaded. A change refuses the +preparation, and a version check or prelude discovery it affected is not +kept. The `--version` check runs once per executable fingerprint (its digest +and its identity) and prelude discovery once per that fingerprint and the +prelude's digest, in the process, shared by every `Runtime` and thread. What +this does not cover is in [LIMITS](../docs/LIMITS.md). Each call runs one +fresh worker. Prepare only +first-party declaration sources: preparation runs Core's `run` command, +without service bindings, to populate the store. + +`new Runtime()` uses `$JACQUARD_BIN` and `$JACQUARD_PRELUDE`; an empty option +means unset. The Core executable must be a regular file this user may execute +(`access(2)` with `X_OK`, not merely any execute bit); one this user may not +run is refused with a message saying so. The runtime refuses a Core whose +`--version` differs from the pinned release. Each Core command must finish within 30 seconds, and keeps at +most 8 MiB of stdout and 4 KiB of stderr; see [Core's commands](#cores-commands). +Cancelling the token `Prepare` takes kills the running command and its +process group, and fails the preparation; a preparation waiting for another +thread's version check or prelude discovery stops waiting too. The SDK +observes only the token: `Thread.Interrupt` is not a stop signal, and the +SDK's own waits (`poll(2)` sleeps) do not consume it. + +Arguments are `Argument`s. A `long`, `int`, `uint`, `short`, `ushort`, +`sbyte`, or `byte` converts implicitly to an `Int`, a `double` to a `Real`, a +`string` to a `Text`, and a `bool`, a protocol `Hash`, a protocol `Value`, or +a `Dynamic` to itself; anything else, lists and records included, is +`codec.Arg(value)`. A `ulong`, `nuint`, `char`, `float`, `Half`, `decimal`, +`Int128`, `UInt128`, or `BigInteger` does not compile as an argument or in +`Dynamic.Of`: those conversions are marked obsolete as errors, with the +reason, because C# would otherwise convert a `ulong` through `double` (2^53 + 1 +would arrive as 2^53) and a `char` to an `Int` (see [data](#data)). +`Codecs.Real` is a codec of `double`, so a `long` given to it +(`Codecs.Real.Arg(total)`, or a record field reader returning a `long` for a +`Codecs.Real` field) is converted by C#'s implicit numeric conversion before +the SDK sees it: that is your explicit choice of a Real, as in Java, and it is +exact only up to 2^53. `Call` +decodes the result with the codec you pass; `Invoke` keeps the whole +`Invocation`. Overloads that take an argument list and a `CancellationToken` +kill the worker when it is cancelled. `MapBatches` reads the number of rows +once and refuses a list whose count differs before a later batch. +`Invocation.Save` writes the frames and evidence, including argument and +result values. The target's directory, resolved once, and every directory +above it must pass the cache's directory rule (owned by you or root; not +writable by others or by its group unless sticky or your private group), or +the save is refused with an `IOException` naming them: otherwise another +user could replace the file before or after the rename. The record is written +through one handle to a new file created exclusively (`O_CREAT|O_EXCL`, never +through a link) in that directory, owner-only (0600) and owned by you, and +flushed to disk (`fsync`); the file at its path is then checked to be the one +written (same device and inode), and only then renamed over the target. A +replacement is refused and left in place; nothing is saved. Whatever was at +the target is replaced, never written through, so an existing file's +permissions and owner are not kept and a symbolic link there is replaced +rather than followed. In a sticky directory such as `/tmp`, a file another +user owns cannot be replaced and the save fails. Not covered: a replacement +by you or root between that check and the rename, and the directory's own +entry, which is not synced, so the rename itself may not survive a power +loss. The API is synchronous; there are no `Task`-returning variants. + +## Data + +Values map through explicit `Codec` objects, built with the factories on +`Codecs`. A record is mapped by naming its constructor and giving, for each +field in the constructor's positional order, a codec and a function that +reads it, then a function that builds the record from the decoded fields. +Records are mapped without reflection (enumerations use `Enum.GetValues` and +`Enum.GetName`, and a type hierarchy checks its cases' types with `Type`), and +there is no attribute or source generator: a codec is an +ordinary immutable, thread-safe object, so the library builds clean under the +trimming and ahead-of-time analyzers (`IsAotCompatible`; a trimmed or +native-AOT publish is not tested), and a binding that does not follow a +convention (a constructor whose name differs from the type's, say) is one +explicit expression. Before any worker starts, the encoded arguments are +checked against the function's checked signature; Core always checks the +actual wire types too. + +| .NET | Jacquard | +|---|---| +| `long` (`Codecs.Int`) | `Int`, Core's full 63-bit range; a `long` outside it is refused. `int` (`Codecs.Int32`) refuses a result that does not fit | +| `double` (`Codecs.Real`) | `Real` as binary64 bits, including NaN payloads and signed zero; an integer at a `Real` parameter is refused | +| `string`, `bool`, protocol `Hash` | `Text` (a lone surrogate is refused), `Bool`, `Hash` | +| `IReadOnlyList` (`Codecs.List`) | `List T`; decoding returns a new array | +| `Option` (`Codecs.Option`) | `Option T`; `Option>` keeps nested options distinct. `Codecs.Nullable` and `Codecs.NullableValue` map `null` to `None` for one level | +| `Unit` (`Codecs.Unit`), value tuples (`Codecs.Tuple`, two to four items) | unit and tuples | +| a record (`Codecs.Record`, `Codecs.Field`) | one constructor, named by public name, its fields in the order given | +| an enumeration (`Codecs.Enum`) | nullary constructors by convention: member `PageNow` is the constructor `PageNow` (store spelling `page-now`), or an explicit map | +| a type hierarchy (`Codecs.Union`) | the constructors of one data type: each case is a record, enumeration, or union codec of a subtype | +| a recursive type (`Codecs.Lazy`) | a codec made once, thread-safely, on first use, so a field can refer to its own type's codec; a factory that throws keeps nothing and runs again on the next use | +| `Dynamic` (`Codecs.Dynamic`) | any of the above by public name, for code with no application type | +| protocol `Value` (`Codecs.Value`) | passed through unchanged | + +```csharp +using System.Collections.Generic; +using JacquardLang.Host.Sdk; + +// type Incident = | Incident(id: Text, service: Text, severity: Int, age: Int, visible: Bool, owner: Option Text) +// type Urgency = | PageNow | WorkToday | WatchOnly +// type Triage = | Routed Text Text Urgency (List Text) | InvalidIncident Text +public static class Incidents +{ + public sealed record Incident(string Id, string Service, long Severity, long Age, bool Visible, string? Owner); + + public enum Urgency + { + PageNow, + WorkToday, + WatchOnly, + } + + public abstract record Triage; + + public sealed record Routed(string Id, string Team, Urgency Urgency, IReadOnlyList Reasons) : Triage; + + public sealed record InvalidIncident(string Id) : Triage; + + // A record is one constructor: its fields in order, each with a codec and + // a reader, and a function that builds it from the decoded fields. + public static readonly RecordCodec IncidentCodec = Codecs.Record( + "Incident", + Codecs.Field("id", Codecs.Text, (Incident incident) => incident.Id), + Codecs.Field("service", Codecs.Text, (Incident incident) => incident.Service), + Codecs.Field("severity", Codecs.Int, (Incident incident) => incident.Severity), + Codecs.Field("age", Codecs.Int, (Incident incident) => incident.Age), + Codecs.Field("visible", Codecs.Bool, (Incident incident) => incident.Visible), + Codecs.Field("owner", Codecs.Nullable(Codecs.Text), (Incident incident) => incident.Owner), + (id, service, severity, age, visible, owner) => new Incident(id, service, severity, age, visible, owner)); + + // An enumeration's members name nullary constructors: PageNow is page-now. + // A union's cases are records, enumerations, or unions of its subtypes. + public static readonly UnionCodec TriageCodec = Codecs.Union( + Codecs.Record( + "Routed", + Codecs.Field("id", Codecs.Text, (Routed routed) => routed.Id), + Codecs.Field("team", Codecs.Text, (Routed routed) => routed.Team), + Codecs.Field("urgency", Codecs.Enum(), (Routed routed) => routed.Urgency), + Codecs.Field("reasons", Codecs.List(Codecs.Text), (Routed routed) => routed.Reasons), + (id, team, urgency, reasons) => new Routed(id, team, urgency, reasons)), + Codecs.Record("InvalidIncident", Codecs.Field("id", Codecs.Text, (InvalidIncident invalid) => invalid.Id), id => new InvalidIncident(id))); + + // triage.batch : (TriagePolicy, List Incident) ->{} List Triage, run in + // explicit batches of eight; each batch is checked before any runs. + public static IReadOnlyList Route(Function batch, Dynamic policy, IReadOnlyList incidents) => + batch.MapBatches(TriageCodec, IncidentCodec, incidents, 8, new Argument[] { policy }); +} +``` + +The fields given must follow the Jacquard constructor's positional order; the +pinned CLI exports no field types, so the order is not checked against Core. +Constructor names may use display spelling (`WorkToday`) or store spelling +(`work-today`). A constructor that is not in the prepared program is refused +when a value is encoded, and a result built by a different constructor fails +to decode. `null` is never a Jacquard value (except through a nullable +codec). **Money crosses as integer minor units** (cents, say) in an `Int`: +Jacquard's `Int` is 63-bit and its `Real` is a binary64 `double`, and there is +no decimal type. The SDK has no `decimal` codec, and a `decimal` argument or +`Dynamic.Of(decimal)` does not compile (the conversion is obsolete as an error +saying why), so a `decimal` is never converted silently; a test checks both. + +Every encoding is bounded by the selected limits as it goes: the JSON depth +the wire form needs, `max_collection_items` for each list, tuple, and +constructor, `max_value_nodes` values, `max_text_bytes` (in UTF-8 bytes) for +each text, and `max_frame_bytes` for all texts together. A list, tuple, +record, or constructor over its count limit is refused from its count before +its items or fields are read, a text over its limit before it is scanned, and +nesting before the nested value is encoded, so a small .NET value that denotes +a huge one (1024 references to one tuple of 1024 references, say) costs no +more than one frame, and no value or codec nesting can recurse past the depth +limit. `ProgramData.Decode` measures a value within the same node and depth +limits before it decodes it. A custom codec may return a `Value` it built +itself rather than through the `Encoder`; such a value is still measured: +before a function call checks the arguments' types, it walks every argument +value under one shared budget (nodes, collection items, text bytes alone and +together, and depth), whatever codec built it, so a small shared graph that +stands for a huge value is refused before the type check walks it. The invoke +frame is then measured as it is rendered, before any worker starts, and the +host measures a handler's result the same way. A value returned by +`ProgramData.Encode` is not measured again. Raw values, decoded values, +nested unions (flattened on first use to their distinct cases), chains of +lazy codecs, and a `Dynamic`'s equality and text are walked iteratively, so +none can exhaust the call stack; equality compares each pair of shared +sub-values once, so values that share sub-values compare in time +proportional to their distinct nodes. `PreparedProgram.Data` gives each public +name's exact identity. `ProgramData.Make` builds a `Dynamic.Variant` after +checking that its constructor exists, and `Invocation.ToDynamic` decodes any +result into `Dynamic`. A `Dynamic` list, tuple, or variant with more items +than the v0 hard limit (1,024) is refused when it is made, from its count +where the collection gives one and otherwise after at most one item past the +limit. + +## Services and failures + +```csharp +using System.Collections.Generic; +using JacquardLang.Host; +using JacquardLang.Host.Sdk; + +public static class Documents +{ + // summarize : (Text) ->{Documents} Text, where Documents has one operation, + // read-document : (Text) -> Text. + public static Function Summarize(PreparedProgram program) + { + Operations documents = new Operations().On("read-document", Codecs.Text, Codecs.Text, (context, key) => + { + // The context reports a cancelled or ended invocation; its + // CancellationToken is cancelled then too. + if (context.IsCancelled) + { + throw HandlerException.CompletionUnknown("cancelled"); + } + + return key == "example" ? "A synthetic document supplied by .NET." : throw HandlerException.Refused("unknown document"); + }); + return program.Function("summarize", new Dictionary { ["Documents"] = documents }); + } +} +``` + +An `Operations` value maps operation names (store spelling such as +`read-document`, or display spelling; `_` is accepted for `-`) to handlers. +One `Operations` may serve several effects. + +- **Parameters:** `On` takes one codec per parameter (up to six) and one for + the result. The handler may take the `HandlerContext` first; the remaining + parameters are decoded from the request. The context's `IsCancelled` and + `CancellationToken` report a cancelled, expired, or ended invocation (see + [handlers and cancellation](#handlers-and-cancellation)). +- **Results:** the handler returns a value for its result codec + (`Unit.Value` for `Codecs.Unit`). +- **Failures:** throw `HandlerException.Unsupported`, `Refused`, + `OutsideFailed`, or `CompletionUnknown` to report that failure. Any other + exception is reported to Core as completion unknown without its message. +- **Binding:** every operation in each checked root effect needs a handler + before the first invocation, including operations a given path never + reaches. A name that is no operation of the program is refused, and a + service binding more names than the program has operations is refused from + its count before its handlers are copied. `PreparedProgram.Provide` shares + services across functions; each function receives only its own checked + effects. + +Handlers run on their own threads, so share state through a lock, as the +fulfillment example's warehouse does. `RuntimeOptions.Deadline` (default +30 s) bounds each worker lifetime. + +## Errors + +Every failure to prepare, bind, encode, call, or decode is a +`JacquardException`, including any exception (other than +`OutOfMemoryException`) thrown by your own codec, a record field's reader, a +record's constructor function, or a lazy codec's factory: while a value is +encoded, by a function or by `ProgramData.Encode`, it is an `InvalidArgument` +naming the value, and while a value is decoded, by `ProgramData.Decode`, +`Invocation.Decode`, `Call`, or `MapBatches`, a `Binding` failure (from +`Invocation.Decode`, `Call`, and `MapBatches`, with the raw result kept); +either way your exception is its `InnerException`. Names and exception +messages the SDK quotes (a constructor, type, field, effect, operation, or +executable name, or your exception's message) appear as an excerpt of at most +64 chars and its length. A function, effect, operation, public, or constructor +name longer than 4,096 chars is refused before it is looked up or hashed. A message you pass to `Encoder.Invalid` or +`Decoder.Mapping` is kept as written, and no message is longer than +`JacquardException.MessageLimit` (8,192 chars), the start kept and the whole +length noted, once, however many prefixes are joined to it. Some failures +keep their usual .NET types: passing `null` where an object is required +throws `ArgumentNullException`; building a codec that cannot work (an +enumeration map missing a member, a union case that is not a subtype) or a +`Dynamic` over the hard limit throws `ArgumentException` at construction; +`Invocation.Save` throws `IOException` or `UnauthorizedAccessException`; an +exception from the source list `Prepare` takes or the bindings dictionary +`Function` takes (its `Count`, indexer, or enumerator) is wrapped, with the +cause, in a `Preparation` or `Binding` exception; and one from any other +collection you pass propagates unchanged, except for `MapBatches` rows, which +are read while a batch is encoded and so are reported as a codec failure. A +`JacquardException`'s `Kind` is: +- `InvalidArgument`: a local refusal, such as a value of the wrong type or + one too large or too deep for a frame. No worker starts. `Invoke` throws + only this; otherwise the `Invocation` holds the raw result. +- `Invocation`: the invocation ran and did not succeed, from `Call`, + `Invocation.Decode`, and `MapBatches`. `Failure` classifies it as + `DeadlineExceeded`, `WorkerLost`, `OperationFailed`, or `Other`, and + `Result` keeps the raw result. +- `Binding`: such as an unknown name, a missing handler, invalid runtime + options, or a result that does not decode into the requested type. In the + decode case, `Result` keeps the successful Core result. +- `Preparation`: checking, the cache, and the Core commands. + +`MapBatches` runs a pure function whose final parameter and result are Lists +over explicit batches of rows, encoded with the row codec you pass, with any +other arguments first. It validates every batch before the first runs and +retries nothing. Validation keeps no batch's encoding: each batch is encoded +and checked by a step that returns nothing, and encoded and checked again +just before it runs. Before each batch, and once more after the last, it +observes the token. As in the other SDKs, validation has no deadline of its +own. A cancellation there is reported as the host's aborted invocation, +without a worker. A failed batch's exception names it in `BatchIndex` (-1 when +nothing ran: a batch was refused, or validation stopped) and keeps the decoded +values of the batches that completed in `Completed`, read-only. + +A request argument that does not decode into a handler's parameter, or a +wrong argument count, is reported as an outside failure, and that handler +does not run. Neither does a handler whose invocation was cancelled, or whose +lifetime passed, while its arguments were decoded. A result that cannot be +encoded is also reported as an outside failure, but only after the handler +returned, so any action it took has happened. None of these says anything +about earlier operations in the invocation. + +## Run the projects + +```sh +cd dotnet +dotnet build examples/JacquardExamples +RUN="dotnet examples/JacquardExamples/bin/Debug/net8.0/JacquardExamples.dll" +$RUN planner +$RUN triage +$RUN fulfillment +$RUN fulfillment +``` + +- Planner returns score 16. +- Triage returns PAGE, TODAY, and WATCH. +- Fulfillment returns Fulfilled, then DuplicateOrder with the same receipt. + +The command reads the rules and synthetic inputs from the repository's +[`examples/`](../examples/README.md), from the repository root or `dotnet/` +(or `JACQUARD_EXAMPLES=DIR`), so it runs from a clone. Custom input JSON may +follow the project name. `--inspect PATH` saves frames; triage adds a +numbered suffix per batch. State defaults to `.scratch/dotnet-examples/`; +its program cache, and a `--inspect` file, are refused while another user +could write a directory above them (one writable by others, or by a group +other than your private group), so run `chmod go-w` on the directories the +message names, or pass `--workdir` and `--inspect` paths under directories +only you or root may write. The +exit status is 1 for a Jacquard or SDK failure and 2 for usage or other +errors. + +Try explicit recovery in a fresh work directory: + +```sh +$RUN --workdir .scratch/dotnet-recovery --fault publish-failed fulfillment +$RUN --workdir .scratch/dotnet-recovery --deliver-pending fulfillment +$RUN --workdir .scratch/dotnet-recovery --deliver-pending fulfillment +``` + +The first command exits 1 with E1613 and one pending booking. Reconciliation +then delivers one receipt, and the next run delivers zero. The `after-commit` +and `after-delivery` faults exercise E1612 around the snapshot commits. + +The example's `Warehouse` is a single-process service. It writes bookings and +outbox entries together as one atomic JSON snapshot (a temporary file it +creates exclusively, then renames; it removes that file only if it created it +and did not rename it), and a simulated delivery sink deduplicates receipts. +It has no cross-process locking and makes no power-loss durability claim. + +## Namespaces + +| Namespace | Contents | +|---|---| +| `JacquardLang.Host.Sdk` | The SDK: `Runtime`, `PreparedProgram`, `ProgramData`, `Function`, `Invocation`, `Operations`, `Codec` and `Codecs`, `Dynamic`, `Option`, `Unit`, `Argument`, and `JacquardException` | +| `JacquardLang.Host` | Low-level `Host`, `OperationRegistry`, `HandlerException`, `HandlerContext`, and `InvocationResult` for exact identities | +| `JacquardLang.Host.Carrier` | The bounded process carrier for `jacquard host worker` | +| `JacquardLang.Host.Protocol` | Strict JSON, v0 values and types, limits, the `Session` state machine | +| `JacquardLang.Host.Conformance` | The embedded copy of the pinned conformance kit and its replayer | + +`Compat` holds the generated pin constants. + +## The protocol-level host + +The example below invokes a checked function whose single operation is served +by a .NET handler: ```csharp using System; @@ -116,7 +674,7 @@ size of what a handler or caller hands over: ## Handlers and cancellation -A handler returns a value or throws a `HandlerException`. The four structured +These rules apply to SDK handlers and to low-level handlers. A handler returns a value or throws a `HandlerException`. The four structured categories are `Unsupported`, `Refused`, `OutsideFailed`, and `CompletionUnknown`. Any other exception a handler throws is reported to Core as completion unknown, and its message is never sent. A returned value with @@ -176,7 +734,7 @@ Unix an anonymous pipe's asynchronous read is a blocking read on a pool thread. The carrier is built around both: - **No read or write ever blocks.** The caller's thread asks the C library's - `poll(2)` (the one native call the adapter makes) whether a pipe is ready, + `poll(2)` (the one native call the carrier makes) whether a pipe is ready, with a poll timeout of at most 5 ms that never ends after the deadline, and only then reads the bytes already in the pipe, or writes at most `PIPE_BUF` bytes (4096 on Linux, 512 on macOS), which a @@ -237,6 +795,64 @@ invocations, and check that a reparented descendant flooding stdout or stderr holds neither the caller nor the heap, at a worker's own exit, at expiry, and at cancellation. +## Core's commands + +The SDK runs Core's `--version`, `check`, `hash`, and `run` commands through +its own bounded runner, which uses the carrier's approach to pipes: stdout and +stderr are read only after `poll(2)` reports them ready, on the caller's +thread, in waits of 1 to 5 ms that recheck the 30-second timeout and the +token; .NET's own stream reads are never left waiting. Each read step takes +at most 1 MiB of each stream, keeps at most 8 MiB of stdout (the rest is read +and counted, and refused as "output exceeds 8 MiB") and 4 KiB of stderr, and +after the exit at most 4 MiB more of each is read. The command's stdin is a +pipe the host closes at once. + +On Linux, when a `setsid` is installed at `/usr/bin/setsid` or `/bin/setsid` +(util-linux's, or a compatible one such as BusyBox's; any executable there is +used), each command runs as `setsid -- `. .NET's child is never a +process group leader, so `setsid` starts a new session and execs Core without +forking: Core keeps its pid and leads a new process group whose id is that pid +(a test checks both). Every descendant that does not leave the group shares +it. The host kills that group with `kill(-pgid, SIGKILL)` when the command +exits, and on a timeout or cancellation (then also killing Core itself, +waiting for its exit until 750 ms after the stop, and killing the group again +for members forked meanwhile). So `sh -c 'sleep 300 & echo $!'` succeeds, as in +the Rust SDK, and its child has been sent SIGKILL by the time it returns. Once .NET has reaped +Core, a live process holding Core's pid means the id was reused, so the old +group is empty and nothing is killed; a process that took Core's pid after a +full pid wraparound, made its own group, and forked between that check and the +kill could still have that group killed, the same window as the Rust SDK's +`kill(-pgid)`. When `setsid` cannot exec Core (exit status 126 or 127 with its +own `setsid: ` message), the command is reported as a start failure. A +descendant that leaves the group (calls `setsid` itself) is neither detected +nor killed; the host does not wait for the end of the output after the exit, +and closes its ends, so that descendant's next write fails and it cannot hold +the caller. + +Without `setsid` (Linux without one at those paths), the command's +descendants are recorded from `/proc` about every 10 ms, by pid and start +time, and every recorded descendant still alive with that start time is +killed when the command returns (the start time is read just before the +kill, as the JDK's process handles do). Each record reads at most 16,384 +`/proc` entries and 4 MiB of `stat` files in all, each read limited to 1 KiB +and to the bytes left, at most 16,384 descendants are kept, and a record ends +at the deadline or a cancellation, checked before each entry. The kills when +Core exits by itself run to the end; a stop that lands during them lets them +go on for at most 250 ms more. On a timeout or cancellation, Core is first +sent `SIGSTOP` (only while its start time matches the one read when it +started), so it starts no more children and its children stay its children; +then a last record is made, within the same caps, until 250 ms after the +stop; then Core is killed, and every recorded descendant until 500 ms after +the stop; then the host waits for Core's exit until 750 ms after the stop. +Each budget is checked before each entry or descendant, so the cleanup +returns within 750 ms of the stop plus one `stat` read and one signal (tests +check the order and each budget's end on a test clock). .NET's `Process.Kill(true)`, which reads every process on the +machine, is not used. Only Core is frozen, so a descendant whose parent exits +before the last record reaches it is reparented and keeps running, as do +one past the caps or the budgets and one started and reparented between two +records. macOS has no `/proc`, so nothing is recorded there: +only .NET's tree kill on a stop applies. + ## Conformance kit `Kit.Load()` verifies the embedded copy of the kit against the pinned digests. @@ -257,16 +873,20 @@ dotnet test test/JacquardHost.Tests -f net10.0 JACQUARD_REQUIRE_LIVE=1 dotnet test test/JacquardHost.Tests -f net8.0 ``` -The live tests in `LiveTests` are ignored without `JACQUARD_BIN` and -`JACQUARD_PRELUDE` (set `JACQUARD_REQUIRE_LIVE` to make them fail instead). -The offline tests drive `test/JacquardHost.StandIn`, a console program that -replays recordings or simulates faulty workers in a fresh process. Its -descendant modes use `sh` and `sleep`, so those tests run on Unix only, and -the descriptor tests run on Linux only. The build treats warnings as errors, +The live tests (`LiveTests`, `SdkLiveTests`, and the live `ExamplesTests`) +are ignored without `JACQUARD_BIN` and `JACQUARD_PRELUDE` (set +`JACQUARD_REQUIRE_LIVE` to make them fail instead). The offline tests drive +`test/JacquardHost.StandIn`, a console program that replays recordings or +simulates faulty workers in a fresh process, and a fake Core (a shell script) +for the SDK's preparation. Their descendant modes use `sh` and `sleep`, so +those tests run on Unix only, and the descriptor, Core command, and +preparation tests run on Linux only. The build treats warnings as errors, with nullable reference types and the `8.0-recommended` analyzers. `src/JacquardHost/Compat.cs` is generated, and `src/JacquardHost/kit/host-protocol-v0/` is a byte copy of the pinned kit. `scripts/core_pin.py` maintains both; do not edit them by hand. `NOTICE` must -stay identical to the repository's `NOTICE`, and the README example must stay -identical to `test/JacquardHost.Tests/Readme/Echo.cs`; tests check both. +stay identical to the repository's `NOTICE`, and the README's examples must +stay identical to `test/JacquardHost.Tests/Readme/` (`Shipping.cs`, +`Incidents.cs`, `Documents.cs`, and `Echo.cs`, in that order), which the live +tests run; tests check both. diff --git a/dotnet/examples/JacquardExamples/Examples.cs b/dotnet/examples/JacquardExamples/Examples.cs new file mode 100644 index 0000000..a893af6 --- /dev/null +++ b/dotnet/examples/JacquardExamples/Examples.cs @@ -0,0 +1,912 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using JacquardLang.Host; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; + +namespace JacquardLang.Host.Examples; + +/// +/// Runs the repository's example Jacquard projects through the .NET SDK: +/// quickstart, planner, triage, and fulfillment. The same rules run through +/// the Python, TypeScript, Go, Rust, and Java SDKs. +/// +/// +/// +/// dotnet examples/JacquardExamples/bin/Debug/net8.0/JacquardExamples.dll \ +/// [--workdir DIR] [--inspect PATH] [--fault NAME] [--deliver-pending] PROJECT [INPUT.json] +/// +/// Core is found through JACQUARD_BIN and JACQUARD_PRELUDE. The +/// rules are read from the repository's examples/ directory +/// (JACQUARD_EXAMPLES, else examples/ or ../examples/ +/// under the working directory). The exit status is 1 for a Jacquard or SDK +/// failure and 2 for usage or other errors. +/// +public static class Examples +{ + /// The example projects in presentation order. + public static IReadOnlyList Projects { get; } = new[] { "quickstart", "planner", "triage", "fulfillment" }; + + /// The faults the fulfillment store can inject. + private static readonly string[] Faults = { string.Empty, "publish-failed", "after-commit", "after-delivery" }; + + /// Runs one command line and exits with its status. + public static int Main(string[] args) + { + try + { + Run(args, Console.Out); + return 0; + } + catch (JacquardException failure) + { + Console.Out.Flush(); + Console.Error.WriteLine(failure.Message); + return 1; + } + catch (Exception failure) when (failure is UsageException or IOException or UnauthorizedAccessException) + { + Console.Out.Flush(); + Console.Error.WriteLine(failure.Message); + return 2; + } + } + + /// The repository's example projects. + internal static string ExamplesDirectory(string? configured) + { + configured ??= Environment.GetEnvironmentVariable("JACQUARD_EXAMPLES"); + if (!string.IsNullOrEmpty(configured)) + { + return Path.GetFullPath(configured); + } + + foreach (string candidate in new[] { "examples", Path.Combine("..", "examples") }) + { + if (File.Exists(Path.Combine(candidate, "quickstart", "rules.jac"))) + { + return Path.GetFullPath(candidate); + } + } + + throw new UsageException("cannot find the examples directory; run from the repository or dotnet/, or set JACQUARD_EXAMPLES"); + } + + // JSON input and output. + private static Json Field(JsonObject json, string key) => json.Get(key) ?? throw new UsageException("input: missing " + key); + + private static string TextOf(JsonObject json, string key) => Field(json, key).AsString() ?? throw new UsageException("input: " + key + " must be text"); + + private static long IntegerOf(JsonObject json, string key) => Field(json, key).AsInt() ?? throw new UsageException("input: " + key + " must be an integer"); + + private static bool BoolOf(JsonObject json, string key) => + Field(json, key) is JsonBool value ? value.Value : throw new UsageException("input: " + key + " must be true or false"); + + private static List Objects(JsonObject json, string key) + { + IReadOnlyList items = Field(json, key).AsArray() ?? throw new UsageException("input: " + key + " must be a list"); + return items.Select(item => item.AsObject() ?? throw new UsageException("input: " + key + " must hold objects")).ToList(); + } + + private static JsonObject Tagged(string tag, params (string Key, Json Value)[] pairs) => + Json.Object(pairs.Append(("$tag", Json.Of(tag))).ToArray()); + + internal static string Compact(Json value) => JsonCodec.Stringify(value, 16); + + private static void Pretty(Json value, int indent, StringBuilder output) + { + string pad = new(' ', indent * 2); + if (value is JsonArray array && array.Items.Count > 0) + { + output.Append("[\n"); + for (int i = 0; i < array.Items.Count; i++) + { + output.Append(pad).Append(" "); + Pretty(array.Items[i], indent + 1, output); + output.Append(i + 1 < array.Items.Count ? ",\n" : "\n"); + } + + output.Append(pad).Append(']'); + } + else if (value is JsonObject json && json.Count > 0) + { + output.Append("{\n"); + int i = 0; + foreach (KeyValuePair field in json) + { + output.Append(pad).Append(" ").Append(Compact(Json.Of(field.Key))).Append(": "); + Pretty(field.Value, indent + 1, output); + output.Append(++i < json.Count ? ",\n" : "\n"); + } + + output.Append(pad).Append('}'); + } + else + { + output.Append(Compact(value)); + } + } + + private static void PrintJson(TextWriter output, Json value) + { + var text = new StringBuilder(); + Pretty(value, 0, text); + output.Write(text.Append('\n').ToString()); + } + + // Planner. + + /// A job to place. + public sealed record Job(string Id, long Effort, long Value, long Latest); + + /// A job placed in a slot. + public sealed record Assignment(string Id, long Slot); + + /// A plan and its score. + public sealed record Plan(long Score, IReadOnlyList Assignments); + + /// The planner's result. + public abstract record Planning; + + /// A valid plan. + public sealed record Planned(Plan Plan) : Planning; + + /// Invalid input. + public sealed record InvalidPlan(string Reason) : Planning; + + internal static readonly RecordCodec JobCodec = Codecs.Record( + "Job", + Codecs.Field("id", Codecs.Text, (Job job) => job.Id), + Codecs.Field("effort", Codecs.Int, (Job job) => job.Effort), + Codecs.Field("value", Codecs.Int, (Job job) => job.Value), + Codecs.Field("latest", Codecs.Int, (Job job) => job.Latest), + (id, effort, value, latest) => new Job(id, effort, value, latest)); + + private static readonly RecordCodec AssignmentCodec = Codecs.Record( + "Assignment", + Codecs.Field("id", Codecs.Text, (Assignment assignment) => assignment.Id), + Codecs.Field("slot", Codecs.Int, (Assignment assignment) => assignment.Slot), + (id, slot) => new Assignment(id, slot)); + + private static readonly RecordCodec PlanCodec = Codecs.Record( + "Plan", + Codecs.Field("score", Codecs.Int, (Plan plan) => plan.Score), + Codecs.Field("assignments", Codecs.List(AssignmentCodec), (Plan plan) => plan.Assignments), + (score, assignments) => new Plan(score, assignments)); + + internal static readonly UnionCodec PlanningCodec = Codecs.Union( + Codecs.Record("Planned", Codecs.Field("plan", PlanCodec, (Planned planned) => planned.Plan), plan => new Planned(plan)), + Codecs.Record("InvalidPlan", Codecs.Field("reason", Codecs.Text, (InvalidPlan invalid) => invalid.Reason), reason => new InvalidPlan(reason))); + + private static JsonObject PlanningJson(Planning planning) => planning switch + { + Planned(Plan plan) => Tagged("Planned", ("plan", Json.Object( + ("assignments", Json.Of(plan.Assignments.Select(assignment => (Json)Json.Object(("id", Json.Of(assignment.Id)), ("slot", Json.Of(assignment.Slot)))))), + ("score", Json.Of(plan.Score))))), + InvalidPlan(string reason) => Tagged("InvalidPlan", ("reason", Json.Of(reason))), + _ => throw new InvalidOperationException("unknown planning result"), + }; + + // Triage. + + /// When an incident pages, and when it is stale. + public sealed record TriagePolicy(long Page, long Stale); + + /// One incident; a missing owner is null. + public sealed record Incident(string Id, string Service, long Severity, long Age, bool Visible, string? Owner); + + /// Members map by convention: PageNow is the constructor PageNow. + public enum Urgency + { + /// Page someone now. + PageNow, + + /// Work on it today. + WorkToday, + + /// Only watch it. + WatchOnly, + } + + /// The triage result. + public abstract record Triage; + + /// A routed incident. + public sealed record Routed(string Id, string Team, Urgency Urgency, IReadOnlyList Reasons) : Triage; + + /// An invalid incident. + public sealed record InvalidIncident(string Id) : Triage; + + internal static readonly RecordCodec PolicyCodec = Codecs.Record( + "TriagePolicy", + Codecs.Field("page", Codecs.Int, (TriagePolicy policy) => policy.Page), + Codecs.Field("stale", Codecs.Int, (TriagePolicy policy) => policy.Stale), + (page, stale) => new TriagePolicy(page, stale)); + + internal static readonly RecordCodec IncidentCodec = Codecs.Record( + "Incident", + Codecs.Field("id", Codecs.Text, (Incident incident) => incident.Id), + Codecs.Field("service", Codecs.Text, (Incident incident) => incident.Service), + Codecs.Field("severity", Codecs.Int, (Incident incident) => incident.Severity), + Codecs.Field("age", Codecs.Int, (Incident incident) => incident.Age), + Codecs.Field("visible", Codecs.Bool, (Incident incident) => incident.Visible), + Codecs.Field("owner", Codecs.Nullable(Codecs.Text), (Incident incident) => incident.Owner), + (id, service, severity, age, visible, owner) => new Incident(id, service, severity, age, visible, owner)); + + internal static readonly UnionCodec TriageCodec = Codecs.Union( + Codecs.Record( + "Routed", + Codecs.Field("id", Codecs.Text, (Routed routed) => routed.Id), + Codecs.Field("team", Codecs.Text, (Routed routed) => routed.Team), + Codecs.Field("urgency", Codecs.Enum(), (Routed routed) => routed.Urgency), + Codecs.Field("reasons", Codecs.List(Codecs.Text), (Routed routed) => routed.Reasons), + (id, team, urgency, reasons) => new Routed(id, team, urgency, reasons)), + Codecs.Record("InvalidIncident", Codecs.Field("id", Codecs.Text, (InvalidIncident invalid) => invalid.Id), id => new InvalidIncident(id))); + + private static JsonObject TriageJson(Triage triage) => triage switch + { + Routed routed => Tagged( + "Routed", + ("id", Json.Of(routed.Id)), + ("team", Json.Of(routed.Team)), + ("urgency", Json.Of(routed.Urgency switch + { + Urgency.PageNow => "PAGE", + Urgency.WorkToday => "TODAY", + _ => "WATCH", + })), + ("reasons", Json.Of(routed.Reasons.Select(reason => (Json)Json.Of(reason))))), + InvalidIncident(string id) => Tagged("InvalidIncident", ("id", Json.Of(id))), + _ => throw new InvalidOperationException("unknown triage result"), + }; + + // Fulfillment. + + /// One order line. + public sealed record Line(string Sku, long Units); + + /// An order. + public sealed record Order(string Id, IReadOnlyList Lines); + + /// A stock lookup: Stock is also the type's name, so the .NET case is , mapped explicitly. + public abstract record Stock; + + /// Units available. + public sealed record Available(long Units) : Stock; + + /// An unknown SKU. + public sealed record UnknownSku : Stock; + + /// A reservation. + public abstract record Reservation; + + /// No booking yet. + public sealed record BookingPending : Reservation; + + /// Booked, with its receipt. + public sealed record Booked(string Receipt) : Reservation; + + /// The same order was booked before. + public sealed record RepeatBooking(string Receipt) : Reservation; + + /// Another order was booked under this id. + public sealed record BookingConflict : Reservation; + + /// Stock changed before the commit. + public sealed record StockChanged : Reservation; + + /// The fulfillment result. + public abstract record Fulfillment; + + /// Fulfilled, with its receipt. + public sealed record Fulfilled(string Receipt) : Fulfillment; + + /// A repeated order. + public sealed record DuplicateOrder(string Receipt) : Fulfillment; + + /// A rejected order. + public sealed record OrderRejected(string Reason) : Fulfillment; + + private static readonly RecordCodec LineCodec = Codecs.Record( + "Line", + Codecs.Field("sku", Codecs.Text, (Line line) => line.Sku), + Codecs.Field("units", Codecs.Int, (Line line) => line.Units), + (sku, units) => new Line(sku, units)); + + internal static readonly RecordCodec OrderCodec = Codecs.Record( + "Order", + Codecs.Field("id", Codecs.Text, (Order order) => order.Id), + Codecs.Field("lines", Codecs.List(LineCodec), (Order order) => order.Lines), + (id, lines) => new Order(id, lines)); + + private static readonly UnionCodec StockCodec = Codecs.Union( + Codecs.Record("Stock", Codecs.Field("units", Codecs.Int, (Available available) => available.Units), units => new Available(units)), + Codecs.Record("UnknownSku", () => new UnknownSku())); + + private static readonly UnionCodec ReservationCodec = Codecs.Union( + Codecs.Record("BookingPending", () => new BookingPending()), + Codecs.Record("Booked", Codecs.Field("receipt", Codecs.Text, (Booked booked) => booked.Receipt), receipt => new Booked(receipt)), + Codecs.Record("RepeatBooking", Codecs.Field("receipt", Codecs.Text, (RepeatBooking repeat) => repeat.Receipt), receipt => new RepeatBooking(receipt)), + Codecs.Record("BookingConflict", () => new BookingConflict()), + Codecs.Record("StockChanged", () => new StockChanged())); + + internal static readonly UnionCodec FulfillmentCodec = Codecs.Union( + Codecs.Record("Fulfilled", Codecs.Field("receipt", Codecs.Text, (Fulfilled done) => done.Receipt), receipt => new Fulfilled(receipt)), + Codecs.Record("DuplicateOrder", Codecs.Field("receipt", Codecs.Text, (DuplicateOrder repeat) => repeat.Receipt), receipt => new DuplicateOrder(receipt)), + Codecs.Record("OrderRejected", Codecs.Field("reason", Codecs.Text, (OrderRejected rejected) => rejected.Reason), reason => new OrderRejected(reason))); + + private static JsonObject FulfillmentJson(Fulfillment fulfillment) => fulfillment switch + { + Fulfilled(string receipt) => Tagged("Fulfilled", ("receipt", Json.Of(receipt))), + DuplicateOrder(string receipt) => Tagged("DuplicateOrder", ("receipt", Json.Of(receipt))), + OrderRejected(string reason) => Tagged("OrderRejected", ("reason", Json.Of(reason))), + _ => throw new InvalidOperationException("unknown fulfillment result"), + }; + + /// Binds fulfillment.place to a shared warehouse serving both effects. + public static Function BindFulfillment(PreparedProgram program, Warehouse warehouse) + { + ArgumentNullException.ThrowIfNull(program); + ArgumentNullException.ThrowIfNull(warehouse); + Operations service = new Operations() + .On("booking-status", OrderCodec, ReservationCodec, warehouse.BookingStatus) + .On("lookup-stock", Codecs.Text, StockCodec, warehouse.LookupStock) + .On("reserve-order", OrderCodec, ReservationCodec, warehouse.ReserveOrder) + .On("publish-booking", Codecs.Text, Codecs.Text, Codecs.Unit, warehouse.PublishBooking); + return program.Function("fulfillment.place", new Dictionary { ["Inventory"] = service, ["Dispatch"] = service }); + } + + private static void Save(Invocation invocation, string? path) + { + if (path is null) + { + return; + } + + try + { + invocation.Save(path); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw new UsageException(error.Message); + } + } + + private static byte[] Read(string path) + { + try + { + return File.ReadAllBytes(path); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw new UsageException(error.Message); + } + } + + /// Runs one command line, writing its output to . + /// A Jacquard or SDK failure (exit status 1). + /// A usage, input, or I/O failure (exit status 2). + [SuppressMessage("Maintainability", "CA1502", Justification = "One command line, one case per project, as in the other SDKs' commands.")] + public static void Run(IReadOnlyList arguments, TextWriter output, string? examples = null) + { + ArgumentNullException.ThrowIfNull(arguments); + ArgumentNullException.ThrowIfNull(output); + string workdir = Path.Combine(".scratch", "dotnet-examples"); + string? inspect = null; + string fault = string.Empty; + bool deliverPending = false; + var positional = new List(); + for (int i = 0; i < arguments.Count; i++) + { + string argument = arguments[i]; + switch (argument) + { + case "--workdir" or "--inspect" or "--fault": + if (i + 1 >= arguments.Count) + { + throw new UsageException(argument + " needs a value"); + } + + string value = arguments[++i]; + if (argument == "--workdir") + { + workdir = value; + } + else if (argument == "--inspect") + { + inspect = value; + } + else + { + fault = value; + } + + break; + case "--deliver-pending": + deliverPending = true; + break; + default: + if (argument.StartsWith("--", StringComparison.Ordinal)) + { + throw new UsageException("unknown flag " + argument); + } + + positional.Add(argument); + break; + } + } + + string project = positional.Count == 0 ? string.Empty : positional[0]; + if (!Projects.Contains(project) || positional.Count > 2) + { + throw new UsageException("choose quickstart, triage, planner, or fulfillment"); + } + + if (!Faults.Contains(fault)) + { + throw new UsageException("unknown fault"); + } + + string root = Path.GetFullPath(workdir); + string directory = ExamplesDirectory(examples); + string rules = Path.Combine(directory, project, "rules.jac"); + var runtime = new Runtime(); + PreparedProgram program = runtime.Prepare(new[] { rules }, new PrepareOptions { CacheDir = Path.Combine(root, "programs") }); + if (project == "quickstart") + { + Function shipping = program.Function("shipping-cost"); + foreach (long total in new long[] { 35, 75 }) + { + long cost = shipping.Call(Codecs.Int, total); + output.Write("Order total " + total.ToString(CultureInfo.InvariantCulture) + ": shipping " + cost.ToString(CultureInfo.InvariantCulture) + "\n"); + } + + return; + } + + string inputPath = positional.Count > 1 ? positional[1] : Path.Combine(directory, project, "input.json"); + JsonObject input; + try + { + input = JsonCodec.Decode(Read(inputPath), 16).AsObject() ?? throw new UsageException("input must be a JSON object"); + } + catch (ProtocolException error) + { + throw new UsageException("input: " + error.Message); + } + + switch (project) + { + case "planner": + { + List jobs = Objects(input, "jobs") + .Select(job => new Job(TextOf(job, "id"), IntegerOf(job, "effort"), IntegerOf(job, "value"), IntegerOf(job, "latest"))) + .ToList(); + IReadOnlyList capacityItems = Field(input, "capacities").AsArray() ?? throw new UsageException("input: capacities must be a list"); + List capacities = capacityItems.Select(capacity => capacity.AsInt() ?? throw new UsageException("input: capacities must be integers")).ToList(); + Invocation invocation = program.Function("planner.plan").Invoke(Codecs.List(JobCodec).Arg(jobs), Codecs.List(Codecs.Int).Arg(capacities)); + Save(invocation, inspect); + PrintJson(output, PlanningJson(invocation.Decode(PlanningCodec))); + output.Write(invocation.Render() + "\n"); + break; + } + + case "triage": + { + JsonObject policyObject = Field(input, "policy").AsObject() ?? throw new UsageException("input: policy must be an object"); + var policy = new TriagePolicy(IntegerOf(policyObject, "page"), IntegerOf(policyObject, "stale")); + var incidents = new List(); + foreach (JsonObject incident in Objects(input, "incidents")) + { + // A missing owner is null, as in the Go command. + Json owner = incident.Get("owner") ?? Json.Null; + string? ownerValue = owner is JsonNull ? null : owner.AsString() ?? throw new UsageException("input: owner must be text or null"); + incidents.Add(new Incident( + TextOf(incident, "id"), TextOf(incident, "service"), IntegerOf(incident, "severity"), IntegerOf(incident, "age"), BoolOf(incident, "visible"), ownerValue)); + } + + Function route = program.Function("triage.batch"); + IReadOnlyList results; + if (inspect is null) + { + results = route.MapBatches(TriageCodec, IncidentCodec, incidents, 8, new[] { PolicyCodec.Arg(policy) }); + } + else + { + var collected = new List(); + for (int start = 0, index = 0; start < incidents.Count; start += 8, index++) + { + List chunk = incidents.GetRange(start, Math.Min(8, incidents.Count - start)); + Invocation invocation = route.Invoke(PolicyCodec.Arg(policy), Codecs.List(IncidentCodec).Arg(chunk)); + Save(invocation, inspect + "." + index.ToString(CultureInfo.InvariantCulture) + ".json"); + collected.AddRange(invocation.Decode(Codecs.List(TriageCodec))); + } + + results = collected; + } + + PrintJson(output, Json.Of(results.Select(result => (Json)TriageJson(result)))); + break; + } + + default: + { + List lines = Objects(input, "lines").Select(line => new Line(TextOf(line, "sku"), IntegerOf(line, "units"))).ToList(); + var order = new Order(TextOf(input, "id"), lines); + Warehouse warehouse = Warehouse.Open(Path.Combine(root, "inventory.json"), fault); + try + { + if (deliverPending) + { + int delivered; + try + { + delivered = warehouse.DeliverPending(); + } + catch (Exception error) when (error is HandlerException or IOException or UnauthorizedAccessException) + { + // Like the Go command, a failed delivery prints no count. + throw new UsageException("delivery failed: " + Describe(error)); + } + + output.Write(Compact(Json.Object(("delivered", Json.Of(delivered)))) + "\n"); + return; + } + + Invocation invocation = BindFulfillment(program, warehouse).Invoke(OrderCodec.Arg(order)); + Save(invocation, inspect); + JacquardException? failure = null; + try + { + PrintJson(output, FulfillmentJson(invocation.Decode(FulfillmentCodec))); + } + catch (JacquardException error) + { + failure = error; + } + + output.Write(invocation.Render() + "\n"); + if (failure is not null) + { + throw failure; + } + } + finally + { + output.Write(Compact(warehouse.Summary()) + "\n"); + } + + break; + } + } + } + + private static string Describe(Exception error) => + error is HandlerException failure ? failure.Category.WireName() + ": " + failure.Message : "completion unknown"; + + /// The canonical payload of an order: its lines sorted by SKU. + internal static string Payload(Order order) => + Compact(Json.Of(order.Lines.OrderBy(line => line.Sku, StringComparer.Ordinal) + .Select(line => (Json)Json.Of(new Json[] { Json.Of(line.Sku), Json.Of(line.Units) })))); + + internal static string Sha256(string text) => Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(text))).ToLowerInvariant(); + + /// + /// A single-process synthetic Inventory and Dispatch service. Bookings and + /// outbox entries are committed together as one atomic JSON snapshot; + /// deliveries are deduplicated by receipt. It has no cross-process + /// locking or power-loss durability claim. Its methods are synchronized, + /// because each handler runs on its own thread. + /// + public sealed class Warehouse + { + internal const string InventorySchema = "jacquard-example-inventory-v1"; + private static long next; + private readonly object sync = new(); + private readonly string path; + private readonly string fault; + private Inventory state; + + private Warehouse(string path, string fault, Inventory state) + { + this.path = path; + this.fault = fault; + this.state = state; + } + + private sealed record Booking(string Id, string Payload, string Receipt); + + private sealed record Outbox(string Id, string Receipt, bool Delivered); + + private sealed record Inventory(SortedDictionary Stock, List Skus, List Bookings, List Outbox, List Deliveries) + { + internal JsonObject ToJson() => Json.Object( + ("schema", Json.Of(InventorySchema)), + ("stock", Json.Of(Skus.Select(sku => (Json)Json.Object(("sku", Json.Of(sku)), ("available", Json.Of(Stock[sku])))))), + ("bookings", Json.Of(Bookings.Select(booking => (Json)Json.Object( + ("id", Json.Of(booking.Id)), ("payload", Json.Of(booking.Payload)), ("receipt", Json.Of(booking.Receipt)))))), + ("outbox", Json.Of(Outbox.Select(entry => (Json)Json.Object( + ("id", Json.Of(entry.Id)), ("receipt", Json.Of(entry.Receipt)), ("delivered", Json.Of(entry.Delivered)))))), + ("deliveries", Json.Of(Deliveries.Select(delivery => (Json)Json.Of(delivery))))); + + [SuppressMessage("Design", "CA1031", Justification = "Any malformed snapshot is reported as a version mismatch.")] + internal static Inventory? FromJson(Json value) + { + if (value.AsObject() is not { } json || !Json.Of(InventorySchema).Equals(json.Get("schema"))) + { + return null; + } + + try + { + var stock = new SortedDictionary(StringComparer.Ordinal); + var skus = new List(); + foreach (Json item in json["stock"].AsArray()!) + { + string sku = item.AsObject()!["sku"].AsString()!; + stock[sku] = item.AsObject()!["available"].AsInt()!.Value; + skus.Add(sku); + } + + List bookings = json["bookings"].AsArray()!.Select(item => item.AsObject()!) + .Select(fields => new Booking(fields["id"].AsString()!, fields["payload"].AsString()!, fields["receipt"].AsString()!)).ToList(); + List outbox = json["outbox"].AsArray()!.Select(item => item.AsObject()!) + .Select(fields => new Outbox(fields["id"].AsString()!, fields["receipt"].AsString()!, ((JsonBool)fields["delivered"]).Value)).ToList(); + List deliveries = json["deliveries"].AsArray()!.Select(item => item.AsString()!).ToList(); + return new Inventory(stock, skus, bookings, outbox, deliveries); + } + catch (Exception) + { + return null; + } + } + + internal Inventory Copy() => new( + new SortedDictionary(Stock, StringComparer.Ordinal), new List(Skus), new List(Bookings), new List(Outbox), new List(Deliveries)); + } + + /// Loads the snapshot at or starts synthetic stock. + /// The snapshot cannot be read or is not this version. + public static Warehouse Open(string path, string fault) + { + ArgumentNullException.ThrowIfNull(path); + ArgumentNullException.ThrowIfNull(fault); + Inventory state; + try + { + byte[] raw = File.ReadAllBytes(path); + Inventory? parsed; + try + { + parsed = Inventory.FromJson(JsonCodec.Decode(raw, 16)); + } + catch (ProtocolException) + { + parsed = null; + } + + state = parsed ?? throw new UsageException("inventory snapshot version mismatch"); + } + catch (FileNotFoundException) + { + state = new Inventory( + new SortedDictionary(StringComparer.Ordinal) { ["sensor"] = 20, ["cable"] = 30, ["battery"] = 15 }, + new List { "sensor", "cable", "battery" }, + new List(), + new List(), + new List()); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw new UsageException(error.Message); + } + + return new Warehouse(path, fault, state.Copy()); + } + + /// + /// Commits : a fresh owner-only file, created + /// exclusively, then renamed over the snapshot. Only a file this call + /// created is ever removed, and not once it was renamed. + /// + private void Commit(Inventory nextState) + { + string parent = Path.GetDirectoryName(Path.GetFullPath(path))!; + Directory.CreateDirectory(parent); + byte[] text = Encoding.UTF8.GetBytes(Compact(nextState.ToJson()) + "\n"); + string temporary = Path.Combine(parent, Path.GetFileName(path) + "." + Environment.ProcessId.ToString(CultureInfo.InvariantCulture) + "-" + + Interlocked.Increment(ref next).ToString(CultureInfo.InvariantCulture) + ".tmp"); + var options = new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write }; + if (!OperatingSystem.IsWindows()) + { + options.UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite; + } + + // Throws if the name exists, so nothing below runs for a file this call did not make. + var stream = new FileStream(temporary, options); + bool moved = false; + try + { + using (stream) + { + stream.Write(text); + } + + File.Move(temporary, path, overwrite: true); + moved = true; + } + finally + { + if (!moved) + { + File.Delete(temporary); + } + } + + state = nextState; + } + + private Booking? Find(string id) => state.Bookings.FirstOrDefault(booking => booking.Id == id); + + /// Serves Inventory.booking-status. + public Reservation BookingStatus(Order order) + { + ArgumentNullException.ThrowIfNull(order); + lock (sync) + { + Booking? previous = Find(order.Id); + if (previous is null) + { + return new BookingPending(); + } + + return previous.Payload == Payload(order) ? new RepeatBooking(previous.Receipt) : new BookingConflict(); + } + } + + /// Serves Inventory.lookup-stock. + public Stock LookupStock(string sku) + { + lock (sync) + { + return state.Stock.TryGetValue(sku, out long available) ? new Available(available) : new UnknownSku(); + } + } + + /// Serves Inventory.reserve-order, committing booking and outbox together. + /// An invalid reservation, or the injected fault after the commit. + public Reservation ReserveOrder(Order order) + { + ArgumentNullException.ThrowIfNull(order); + lock (sync) + { + var seen = new HashSet(StringComparer.Ordinal); + bool valid = order.Id.Length > 0 && order.Lines.Count >= 1 && order.Lines.Count <= 20 + && order.Lines.All(line => seen.Add(line.Sku) && line.Units >= 1 && line.Units <= 20); + if (!valid) + { + throw HandlerException.Refused("invalid synthetic reservation"); + } + + if (Find(order.Id) is not null) + { + return BookingStatus(order); + } + + Inventory nextState = state.Copy(); + foreach (Line line in order.Lines) + { + if (!nextState.Stock.TryGetValue(line.Sku, out long available) || available < line.Units) + { + return new StockChanged(); + } + + nextState.Stock[line.Sku] = available - line.Units; + } + + string body = Payload(order); + string receipt = Sha256(order.Id + "\0" + body); + nextState.Bookings.Add(new Booking(order.Id, body, receipt)); + nextState.Outbox.Add(new Outbox(order.Id, receipt, false)); + Commit(nextState); + if (fault == "after-commit") + { + throw HandlerException.CompletionUnknown("synthetic fault after the booking committed"); + } + + return new Booked(receipt); + } + } + + /// Serves Dispatch.publish-booking. + /// The injected delivery failure, no matching entry, or the fault after the delivery. + public Unit PublishBooking(string id, string receipt) + { + lock (sync) + { + if (fault == "publish-failed") + { + throw HandlerException.OutsideFailed("synthetic delivery unavailable; outbox retained"); + } + + Inventory nextState = state.Copy(); + int index = nextState.Outbox.FindIndex(entry => entry.Receipt == receipt && entry.Id == id); + if (index < 0) + { + throw HandlerException.Refused("no matching outbox entry"); + } + + nextState.Outbox[index] = new Outbox(id, receipt, true); + if (!nextState.Deliveries.Contains(receipt)) + { + nextState.Deliveries.Add(receipt); + } + + Commit(nextState); + if (fault == "after-delivery") + { + throw HandlerException.CompletionUnknown("synthetic fault after the delivery committed"); + } + + return Unit.Value; + } + } + + /// Publishes every undelivered outbox entry explicitly, returning how many were delivered. + public int DeliverPending() + { + lock (sync) + { + List pending = state.Outbox.Where(entry => !entry.Delivered).ToList(); + foreach (Outbox entry in pending) + { + PublishBooking(entry.Id, entry.Receipt); + } + + return pending.Count; + } + } + + /// Counts bookings, pending deliveries, and deliveries. + public JsonObject Summary() + { + lock (sync) + { + return Json.Object( + ("bookings", Json.Of(state.Bookings.Count)), + ("deliveries", Json.Of(state.Deliveries.Count)), + ("pendingDeliveries", Json.Of(state.Outbox.Count(entry => !entry.Delivered)))); + } + } + } +} + +/// A usage, input, or I/O failure: exit status 2. +public sealed class UsageException : Exception +{ + /// A failure with a message. + public UsageException() + { + } + + /// A failure with a message. + public UsageException(string message) + : base(message) + { + } + + /// A failure with a message and its cause. + public UsageException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/dotnet/examples/JacquardExamples/JacquardExamples.csproj b/dotnet/examples/JacquardExamples/JacquardExamples.csproj new file mode 100644 index 0000000..db872b7 --- /dev/null +++ b/dotnet/examples/JacquardExamples/JacquardExamples.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + JacquardLang.Host.Examples + JacquardExamples + false + + false + + + + + + + diff --git a/dotnet/src/JacquardHost/Sdk/Builtins.cs b/dotnet/src/JacquardHost/Sdk/Builtins.cs new file mode 100644 index 0000000..2e1930a --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Builtins.cs @@ -0,0 +1,797 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// The codec implementations behind 's properties and factories. +internal static class Builtins +{ + /// One field of a record codec. + internal interface IField + { + string Name { get; } + + Value Encode(Encoder encoder, TRecord record); + + object? Decode(Decoder decoder, Value value); + } + + /// A union, whose cases a flattening walk enters. + internal interface IUnion + { + IReadOnlyList Cases { get; } + } + + /// A lazy codec, which a walk resolves to what it stands for. + internal interface ILazy + { + object Target { get; } + } + + internal static IField[] Fields(params IField[] fields) => fields; + + /// + /// What stands for once every lazy codec on the + /// way is made, in a loop (so a chain of any length cannot exhaust the + /// stack) that refuses a lazy codec standing for itself. + /// + internal static object Resolve(object codec) + { + HashSet? seen = null; + while (codec is ILazy lazy) + { + seen ??= new HashSet(ReferenceEqualityComparer.Instance); + if (!seen.Add(codec)) + { + throw new InvalidOperationException("a lazy codec stands for itself"); + } + + codec = lazy.Target; + } + + return codec; + } + + // Built-in types. + internal sealed class IntCodec : Codec + { + public override Value Encode(Encoder encoder, long value) => encoder.Integer(value); + + public override long Decode(Decoder decoder, Value value) => value is IntValue number ? number.Value : throw decoder.Mapping("expected Int"); + } + + internal sealed class Int32Codec : Codec + { + public override Value Encode(Encoder encoder, int value) => encoder.Integer(value); + + public override int Decode(Decoder decoder, Value value) => + value is IntValue number && number.Value >= int.MinValue && number.Value <= int.MaxValue + ? (int)number.Value + : throw decoder.Mapping("expected Int that fits int"); + } + + internal sealed class RealCodec : Codec + { + public override Value Encode(Encoder encoder, double value) => encoder.Real(value); + + public override double Decode(Decoder decoder, Value value) => value is RealValue real ? real.ToDouble() : throw decoder.Mapping("expected Real"); + } + + internal sealed class TextCodec : Codec + { + public override Value Encode(Encoder encoder, string value) => encoder.Text(value); + + public override string Decode(Decoder decoder, Value value) => value is TextValue text ? text.Value : throw decoder.Mapping("expected Text"); + } + + internal sealed class BoolCodec : Codec + { + public override Value Encode(Encoder encoder, bool value) => encoder.Bool(value); + + public override bool Decode(Decoder decoder, Value value) => + decoder.Data.IsNullary(value, "true") || (decoder.Data.IsNullary(value, "false") ? false : throw decoder.Mapping("expected Bool")); + } + + internal sealed class HashCodec : Codec + { + public override Value Encode(Encoder encoder, Hash value) => encoder.Hash(value); + + public override Hash Decode(Decoder decoder, Value value) => value is HashValue hash ? hash.Hash : throw decoder.Mapping("expected Hash"); + } + + internal sealed class UnitCodec : Codec + { + public override Value Encode(Encoder encoder, Unit value) => encoder.Tuple(Array.Empty()); + + public override Unit Decode(Decoder decoder, Value value) => + value is TupleValue { Items.Count: 0 } ? default : throw decoder.Mapping("expected unit"); + } + + internal sealed class ValueCodec : Codec + { + public override Value Encode(Encoder encoder, Value value) => encoder.Raw(value); + + public override Value Decode(Decoder decoder, Value value) => value; + } + + internal sealed class CustomCodec : Codec + { + private readonly Func encode; + private readonly Func decode; + + internal CustomCodec(Func encode, Func decode) + { + this.encode = encode; + this.decode = decode; + } + + public override Value Encode(Encoder encoder, T value) => encode(encoder, value); + + public override T Decode(Decoder decoder, Value value) => decode(decoder, value); + } + + internal sealed class ListCodec : Codec> + { + private readonly Codec item; + + internal ListCodec(Codec item) + { + this.item = item; + } + + public override Value Encode(Encoder encoder, IReadOnlyList value) => encoder.List(item, value, 0, value.Count); + + public override IReadOnlyList Decode(Decoder decoder, Value value) + { + List items = decoder.Data.ListItems(value); + var output = new T[items.Count]; + for (int i = 0; i < output.Length; i++) + { + output[i] = item.Decode(decoder, items[i]); + } + + return output; + } + } + + /// The value inside Some, null for None, or a mapping failure for anything else. + private static Value? Some(Decoder decoder, Value value) + { + if (decoder.Data.IsNullary(value, "none")) + { + return null; + } + + if (value is ConstructorValue some && some.Arguments.Count == 1 && decoder.Data.Label(some.Identity) == "some") + { + return some.Arguments[0]; + } + + throw decoder.Mapping("expected Option"); + } + + internal sealed class OptionCodec : Codec> + { + private readonly Codec inner; + + internal OptionCodec(Codec inner) + { + this.inner = inner; + } + + public override Value Encode(Encoder encoder, Option value) => + encoder.Option(inner, value.HasValue, value.HasValue ? value.Value : default!); + + public override Option Decode(Decoder decoder, Value value) => + Some(decoder, value) is { } some ? Option.Some(inner.Decode(decoder, some)) : Option.None; + } + + internal sealed class NullableCodec : Codec + where T : class + { + private readonly Codec inner; + + internal NullableCodec(Codec inner) + { + this.inner = inner; + } + + internal override bool AcceptsNull => true; + + public override Value Encode(Encoder encoder, T? value) => encoder.Option(inner, value is not null, value!); + + public override T? Decode(Decoder decoder, Value value) + { + if (Some(decoder, value) is not { } some) + { + return null; + } + + return inner.Decode(decoder, some) ?? throw decoder.Mapping("Some of a value that decodes to null has no nullable form; use Codecs.Option"); + } + } + + internal sealed class NullableValueCodec : Codec + where T : struct + { + private readonly Codec inner; + + internal NullableValueCodec(Codec inner) + { + this.inner = inner; + } + + internal override bool AcceptsNull => true; + + public override Value Encode(Encoder encoder, T? value) => encoder.Option(inner, value.HasValue, value.GetValueOrDefault()); + + public override T? Decode(Decoder decoder, Value value) => Some(decoder, value) is { } some ? inner.Decode(decoder, some) : null; + } + + private static TupleValue Shape(Decoder decoder, Value value, int count) => + value is TupleValue tuple && tuple.Items.Count == count ? tuple : throw decoder.Mapping("tuple shape mismatch"); + + internal sealed class TupleCodec : Codec<(T1, T2)> + { + private readonly Codec first; + private readonly Codec second; + + internal TupleCodec(Codec first, Codec second) + { + this.first = first; + this.second = second; + } + + public override Value Encode(Encoder encoder, (T1, T2) value) + { + encoder.CheckCount(2, "tuple"); + return encoder.Tuple(new[] { encoder.Item(0, first, value.Item1), encoder.Item(1, second, value.Item2) }); + } + + public override (T1, T2) Decode(Decoder decoder, Value value) + { + TupleValue tuple = Shape(decoder, value, 2); + return (first.Decode(decoder, tuple.Items[0]), second.Decode(decoder, tuple.Items[1])); + } + } + + internal sealed class TupleCodec : Codec<(T1, T2, T3)> + { + private readonly Codec first; + private readonly Codec second; + private readonly Codec third; + + internal TupleCodec(Codec first, Codec second, Codec third) + { + this.first = first; + this.second = second; + this.third = third; + } + + public override Value Encode(Encoder encoder, (T1, T2, T3) value) + { + encoder.CheckCount(3, "tuple"); + return encoder.Tuple(new[] + { + encoder.Item(0, first, value.Item1), encoder.Item(1, second, value.Item2), encoder.Item(2, third, value.Item3), + }); + } + + public override (T1, T2, T3) Decode(Decoder decoder, Value value) + { + TupleValue tuple = Shape(decoder, value, 3); + return (first.Decode(decoder, tuple.Items[0]), second.Decode(decoder, tuple.Items[1]), third.Decode(decoder, tuple.Items[2])); + } + } + + internal sealed class TupleCodec : Codec<(T1, T2, T3, T4)> + { + private readonly Codec first; + private readonly Codec second; + private readonly Codec third; + private readonly Codec fourth; + + internal TupleCodec(Codec first, Codec second, Codec third, Codec fourth) + { + this.first = first; + this.second = second; + this.third = third; + this.fourth = fourth; + } + + public override Value Encode(Encoder encoder, (T1, T2, T3, T4) value) + { + encoder.CheckCount(4, "tuple"); + return encoder.Tuple(new[] + { + encoder.Item(0, first, value.Item1), + encoder.Item(1, second, value.Item2), + encoder.Item(2, third, value.Item3), + encoder.Item(3, fourth, value.Item4), + }); + } + + public override (T1, T2, T3, T4) Decode(Decoder decoder, Value value) + { + TupleValue tuple = Shape(decoder, value, 4); + return ( + first.Decode(decoder, tuple.Items[0]), + second.Decode(decoder, tuple.Items[1]), + third.Decode(decoder, tuple.Items[2]), + fourth.Decode(decoder, tuple.Items[3])); + } + } + + internal sealed class DynamicCodec : Codec + { + public override Value Encode(Encoder encoder, Dynamic value) + { + switch (value) + { + case Dynamic.Int number: + return encoder.Integer(number.Value); + case Dynamic.Real real: + return encoder.Real(real.Value); + case Dynamic.Text text: + return encoder.Text(text.Value); + case Dynamic.Bool flag: + return encoder.Bool(flag.Value); + case Dynamic.HashValue hash: + return encoder.Hash(hash.Value); + case Dynamic.Tuple tuple: + { + encoder.CheckCount(tuple.Items.Count, "tuple"); + var items = new Value[tuple.Items.Count]; + for (int i = 0; i < items.Length; i++) + { + items[i] = encoder.Item(i, this, tuple.Items[i]); + } + + return encoder.Tuple(items); + } + + case Dynamic.List list: + return encoder.List(this, list.Items, 0, list.Items.Count); + case Dynamic.Option option: + return encoder.Option(this, option.Value is not null, option.Value!); + case Dynamic.Variant variant: + { + encoder.CheckCount(variant.Fields.Count, "constructor"); + var fields = new Value[variant.Fields.Count]; + for (int i = 0; i < fields.Length; i++) + { + fields[i] = encoder.Field(i.ToString(System.Globalization.CultureInfo.InvariantCulture), this, variant.Fields[i]); + } + + return encoder.Constructor(variant.Name, fields); + } + + default: + throw encoder.Invalid("unknown Dynamic kind " + value.GetType().Name); + } + } + + /// Decodes a value Core sent, or one measured within the depth limit, so the recursion is bounded by that depth. + public override Dynamic Decode(Decoder decoder, Value value) + { + switch (value) + { + case IntValue number: + return new Dynamic.Int(number.Value); + case RealValue real: + return new Dynamic.Real(real.ToDouble()); + case TextValue text: + return new Dynamic.Text(text.Value); + case HashValue hash: + return new Dynamic.HashValue(hash.Hash); + case TupleValue tuple: + { + var items = new Dynamic[tuple.Items.Count]; + for (int i = 0; i < items.Length; i++) + { + items[i] = Decode(decoder, tuple.Items[i]); + } + + return new Dynamic.Tuple(items); + } + + case ConstructorValue constructor: + { + string label = decoder.Data.Label(constructor.Identity) ?? throw decoder.Mapping("unmapped constructor " + constructor.Identity.Hex); + switch (label) + { + case "true" or "false": + return new Dynamic.Bool(Codecs.Bool.Decode(decoder, value)); + case "cons" or "nil": + { + List items = decoder.Data.ListItems(value); + var decoded = new Dynamic[items.Count]; + for (int i = 0; i < decoded.Length; i++) + { + decoded[i] = Decode(decoder, items[i]); + } + + return new Dynamic.List(decoded); + } + + case "some" or "none": + return new Dynamic.Option(Some(decoder, value) is { } inner ? Decode(decoder, inner) : null); + default: + { + var fields = new Dynamic[constructor.Arguments.Count]; + for (int i = 0; i < fields.Length; i++) + { + fields[i] = Decode(decoder, constructor.Arguments[i]); + } + + return new Dynamic.Variant(label, fields); + } + } + } + + default: + throw decoder.Mapping("unknown value kind " + value.GetType().Name); + } + } + } +} + +/// A record (any .NET type) as one Jacquard constructor; made by and its overloads. +/// The record type. +public sealed class RecordCodec : Codec, ICase +{ + private readonly Builtins.IField[] fields; + private readonly Func make; + + internal RecordCodec(string constructor, Builtins.IField[] fields, Func make) + { + Constructor = constructor ?? throw new ArgumentNullException(nameof(constructor)); + foreach (Builtins.IField field in fields) + { + ArgumentNullException.ThrowIfNull(field, nameof(fields)); + } + + this.fields = fields; + this.make = make; + } + + /// The constructor's public name. + public string Constructor { get; } + + Type ICase.CaseType => typeof(TRecord); + + bool ICase.Decodes(Decoder decoder, Value value) => decoder.Is(value, Constructor); + + Value ICase.EncodeCase(Encoder encoder, object value) => Encode(encoder, (TRecord)value); + + object? ICase.DecodeCase(Decoder decoder, Value value) => Decode(decoder, value); + + /// + public override Value Encode(Encoder encoder, TRecord value) + { + ArgumentNullException.ThrowIfNull(encoder); + + // Counted before any field is read. + encoder.CheckCount(fields.Length, "constructor"); + var encoded = new Value[fields.Length]; + for (int i = 0; i < fields.Length; i++) + { + encoded[i] = fields[i].Encode(encoder, value); + } + + return encoder.Constructor(Constructor, encoded); + } + + /// + public override TRecord Decode(Decoder decoder, Value value) + { + ArgumentNullException.ThrowIfNull(decoder); + IReadOnlyList values = decoder.Fields(value, Constructor, fields.Length); + object?[] decoded = new object?[fields.Length]; + for (int i = 0; i < fields.Length; i++) + { + decoded[i] = fields[i].Decode(decoder, values[i]); + } + + try + { + return make(decoded); + } + catch (Exception error) when (Caller.Wraps(error)) + { + throw JacquardException.Binding(typeof(TRecord).Name + " refused the decoded fields: " + Identities.Describe(error), error); + } + } +} + +/// An enumeration as nullary constructors; made by . +/// The enumeration. +public sealed class EnumCodec : Codec, ICase + where TEnum : struct, Enum +{ + private readonly List<(TEnum Member, string Name)> members = new(); + + internal EnumCodec(IReadOnlyDictionary? explicitNames) + { + var seen = new HashSet(); + foreach (TEnum member in Enum.GetValues()) + { + if (!seen.Add(member)) + { + continue; + } + + string? name = explicitNames is null ? Enum.GetName(member) : explicitNames.TryGetValue(member, out string? given) ? given : null; + if (name is null) + { + throw new ArgumentException(typeof(TEnum).Name + "." + member + " has no constructor name", nameof(explicitNames)); + } + + members.Add((member, name)); + } + } + + Type ICase.CaseType => typeof(TEnum); + + bool ICase.Decodes(Decoder decoder, Value value) + { + foreach ((TEnum _, string name) in members) + { + if (decoder.Is(value, name)) + { + return true; + } + } + + return false; + } + + Value ICase.EncodeCase(Encoder encoder, object value) => Encode(encoder, (TEnum)value); + + object? ICase.DecodeCase(Decoder decoder, Value value) => Decode(decoder, value); + + /// + public override Value Encode(Encoder encoder, TEnum value) + { + ArgumentNullException.ThrowIfNull(encoder); + foreach ((TEnum member, string name) in members) + { + if (EqualityComparer.Default.Equals(member, value)) + { + return encoder.Constructor(name, Array.Empty()); + } + } + + throw encoder.Invalid(typeof(TEnum).Name + " has no constructor for " + value); + } + + /// + public override TEnum Decode(Decoder decoder, Value value) + { + ArgumentNullException.ThrowIfNull(decoder); + foreach ((TEnum member, string name) in members) + { + if (decoder.Is(value, name)) + { + decoder.Fields(value, name, 0); + return member; + } + } + + throw decoder.Mismatch(typeof(TEnum).Name); + } +} + +/// The constructors of one data type as a .NET type hierarchy; made by . +/// The common .NET type of the cases. +public sealed class UnionCodec : Codec, ICase, Builtins.IUnion +{ + private readonly ICase[] cases; + + /// + /// The distinct non-union cases, nested unions flattened in order, so + /// decoding asks each once. Computed on first use, when every lazy case of + /// a recursive hierarchy can be made, by one iterative walk over the whole + /// case graph; published once complete, so a race computes the same list + /// twice. + /// + private volatile ICase[]? leaves; + + internal UnionCodec(IUnionCase[] given) + { + cases = new ICase[given.Length]; + for (int i = 0; i < given.Length; i++) + { + if (given[i] is not ICase known) + { + throw new ArgumentException("a union case must be a record, enumeration, union, or lazy codec", nameof(given)); + } + + // A lazy case is checked when it is made, on first use. + if (known is not Builtins.ILazy && !typeof(TUnion).IsAssignableFrom(known.CaseType)) + { + throw new ArgumentException(known.CaseType.Name + " is not a " + typeof(TUnion).Name, nameof(given)); + } + + cases[i] = known; + } + } + + IReadOnlyList Builtins.IUnion.Cases => cases; + + Type ICase.CaseType => typeof(TUnion); + + /// Test seam: how many distinct cases this union decodes through. + internal int LeafCount => Leaves().Length; + + private ICase[] Leaves() + { + ICase[]? known = leaves; + if (known is null) + { + var flat = new List(); + var seen = new HashSet(ReferenceEqualityComparer.Instance) { this }; + var stack = new Stack>(); + stack.Push(((IEnumerable)cases).GetEnumerator()); + while (stack.Count > 0) + { + IEnumerator top = stack.Peek(); + if (!top.MoveNext()) + { + stack.Pop().Dispose(); + continue; + } + + object each = Builtins.Resolve(top.Current); + if (!seen.Add(each)) + { + continue; + } + + if (each is Builtins.IUnion nested) + { + stack.Push(nested.Cases.GetEnumerator()); + } + else if (each is ICase leaf && typeof(TUnion).IsAssignableFrom(leaf.CaseType)) + { + flat.Add(leaf); + } + else + { + throw new InvalidOperationException("a case of the " + typeof(TUnion).Name + " union is not a record or enumeration of it"); + } + } + + known = flat.ToArray(); + leaves = known; + } + + return known; + } + + bool ICase.Decodes(Decoder decoder, Value value) + { + foreach (ICase each in Leaves()) + { + if (each.Decodes(decoder, value)) + { + return true; + } + } + + return false; + } + + Value ICase.EncodeCase(Encoder encoder, object value) => Encode(encoder, (TUnion)value); + + object? ICase.DecodeCase(Decoder decoder, Value value) => Decode(decoder, value); + + /// + /// Encodes with the first flattened case, in the order decoding asks + /// them, whose type has: no call per nested + /// layer, and the cost is the number of distinct cases, not their nesting. + /// + public override Value Encode(Encoder encoder, TUnion value) + { + ArgumentNullException.ThrowIfNull(encoder); + object boxed = value!; + foreach (ICase each in Leaves()) + { + if (each.CaseType.IsInstanceOfType(boxed)) + { + return each.EncodeCase(encoder, boxed); + } + } + + throw encoder.Invalid(typeof(TUnion).Name + " has no case for " + boxed.GetType().Name); + } + + /// + public override TUnion Decode(Decoder decoder, Value value) + { + ArgumentNullException.ThrowIfNull(decoder); + foreach (ICase each in Leaves()) + { + if (each.Decodes(decoder, value)) + { + return (TUnion)each.DecodeCase(decoder, value)!; + } + } + + throw decoder.Mismatch(typeof(TUnion).Name); + } +} + +/// +/// A codec made on first use, for recursive types; made by +/// . The factory runs on one thread at a time +/// and its codec, once made, is kept and shared by every thread; a factory +/// that throws keeps nothing, so the next use runs it again. +/// +/// The .NET type. +public sealed class LazyCodec : Codec, ICase, Builtins.ILazy +{ + private readonly object gate = new(); + private readonly Func> make; + private volatile Codec? made; + private bool making; + + internal LazyCodec(Func> make) + { + this.make = make; + } + + object Builtins.ILazy.Target => made ?? Make(); + + private Codec Make() + { + lock (gate) + { + if (made is { } done) + { + return done; + } + + // The lock is reentrant: a factory that asks for its own codec would run again. + if (making) + { + throw new InvalidOperationException("a lazy codec's factory used the codec it makes"); + } + + making = true; + try + { + Codec codec = make() ?? throw new InvalidOperationException("a lazy codec's factory returned null"); + made = codec; + return codec; + } + finally + { + making = false; + } + } + } + + internal override bool AcceptsNull => Resolved().AcceptsNull; + + Type ICase.CaseType => Case().CaseType; + + private Codec Resolved() => (Codec)Builtins.Resolve(this); + + private ICase Case() => Builtins.Resolve(this) as ICase + ?? throw new InvalidOperationException("a lazy union case stands for a codec that is not a record, enumeration, or union"); + + bool ICase.Decodes(Decoder decoder, Value value) => Case().Decodes(decoder, value); + + Value ICase.EncodeCase(Encoder encoder, object value) => Case().EncodeCase(encoder, value); + + object? ICase.DecodeCase(Decoder decoder, Value value) => Case().DecodeCase(decoder, value); + + /// + public override Value Encode(Encoder encoder, T value) => Resolved().Encode(encoder, value); + + /// + public override T Decode(Decoder decoder, Value value) => Resolved().Decode(decoder, value); +} diff --git a/dotnet/src/JacquardHost/Sdk/Codec.cs b/dotnet/src/JacquardHost/Sdk/Codec.cs new file mode 100644 index 0000000..face148 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Codec.cs @@ -0,0 +1,328 @@ +using System; +using System.Collections.Generic; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// Maps one .NET type to Jacquard values and back. Codecs are explicit, +/// immutable, and thread-safe; they hold no program state, and encode and +/// decode against the of the program at hand. Build +/// them with the factories on , or derive from this class +/// for a custom mapping (use the 's methods so the +/// encoding is bounded as it goes, and report a value that does not decode +/// with ). +/// +/// The .NET type. +public abstract class Codec +{ + /// Encodes ; names its location for errors and bounds it by the selected limits. + /// An invalid argument. + public abstract Value Encode(Encoder encoder, T value); + + /// Decodes . + /// A binding failure when the value does not map to . + public abstract T Decode(Decoder decoder, Value value); + + /// as a function argument encoded with this codec. + public Argument Arg(T value) => Argument.Of(this, value); + + /// Whether null is a value of this codec (unit and nullable codecs); otherwise a null is refused before the codec runs. + internal virtual bool AcceptsNull => false; +} + +/// +/// A codec that can be a case of a union (): +/// a record, an enumeration, a union, or a lazy codec standing for one. +/// +public interface IUnionCase +{ +} + +/// What a union asks of each of its cases. +internal interface ICase : IUnionCase +{ + /// The .NET type this case encodes. + Type CaseType { get; } + + /// Whether is built by one of this case's constructors. + bool Decodes(Decoder decoder, Value value); + + Value EncodeCase(Encoder encoder, object value); + + object? DecodeCase(Decoder decoder, Value value); +} + +/// +/// One field of a record: its name (for error locations), its codec, and how +/// to read it from a record. Made by . +/// +/// The record type. +/// The field type. +public sealed class Field : Builtins.IField +{ + private readonly Func read; + + internal Field(string name, Codec codec, Func read) + { + Name = name; + Codec = codec; + this.read = read; + } + + /// The field's name, used in error locations. + public string Name { get; } + + /// The field's codec. + public Codec Codec { get; } + + /// Encodes the field of , reporting a reader that throws as an invalid argument with its exception kept. + internal Value Encode(Encoder encoder, TRecord record) + { + TField value; + try + { + value = read(record); + } + catch (Exception error) when (Caller.Wraps(error)) + { + throw encoder.Invalid("cannot read " + typeof(TRecord).Name + "." + Identities.Excerpt(Name) + ": " + Identities.Describe(error), error); + } + + return encoder.Field(Name, Codec, value); + } + + string Builtins.IField.Name => Name; + + Value Builtins.IField.Encode(Encoder encoder, TRecord record) => Encode(encoder, record); + + object? Builtins.IField.Decode(Decoder decoder, Value value) => Codec.Decode(decoder, value); +} + +/// +/// The codec factories: built-in types, lists, options, tuples, records, +/// enumerations, unions, lazy (recursive) codecs, raw protocol values, and +/// . Records are mapped by explicit field readers and +/// a constructor function rather than by reflection, so the library stays +/// clean under the trimming and ahead-of-time analyzers. +/// +public static class Codecs +{ + /// Int as a , Core's full 63-bit range; a outside it is refused. + public static Codec Int { get; } = new Builtins.IntCodec(); + + /// Int as an ; a result that does not fit an does not decode. + public static Codec Int32 { get; } = new Builtins.Int32Codec(); + + /// Real as the exact binary64 bits of a , including NaN payloads and signed zero. + public static Codec Real { get; } = new Builtins.RealCodec(); + + /// Text as a ; a lone surrogate is refused. + public static Codec Text { get; } = new Builtins.TextCodec(); + + /// Bool as a . + public static Codec Bool { get; } = new Builtins.BoolCodec(); + + /// Hash as a protocol . + public static Codec Hash { get; } = new Builtins.HashCodec(); + + /// Unit, the empty tuple, as . + public static Codec Unit { get; } = new Builtins.UnitCodec(); + + /// A raw protocol , passed through unchanged (within the same bounds). + public static Codec Value { get; } = new Builtins.ValueCodec(); + + /// Any value by public name, as . + public static Codec Dynamic { get; } = new Builtins.DynamicCodec(); + + /// List T as a read-only list; decoding returns a new array. + public static Codec> List(Codec item) => new Builtins.ListCodec(item ?? throw new ArgumentNullException(nameof(item))); + + /// Option T as an , which nests. + public static Codec> Option(Codec inner) => new Builtins.OptionCodec(inner ?? throw new ArgumentNullException(nameof(inner))); + + /// Option T as a nullable reference: null is None. Nested options need . + public static Codec Nullable(Codec inner) + where T : class => new Builtins.NullableCodec(inner ?? throw new ArgumentNullException(nameof(inner))); + + /// Option T as a nullable value type: null is None. + public static Codec NullableValue(Codec inner) + where T : struct => new Builtins.NullableValueCodec(inner ?? throw new ArgumentNullException(nameof(inner))); + + /// A two-item tuple. + public static Codec<(T1, T2)> Tuple(Codec first, Codec second) => + new Builtins.TupleCodec(first ?? throw new ArgumentNullException(nameof(first)), second ?? throw new ArgumentNullException(nameof(second))); + + /// A three-item tuple. + public static Codec<(T1, T2, T3)> Tuple(Codec first, Codec second, Codec third) => + new Builtins.TupleCodec( + first ?? throw new ArgumentNullException(nameof(first)), + second ?? throw new ArgumentNullException(nameof(second)), + third ?? throw new ArgumentNullException(nameof(third))); + + /// A four-item tuple. + public static Codec<(T1, T2, T3, T4)> Tuple(Codec first, Codec second, Codec third, Codec fourth) => + new Builtins.TupleCodec( + first ?? throw new ArgumentNullException(nameof(first)), + second ?? throw new ArgumentNullException(nameof(second)), + third ?? throw new ArgumentNullException(nameof(third)), + fourth ?? throw new ArgumentNullException(nameof(fourth))); + + /// A record field: its name (for error locations), its codec, and how to read it. + public static Field Field(string name, Codec codec, Func read) => + new(name ?? throw new ArgumentNullException(nameof(name)), codec ?? throw new ArgumentNullException(nameof(codec)), read ?? throw new ArgumentNullException(nameof(read))); + + /// + /// A record (any .NET type) as one Jacquard constructor with no fields, + /// such as UnknownSku. The constructor is named by public name: + /// display spelling (UnknownSku) or store spelling (unknown-sku). + /// + public static RecordCodec Record(string constructor, Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Array.Empty>(), _ => make()); + } + + /// A record as one constructor with one field; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1), a => make((T1)a[0]!)); + } + + /// A record as one constructor with two fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2), a => make((T1)a[0]!, (T2)a[1]!)); + } + + /// A record as one constructor with three fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Field f3, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2, f3), a => make((T1)a[0]!, (T2)a[1]!, (T3)a[2]!)); + } + + /// A record as one constructor with four fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Field f3, + Field f4, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2, f3, f4), a => make((T1)a[0]!, (T2)a[1]!, (T3)a[2]!, (T4)a[3]!)); + } + + /// A record as one constructor with five fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Field f3, + Field f4, + Field f5, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2, f3, f4, f5), a => make((T1)a[0]!, (T2)a[1]!, (T3)a[2]!, (T4)a[3]!, (T5)a[4]!)); + } + + /// A record as one constructor with six fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Field f3, + Field f4, + Field f5, + Field f6, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2, f3, f4, f5, f6), a => make((T1)a[0]!, (T2)a[1]!, (T3)a[2]!, (T4)a[3]!, (T5)a[4]!, (T6)a[5]!)); + } + + /// A record as one constructor with seven fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Field f3, + Field f4, + Field f5, + Field f6, + Field f7, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2, f3, f4, f5, f6, f7), a => make((T1)a[0]!, (T2)a[1]!, (T3)a[2]!, (T4)a[3]!, (T5)a[4]!, (T6)a[5]!, (T7)a[6]!)); + } + + /// A record as one constructor with eight fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Field f3, + Field f4, + Field f5, + Field f6, + Field f7, + Field f8, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2, f3, f4, f5, f6, f7, f8), a => make((T1)a[0]!, (T2)a[1]!, (T3)a[2]!, (T4)a[3]!, (T5)a[4]!, (T6)a[5]!, (T7)a[6]!, (T8)a[7]!)); + } + + /// + /// An enumeration as nullary constructors named by convention: a member + /// such as PageNow is the constructor PageNow (store + /// spelling page-now). + /// + public static EnumCodec Enum() + where TEnum : struct, Enum => new(null); + + /// An enumeration with an explicit constructor name for every member. + /// A member has no name. + public static EnumCodec Enum(IReadOnlyDictionary constructors) + where TEnum : struct, Enum => new(constructors ?? throw new ArgumentNullException(nameof(constructors))); + + /// + /// The constructors of one data type as a .NET type hierarchy: each case + /// is a record, enumeration, or union codec (or a lazy one standing for + /// one) whose .NET type derives from . + /// Encoding takes the first case, in order with nested unions flattened, + /// whose type the value has; decoding takes the first whose constructor + /// built the value. + /// + /// A case is none of these. + public static UnionCodec Union(params IUnionCase[] cases) => new(cases ?? throw new ArgumentNullException(nameof(cases))); + + /// + /// A codec made on first use by , for recursive + /// types: a field of a type can refer to the type's own codec through a + /// lazy one. The codec is made once, thread-safely. + /// + public static LazyCodec Lazy(Func> make) => new(make ?? throw new ArgumentNullException(nameof(make))); + + /// A custom codec from two functions. + public static Codec Create(Func encode, Func decode) => + new Builtins.CustomCodec(encode ?? throw new ArgumentNullException(nameof(encode)), decode ?? throw new ArgumentNullException(nameof(decode))); +} diff --git a/dotnet/src/JacquardHost/Sdk/Core.cs b/dotnet/src/JacquardHost/Sdk/Core.cs new file mode 100644 index 0000000..8f4b84f --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Core.cs @@ -0,0 +1,987 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using JacquardLang.Host.Carrier; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// Bounded runs of Core's public commands, digests of prepared files, and the cache's trust boundary. +internal static class Core +{ + /// Each Core command must finish within this time. + internal static readonly TimeSpan CommandTimeout = TimeSpan.FromSeconds(30); + + internal const int StdoutLimit = 8 << 20; + internal const int StderrLimit = 4096; + + /// Why a preparation stopped early. + internal const string Aborted = "preparation was cancelled"; + + /// The most bytes of the Core executable that are fingerprinted. + internal const long ExecutableLimit = 512L << 20; + + /// The most bytes reads under one root. + internal const long TreeLimit = 256L << 20; + + /// The most entries (files and directories, empty ones included) visits under one root. + internal const int TreeEntries = 100_000; + + /// The longest path, in chars, the SDK passes to the file system (Linux's PATH_MAX). + internal const int PathLimit = 4096; + + private const int Chunk = 1 << 16; + + /// Test seam: the entry cap and the failure cleanup apply. + internal static volatile int TreeEntryCap = TreeEntries; + + /// Test seam: runs after each chunk reads. Null outside tests. + internal static volatile Action? DigestChunkHook; + + /// Test seam: runs after digests each file, with its path. Null outside tests. + internal static volatile Action? TreeFileHook; + + /// What does about who may write what it walks. + internal enum Access + { + /// Nothing: a Core installation's prelude. + Any, + + /// Removes every group and other permission, as each new cache entry's files are made owner-only. + Restrict, + + /// Refuses the root or any entry this user does not own or that its group or others may write. + OwnerOnly, + } + + /// + /// Runs one Core subcommand with the prelude set, returning its stdout. + /// The command is bounded by , its output by the + /// limits above, and it stops at once when the token is cancelled. + /// + /// A preparation failure. + internal static string Run(string binary, string prelude, TimeSpan timeout, CancellationToken cancellationToken, params string[] arguments) + { + string name = arguments.Length == 0 ? string.Empty : arguments[0]; + var command = new List { binary }; + command.AddRange(arguments); + CoreCommand.Output output; + try + { + output = CoreCommand.Run( + command, + new Dictionary(StringComparer.Ordinal) { ["JACQUARD_PRELUDE"] = prelude }, + timeout, + StdoutLimit, + StderrLimit, + cancellationToken); + } + catch (CarrierException error) + { + string detail = error.IsTimeout + ? "no exit within " + (timeout.TotalMilliseconds % 1000 == 0 + ? ((long)timeout.TotalSeconds).ToString(CultureInfo.InvariantCulture) + " seconds" + : ((long)timeout.TotalMilliseconds).ToString(CultureInfo.InvariantCulture) + " ms") + : error.IsAborted ? Aborted : error.Message; + throw JacquardException.Preparation("Core " + name + " failed: " + detail); + } + + if (output.ExitStatus != 0) + { + string detail = Encoding.UTF8.GetString(output.Stderr).Trim(); + throw JacquardException.Preparation("Core " + name + " failed: " + + (detail.Length == 0 ? "exit status " + output.ExitStatus.ToString(CultureInfo.InvariantCulture) : detail)); + } + + if (output.StdoutOverflow) + { + throw JacquardException.Preparation("Core " + name + " output exceeds 8 MiB"); + } + + return Utf8(output.Stdout) ?? throw JacquardException.Preparation("Core " + name + " output is not UTF-8"); + } + + private static readonly UTF8Encoding StrictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + + /// The bytes as strict UTF-8 text, or null when they are not. + internal static string? Utf8(byte[] bytes) + { + try + { + return StrictUtf8.GetString(bytes); + } + catch (DecoderFallbackException) + { + return null; + } + } + + /// The SHA-256 digest of bytes, in lowercase hexadecimal. + internal static string Digest(ReadOnlySpan data) => Convert.ToHexString(SHA256.HashData(data)).ToLowerInvariant(); + + /// + /// The SHA-256 digest of the Core executable's bytes, read in pieces. A + /// file larger than is refused from its size + /// before it is read, and the limit is enforced again while it is read. + /// Before each piece it observes the token. + /// is the file's status before the read, which must be unchanged after + /// it, so the digest describes the file with that identity. + /// + /// A preparation failure. + internal static string FileDigest(string path, CancellationToken cancellationToken, out Unix.Status identity) + { + JacquardException over = JacquardException.Preparation("Core executable " + path + " exceeds " + + (ExecutableLimit >> 20).ToString(CultureInfo.InvariantCulture) + " MiB"); + Unix.Status status = StatOrFail(path, follow: true, "cannot read "); + if (status.Kind != Unix.Kind.Regular) + { + throw JacquardException.Preparation("cannot read " + path + ": not a regular file"); + } + + if (status.Size > ExecutableLimit) + { + throw over; + } + + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + byte[] buffer = new byte[Chunk]; + long read = 0; + try + { + using FileStream stream = OpenRead(path); + while (true) + { + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Aborted); + } + + int n = stream.Read(buffer, 0, buffer.Length); + if (n <= 0) + { + break; + } + + read += n; + if (read > ExecutableLimit) + { + throw over; + } + + hash.AppendData(buffer, 0, n); + DigestChunkHook?.Invoke(); + } + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw JacquardException.Preparation("cannot read " + path + ": " + error.Message); + } + + if (!StatOrFail(path, follow: true, "cannot read ").SameFile(status)) + { + throw Changed("Core executable", path); + } + + identity = status; + return Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); + } + + /// One file or directory whose identity a preparation keeps until Core's commands have finished. + internal readonly record struct Pin(string What, string Path, bool Follow, Unix.Status Status); + + /// The identity of as text, for a key: device, inode, size, and both times. + internal static string IdentityKey(Unix.Status status) => string.Join( + ":", + status.Device.ToString(CultureInfo.InvariantCulture), + status.Inode.ToString(CultureInfo.InvariantCulture), + status.Size.ToString(CultureInfo.InvariantCulture), + status.Modified.ToString(CultureInfo.InvariantCulture), + status.Changed.ToString(CultureInfo.InvariantCulture)); + + /// + /// Refuses the preparation unless every pinned file and directory still + /// has the identity it had when it was fingerprinted (see + /// ). It reads one status per pin, at + /// most plus one, and observes the token before + /// each. + /// + /// A preparation failure. + internal static void Unchanged(IReadOnlyList pins, CancellationToken cancellationToken) + { + foreach (Pin pin in pins) + { + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Aborted); + } + + Unix.Status now; + try + { + now = Unix.Stat(pin.Path, pin.Follow); + } + catch (IOException) + { + throw Changed(pin.What, pin.Path); + } + + if (!now.SameFile(pin.Status)) + { + throw Changed(pin.What, pin.Path); + } + } + } + + private static JacquardException Changed(string what, string path) => + JacquardException.Preparation(what + " " + path + " changed during preparation; prepare again once it no longer changes"); + + /// Options that write a file, created owner-only (read and write for its owner) where the file system has POSIX permissions. + internal static FileStreamOptions OwnerOnly(FileMode mode) + { + var options = new FileStreamOptions { Mode = mode, Access = FileAccess.Write }; + if (!OperatingSystem.IsWindows()) + { + options.UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite; + } + + return options; + } + + private static FileStream OpenRead(string path) => + new(path, new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Share = FileShare.ReadWrite, BufferSize = 0 }); + + /// The status of , or a preparation failure starting with . + internal static Unix.Status StatOrFail(string path, bool follow, string context) + { + Unix.Status status; + try + { + status = Unix.Stat(path, follow); + } + catch (IOException error) + { + throw JacquardException.Preparation(context + path + ": " + error.Message); + } + + if (status.Kind == Unix.Kind.Missing) + { + throw JacquardException.Preparation(context + path + ": no such file or directory"); + } + + return status; + } + + /// + /// Digests every regular file under , keyed by + /// /-separated relative path, refusing other entry kinds (symbolic + /// links, FIFOs, and devices included, so no read can block on one), more + /// than entries, and more than + /// bytes in all, which is checked against each + /// file's size before it is read and again while it is read. The walk + /// streams each directory and keeps an explicit stack, so nothing beyond + /// the entry cap is held and no directory depth can exhaust the call + /// stack; before each entry and each piece of a file it observes the + /// token. says whether entries are also checked + /// for owner-only access, or made owner-only. When + /// is given, the root's status and each entry's + /// status, read before the entry is digested, are added to it. + /// + /// A preparation failure. + internal static SortedDictionary Tree(string root, Access access, CancellationToken cancellationToken, List? pins = null) + { + var files = new SortedDictionary(Json.CodePointOrder); + Check(root, access); + pins?.Add(new Pin("prelude", root, Follow: true, StatOrFail(root, follow: true, "cannot read "))); + long total = 0; + int entries = 0; + int cap = TreeEntryCap; + var stack = new Stack>(); + try + { + stack.Push(Directory.EnumerateFileSystemEntries(root).GetEnumerator()); + while (stack.Count > 0) + { + IEnumerator top = stack.Peek(); + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Aborted); + } + + if (!top.MoveNext()) + { + stack.Pop().Dispose(); + continue; + } + + string path = top.Current; + if (++entries > cap) + { + throw JacquardException.Preparation("more than " + cap.ToString(CultureInfo.InvariantCulture) + " entries under " + root); + } + + string relative = Path.GetRelativePath(root, path).Replace(Path.DirectorySeparatorChar, '/'); + Unix.Status status = StatOrFail(path, follow: false, "cannot read "); + pins?.Add(new Pin("prelude entry", path, Follow: false, status)); + if (status.Kind == Unix.Kind.Directory) + { + Check(path, access); + stack.Push(Directory.EnumerateFileSystemEntries(path).GetEnumerator()); + } + else if (status.Kind == Unix.Kind.Regular) + { + Check(path, access); + JacquardException over = JacquardException.Preparation("files under " + root + " exceed " + + (TreeLimit >> 20).ToString(CultureInfo.InvariantCulture) + " MiB"); + if (status.Size > TreeLimit - total) + { + throw over; + } + + files[relative] = BoundedDigest(path, TreeLimit - total, ref total, over, cancellationToken); + TreeFileHook?.Invoke(path); + } + else + { + throw JacquardException.Preparation("unsupported filesystem entry in prepared artifacts: " + relative); + } + } + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw JacquardException.Preparation("cannot read " + root + ": " + error.Message); + } + finally + { + while (stack.Count > 0) + { + stack.Pop().Dispose(); + } + } + + return files; + } + + /// The digest of at most bytes of a file, adding what it read to . + private static string BoundedDigest(string path, long room, ref long total, JacquardException over, CancellationToken cancellationToken) + { + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + byte[] buffer = new byte[Chunk]; + long read = 0; + using (FileStream stream = OpenRead(path)) + { + int n; + while ((n = stream.Read(buffer, 0, buffer.Length)) > 0) + { + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Aborted); + } + + read += n; + if (read > room) + { + throw over; + } + + hash.AppendData(buffer, 0, n); + } + } + + total += read; + return Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); + } + + /// Canonical JSON for a string map: keys in code-point order. + internal static string Canonical(IEnumerable> map) + { + var fields = new List>(); + foreach (KeyValuePair pair in map) + { + fields.Add(new(pair.Key, Json.Of(pair.Value))); + } + + return JsonCodec.Stringify(Json.Object(fields), 4); + } + + /// The digest of a string map's canonical JSON. + internal static string MapDigest(IEnumerable> map) => Digest(Encoding.UTF8.GetBytes(Canonical(map))); + + /// + /// , refused when it is empty, holds a NUL (no path + /// does), or is longer than any path, as given or made absolute, before + /// the file system or any message sees it whole. + /// + /// A preparation failure quoting an excerpt. + internal static string CheckedPath(string? path, string what) + { + if (path is null) + { + throw new ArgumentNullException(nameof(path), what + " is null"); + } + + if (path.Length == 0 || path.Contains('\0', StringComparison.Ordinal)) + { + throw JacquardException.Preparation(what + " " + Identities.Excerpt(path) + " is not a path"); + } + + // As given first, so a huge path is not copied again; then as the + // file system will see it, since a relative path grows. + if (path.Length > PathLimit || AbsoluteLength(path) > PathLimit) + { + throw JacquardException.Preparation(what + " " + Identities.Excerpt(path) + " is longer than any path"); + } + + return path; + } + + /// The length of made absolute, or one past the limit when it cannot be. + internal static int AbsoluteLength(string path) + { + try + { + return Path.GetFullPath(path).Length; + } + catch (Exception error) when (error is ArgumentException or NotSupportedException or PathTooLongException or System.Security.SecurityException) + { + return PathLimit + 1; + } + } + + /// Finds an executable the way a shell would, then resolves a final symbolic link. + /// A preparation failure for a missing executable. + internal static string Executable(string binary) + { + JacquardException Missing() => JacquardException.Preparation("cannot find Core executable " + Identities.Excerpt(binary) + + "; configure JACQUARD_BIN using the setup guide"); + + // No path is this long (PATH_MAX is 4096 on Linux), and none holds a + // NUL, so such a name is refused before it is used. + if (binary.Length > PathLimit || binary.Contains('\0', StringComparison.Ordinal)) + { + throw Missing(); + } + + string? candidate = null; + if (binary.Contains('/', StringComparison.Ordinal)) + { + candidate = binary; + } + else + { + // Like Go's exec.LookPath, relative entries (including empty ones, + // which mean the current directory) are not searched. + foreach (string entry in (Environment.GetEnvironmentVariable("PATH") ?? string.Empty).Split(':')) + { + if (Path.IsPathRooted(entry) && IsExecutable(Path.Combine(entry, binary))) + { + candidate = Path.Combine(entry, binary); + break; + } + } + } + + if (candidate is null) + { + throw Missing(); + } + + if (!IsExecutable(candidate)) + { + if (IsRegular(candidate)) + { + throw JacquardException.Preparation("Core executable " + Identities.Excerpt(binary) + + " may not be executed by the current user; give it execute permission or configure JACQUARD_BIN"); + } + + throw Missing(); + } + + try + { + FileSystemInfo? target = new FileInfo(candidate).ResolveLinkTarget(returnFinalTarget: true); + return Path.GetFullPath(target?.FullName ?? candidate); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException or ArgumentException) + { + throw JacquardException.Preparation("cannot resolve Core executable " + Identities.Excerpt(binary) + ": " + error.Message); + } + } + + /// Whether is a regular file this process may execute (access(X_OK)), not merely one with some execute bit. + private static bool IsExecutable(string path) => IsRegular(path) && Unix.MayExecute(path); + + private static bool IsRegular(string path) + { + if (path.Length > PathLimit) + { + return false; + } + + try + { + return Unix.Stat(path, follow: true).Kind == Unix.Kind.Regular; + } + catch (IOException) + { + return false; + } + } + + /// + /// Applies to , which is + /// not followed if it is a link: refuses it unless this user owns it and + /// neither its group nor others may write it, or removes every group and + /// other permission. Nothing inside a cache entry can resist someone who + /// may rewrite it, so the guarantee is that only the cache's owner can. + /// + /// The status that was checked; empty for or where there are no POSIX permissions. + /// A preparation failure. + internal static Unix.Status Check(string path, Access access) + { + if (access == Access.Any || !Unix.Supported) + { + return default; + } + + Unix.Status status = StatOrFail(path, follow: false, "cannot read "); + if (access == Access.Restrict) + { + if ((status.Mode & 0x3F) != 0) + { + try + { + if (!OperatingSystem.IsWindows()) + { + File.SetUnixFileMode(path, (UnixFileMode)(status.Mode & 0x1C0)); + } + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw JacquardException.Preparation("cannot make " + path + " owner-only: " + error.Message); + } + } + + return status; + } + + if (status.Owner != Unix.EffectiveUser()) + { + throw JacquardException.Preparation("cache path " + path + " is not owned by the current user; only its owner may write the cache"); + } + + if ((status.Mode & 0x12) != 0) + { + throw JacquardException.Preparation("cache path " + path + " is group- or world-writable; only its owner may write the cache" + + " (make it owner-only, for example chmod " + (status.Kind == Unix.Kind.Directory ? "700 " : "600 ") + path + ")"); + } + + return status; + } + + /// + /// Refuses the cache root , a real path with no + /// links in it, unless every directory above it, up to /, is safe + /// (see ). Otherwise another user could + /// rename the root away and put another directory at its path at any + /// time, and every later use of the path, a prepared program's workers + /// included, would use it. + /// + /// A preparation failure naming every unsafe directory, or cancelled. + internal static void CheckAncestors(string root, PasswordListing listing, CancellationToken cancellationToken) + { + if (Path.GetDirectoryName(root) is { } parent && UnsafeDirectories(parent, "cache directory " + root, listing, cancellationToken) is { } problem) + { + throw JacquardException.Preparation(problem); + } + } + + /// + /// Refuses , a real path with no links in it, + /// and the directories above it before any missing level below it is + /// made, so a refused preparation makes nothing. + /// + /// A preparation failure naming every unsafe directory, or cancelled. + internal static void CheckDirectories(string directory, string root, PasswordListing listing, CancellationToken cancellationToken) + { + if (UnsafeDirectories(directory, "cache directory " + root, listing, cancellationToken) is { } problem) + { + throw JacquardException.Preparation(problem); + } + } + + /// + /// The password database verdict of the private-group rule + /// () within one preparation or one + /// save: the listing runs at most once, for the account, user id, and + /// group of the first request; a later request for another account, + /// user id, or group is refused (not private) without listing again. + /// The verdict is not kept after that call. Not thread-safe; one call + /// uses it. + /// + internal sealed class PasswordListing + { + private string? name; + private uint uid; + private uint gid; + private bool verdict; + + /// + /// The verdict for the account, listing the password database on the + /// first request only; false for a request naming another account, + /// user id, or group than the first, and after a first listing that + /// was cancelled. + /// + /// The token was cancelled while getent ran. + internal bool OnlyUserHasPrimaryGroup(string name, uint uid, uint gid, CancellationToken cancellationToken) + { + if (this.name is null) + { + // Recorded before listing, so no later request lists again, even after a cancelled first one. + (this.name, this.uid, this.gid) = (name, uid, gid); + verdict = Core.OnlyUserHasPrimaryGroup(name, uid, gid, cancellationToken); + return verdict; + } + + return verdict && string.Equals(this.name, name, StringComparison.Ordinal) && this.uid == uid && this.gid == gid; + } + } + + /// + /// Test seam: answers the password database lookups of the private-group + /// rule in place of . Null outside tests. + /// + internal static volatile Func? UserHook; + + /// + /// Test seam: answers the group database lookups of the private-group + /// rule in place of . Null outside tests. + /// + internal static volatile Func? GroupHook; + + /// + /// Test seam: runs in place of getent passwd for the private-group + /// rule, with the caller's token, returning what the command printed and + /// its exit status, or throwing as does. + /// Null outside tests. + /// + internal static volatile Func? GetentHook; + + /// Test seam: where getent is looked for, in order; the first present is used. + internal static volatile string[] GetentPaths = { "/usr/bin/getent", "/bin/getent" }; + + /// + /// Test seam: reads the status (not following a final link) of the + /// getent and setsid files and the directories above them in place of + /// , given the real path of the + /// file being checked (getent's or setsid's) and the path read, so a + /// directory above both can be altered for one of them. Null outside + /// tests. + /// + internal static volatile Func? GetentStatHook; + + /// Test seam: the arguments getent runs with. + internal static volatile string[] GetentArguments = { "passwd" }; + + /// How long getent passwd may run, in milliseconds. + internal const int GetentTimeoutMs = 5000; + + /// Test seam: the time limit applied, in milliseconds. + internal static volatile int GetentTimeoutCap = GetentTimeoutMs; + + /// The most stdout of getent passwd read; more is refused. + internal const int GetentStdoutLimit = 16 << 20; + + /// The most password database entries the private-group rule reads; more is refused. + internal const int PasswdEntries = 100_000; + + /// The most unsafe directories one refusal names; the rest are counted. + internal const int UnsafeListed = 16; + + /// + /// Checks , a real path with no links in it, and + /// every directory above it up to /, with lstat: each must + /// be a directory owned by this user or by root, and neither its group + /// nor others may write it, unless it has the sticky bit (as OpenSSH + /// checks the directories above its configuration) or the group is this + /// user's private group. In a sticky directory only the owner of an entry + /// (or the directory's owner, or root) may rename or remove it, and the + /// next directory down is checked in turn. The private group is Debian + /// OpenSSH's rule (no account but the user has the group as its primary + /// group, and the group lists no member but the user) with stricter + /// conditions: the directory's group is the user's primary group, that + /// group's name is the user's name, it lists no members or only the + /// user, the password database as enumerated lists the user with that + /// primary group and no other account with it (see + /// ), and the directory has no + /// access ACL (whose mask, on Linux, is what the group bits show). Every + /// unsafe directory is named, nearest first, up to + /// . The enumeration runs at most once per + /// , which one preparation or save shares + /// between its checks, and its verdict is not kept after it. + /// + /// The nearest directory to check. + /// What the directories hold, which starts the message. + /// The password database verdict of this preparation or save. + /// Stops the password database enumeration. + /// The refusal naming every unsafe directory, or null when all are safe or there are no POSIX permissions. + /// A directory's status cannot be read, or the token was cancelled during the enumeration. + internal static string? UnsafeDirectories(string first, string subject, PasswordListing listing, CancellationToken cancellationToken) + { + if (!Unix.Supported) + { + return null; + } + + uint user = Unix.EffectiveUser(); + bool looked = false; + Unix.Account? account = null; + Unix.GroupEntry? group = null; + var unsafeOnes = new List(); + int count = 0; + for (string? directory = first; directory is not null; directory = Path.GetDirectoryName(directory)) + { + Unix.Status status = StatOrFail(directory, follow: false, "cannot read "); + string? reason = null; + if (status.Kind != Unix.Kind.Directory) + { + reason = "not a directory"; + } + else if (status.Owner != user && status.Owner != 0) + { + reason = "owned by another user"; + } + else if ((status.Mode & 0x200) == 0) + { + // 0o002: other write; 0o020: group write; 0o1000: sticky. + if ((status.Mode & 0x2) != 0) + { + reason = "writable by others"; + } + else if ((status.Mode & 0x10) != 0) + { + if (!looked) + { + looked = true; + account = UserHook is { } users ? users(user) : Unix.User(user); + if (account is { } known) + { + group = GroupHook is { } groups ? groups(known.Group) : Unix.GroupOf(known.Group); + } + } + + if (!PrivateGroup(status, account, group) || !listing.OnlyUserHasPrimaryGroup(account!.Value.Name, user, status.Group, cancellationToken)) + { + reason = "writable by its group, which is not your private group"; + } + else if (Unix.HasAccessAcl(directory)) + { + // On Linux an ACL's mask shows in the group bits, and its entries may grant others write. + reason = "writable by its group, with an access control list"; + } + } + } + + if (reason is not null) + { + if (++count <= UnsafeListed) + { + unsafeOnes.Add(directory + " (" + reason + ")"); + } + } + } + + if (count == 0) + { + return null; + } + + string more = count > UnsafeListed ? "; and " + (count - UnsafeListed).ToString(CultureInfo.InvariantCulture) + " more" : string.Empty; + + // The remedy comes before the list, so a message cut at its limit keeps it. + return subject + " could be replaced by another user through the directories above it." + + " Make each of these writable only by you (for example chmod go-w DIR), or use a location under directories only you or root may write: " + + string.Join("; ", unsafeOnes) + more; + } + + /// + /// The group-database half of the private-group rule for a + /// group-writable ; the password database half + /// is . + /// + private static bool PrivateGroup(Unix.Status status, Unix.Account? account, Unix.GroupEntry? group) => + account is { } user && group is not null + && status.Group == user.Group + && string.Equals(group.Name, user.Name, StringComparison.Ordinal) + && (group.Members.Count == 0 || (group.Members.Count == 1 && string.Equals(group.Members[0], user.Name, StringComparison.Ordinal))); + + /// + /// Whether the password database, as getent passwd lists it in a + /// separate process, lists the account with user + /// id and primary group , + /// and no account of another user id with that primary group + /// (gr_mem omits primary-group members, so this is what shows + /// them), as Debian OpenSSH's secure_permissions scans with + /// getpwent. Listing it in another process shares no enumeration + /// position with code in this one. getent is the first of + /// /usr/bin/getent and /bin/getent present; it and every + /// directory above its real path must be root's and not writable by + /// group or others, and so must setsid where the command runs + /// through it (); each runs by its + /// checked real path. It runs through + /// (bounded by , its stdout by + /// , and stopped by the token). False, so + /// the directory is refused, when there is no such getent or + /// setsid (or not on Linux), when it times out, exits nonzero, or prints more than the + /// limit, invalid UTF-8, more than entries, + /// or a line that is not seven colon-separated fields with a nonempty + /// name and decimal user and group ids, or ends without a newline, when + /// a name is listed twice, or when the account is missing or listed with + /// other ids. Accounts a source does not enumerate cannot be seen. + /// + /// The token was cancelled while getent ran. + internal static bool OnlyUserHasPrimaryGroup(string name, uint uid, uint gid, CancellationToken cancellationToken) + { + CoreCommand.Output output; + try + { + if (GetentHook is { } hook) + { + output = hook(cancellationToken); + } + else + { + if (!OperatingSystem.IsLinux() || RootOnlyFile(GetentPaths) is not { } getent) + { + return false; + } + + // getent runs as setsid -- getent where groups are used, so setsid must be root's alone too. + string? setsid = null; + if (CoreCommand.Grouped && (setsid = RootOnlyFile(new[] { CoreCommand.Setsid! })) is null) + { + return false; + } + + var command = new List { getent }; + command.AddRange(GetentArguments); + output = CoreCommand.Run(command, null, TimeSpan.FromMilliseconds(GetentTimeoutCap), GetentStdoutLimit, StderrLimit, cancellationToken, setsid); + } + } + catch (CarrierException error) + { + if (error.IsAborted) + { + throw JacquardException.Preparation(Aborted); + } + + return false; + } + catch (IOException) + { + return false; + } + + return output.ExitStatus == 0 && !output.StdoutOverflow && Utf8(output.Stdout) is { } text && SoleAccount(text, name, uid, gid); + } + + /// + /// The real path of the first of present, + /// when it is a regular file and it and every directory above it are + /// root's and not writable by group or others; null otherwise, without + /// trying a later candidate. + /// + /// A status cannot be read. + private static string? RootOnlyFile(string[] candidates) + { + foreach (string candidate in candidates) + { + if (Unix.Stat(candidate, follow: true).Kind == Unix.Kind.Missing) + { + continue; + } + + string real = Unix.RealPath(candidate); + if (Lstat(real, real) is not { Kind: Unix.Kind.Regular } file || !RootOnly(file)) + { + return null; + } + + for (string? directory = Path.GetDirectoryName(real); directory is not null; directory = Path.GetDirectoryName(directory)) + { + if (Lstat(real, directory) is not { Kind: Unix.Kind.Directory } level || !RootOnly(level)) + { + return null; + } + } + + return real; + } + + return null; + + static Unix.Status Lstat(string file, string path) => GetentStatHook is { } hook ? hook(file, path) : Unix.Stat(path, follow: false); + } + + /// Whether is root's and neither its group nor others may write it. + internal static bool RootOnly(Unix.Status status) => status.Owner == 0 && (status.Mode & 0x12) == 0; + + /// + /// Parses getent passwd's output strictly (see + /// ); whether it holds. + /// + internal static bool SoleAccount(string text, string name, uint uid, uint gid) + { + if (text.Length == 0 || text[^1] != '\n') + { + return false; + } + + // At most the stdout limit of names is kept. + var names = new HashSet(StringComparer.Ordinal); + bool listed = false; + int read = 0; + for (int start = 0; start < text.Length;) + { + int end = text.IndexOf('\n', start); + ReadOnlySpan line = text.AsSpan(start, end - start); + start = end + 1; + if (++read > PasswdEntries || line.Count(':') != 6) + { + return false; + } + + string[] fields = line.ToString().Split(':'); + if (fields[0].Length == 0 || Id(fields[2]) is not { } entryUid || Id(fields[3]) is not { } entryGid || !names.Add(fields[0])) + { + return false; + } + + if (string.Equals(fields[0], name, StringComparison.Ordinal)) + { + if (entryUid != uid || entryGid != gid) + { + return false; + } + + listed = true; + } + else if (entryGid == gid && entryUid != uid) + { + return false; + } + } + + return listed; + } + + /// A decimal id with no sign, no leading zero, and no other character; null otherwise. + private static uint? Id(string text) => + text.Length > 0 && (text.Length == 1 || text[0] != '0') + && uint.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out uint id) + ? id + : null; +} diff --git a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs new file mode 100644 index 0000000..5757a0d --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs @@ -0,0 +1,894 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.IO.Pipes; +using System.Threading; +using JacquardLang.Host.Carrier; + +namespace JacquardLang.Host.Sdk; + +/// +/// Runs one short command to completion, bounded in time and output, the way +/// the SDK runs Core's --version, check, hash, and +/// run commands. +/// +/// +/// +/// As in the adapter's worker carrier, the command's stdout and stderr are +/// read only after poll(2) reports them ready, on the caller's thread, +/// in waits of 1 to 5 ms that recheck the timeout and the +/// ; .NET's own stream reads, which cannot be +/// interrupted, are never left waiting. Its stdin is a pipe the host closes +/// at once, so it reads end of file. +/// +/// +/// On Linux, when a setsid is installed at /usr/bin/setsid or +/// /bin/setsid (util-linux's, or a compatible one such as BusyBox's), +/// the command runs as setsid -- <command>. .NET's child is never +/// a process group leader, so setsid starts a new session and execs +/// the command without forking: the command keeps the child's pid and leads +/// a new process group whose id is that pid, shared by every descendant that +/// does not leave it. The host kills that group with kill(-pgid, +/// SIGKILL) when the command exits, and on timeout or cancellation. +/// After the leader exits and .NET has reaped it, a live process holding the +/// leader's pid means that pid was reused, so the old group is empty and +/// nothing is killed; the window between that check and the kill is the same +/// as Rust's kill(-pgid) after a full pid wraparound. When +/// setsid cannot exec the command (exit status 126 or 127 with its own +/// "setsid: " message), the run is reported as a start failure. A +/// descendant that leaves the group (calls setsid itself) is neither +/// detected nor killed; the host closes its ends of the output after the +/// exit, so its next write fails. +/// +/// +/// Elsewhere (Linux without setsid), the command's descendants are +/// recorded from /proc about every 10 ms, each by pid and start time, +/// and every recorded descendant still alive with that start time is killed +/// when the command returns. Each record reads at most +/// process entries and +/// bytes in all (at most of each stat file, +/// and never more than the bytes left), keeps at most +/// descendants in all, and observes the deadline +/// and the token before each entry, ending at once on a stop. On a timeout +/// or cancellation (a stop), the command is first sent SIGSTOP +/// (guarded against pid reuse by its start time), so it can start no more +/// children and its descendants stay its descendants; then a last record is +/// made within the same caps and at most after the +/// stop; then the command is killed and every recorded descendant, the kills +/// ending + after the stop; +/// then the host waits for the command to exit until +/// after the stop. Each budget is checked before +/// each entry or descendant, so the cleanup returns within +/// plus one stat read and one signal. The +/// kills when the command exits by itself run to the end; a stop that lands +/// there lets them go on for at most . A descendant +/// past the caps or the budgets, or one started and reparented between two +/// records, keeps running. On macOS, which has no /proc, nothing is +/// recorded: only .NET's tree kill on a +/// stop applies. +/// +/// +internal static class CoreCommand +{ + /// The most of each stream read after the command has exited. + internal const int ExitDrain = 4 << 20; + + /// How long after a stop the last record of descendants may run (without groups). + internal const int StopScanMs = 250; + + /// How much longer after that the kills of recorded descendants may run. + internal const int StopKillMs = 250; + + /// + /// The most time from a stop to the end of the cleanup, the wait for the + /// killed command to exit included: + + /// + 250 ms, each budget checked before each + /// step. + /// + internal const int StopBudgetMs = StopScanMs + StopKillMs + 250; + + /// The longest single wait between checks. + private const int PollSliceMs = 5; + + /// The most of each stream one step reads, so a fast writer cannot hold the caller. + private const int Slice = 1 << 20; + + /// How often descendants are recorded where there are no process groups. + private const long SampleIntervalMs = 10; + + /// The most /proc entries one record reads, and the most descendants recorded in all. + internal const int ProcEntries = 16_384; + + /// The most bytes of stat files one record reads. + internal const int ProcBytes = 4 << 20; + + /// The most bytes read of one stat file; the fields read come well before. + internal const int StatBytes = 1024; + + private const short Ready = Posix.PollIn | Posix.PollHup | Posix.PollErr | Posix.PollNval; + + /// Test seam: when set, commands run without a process group, as where setsid is absent. + internal static volatile bool WithoutGroups; + + /// Test seam: whether descendants are recorded where there are no groups. + internal static volatile bool RecordDescendants = true; + + /// Test seam: runs after each record with every pid recorded so far. Null outside tests. + internal static volatile Action>? RecordedHook; + + /// Test seam: the directory read as /proc. + internal static volatile string ProcRoot = "/proc"; + + /// Test seam: the entry cap of one record. + internal static volatile int ProcEntryCap = ProcEntries; + + /// Test seam: the byte cap of one record. + internal static volatile int ProcByteCap = ProcBytes; + + /// + /// Test seam: runs after each stat file a record or the kills of + /// recorded descendants read, with its path, how many bytes were read, + /// and whether a stop ends that work. Null outside tests. + /// + internal static volatile Action? StatReadHook; + + /// Test seam: the clock the budgets after a stop are measured with, in ticks. Null outside tests. + internal static volatile Func? ClockHook; + + /// Test seam: runs before the host sends the command itself a signal on a stop without groups, with its pid and the signal. Null outside tests. + internal static volatile Action? SignalHook; + + /// Test seam: receives the end of each wait for a killed command to exit, on the budgets' clock. Null outside tests. + internal static volatile Action? ExitWaitHook; + + /// Test seam: receives the pid of each command once it has started. Null outside tests. + internal static volatile Action? StartedHook; + + /// Test seam: runs after the cleanup that follows the exit, before the stop is checked again. Null outside tests. + internal static volatile Action? ExitedHook; + + /// Test seam: whether the group is killed when the command exits. + internal static volatile bool KillGroupAtExit = true; + + /// Test seam: whether the group is killed on a stop. + internal static volatile bool KillGroupOnStop = true; + + /// The setsid that starts each command in its own group, or null where groups are not used. + internal static string? Setsid { get; } = FindSetsid(); + + /// Whether the next command runs in its own process group. + internal static bool Grouped => Setsid is not null && !WithoutGroups; + + private static string? FindSetsid() + { + if (!OperatingSystem.IsLinux() || !File.Exists("/proc/self/stat")) + { + return null; + } + + foreach (string candidate in new[] { "/usr/bin/setsid", "/bin/setsid" }) + { + if (IsExecutable(candidate)) + { + return candidate; + } + } + + return null; + } + + /// Whether is a regular file this process may execute (access(X_OK)). + private static bool IsExecutable(string path) + { + try + { + return Unix.Stat(path, follow: true).Kind == Unix.Kind.Regular && Unix.MayExecute(path); + } + catch (IOException) + { + return false; + } + } + + /// + /// The command's executable, resolved on the PATH when it names no + /// directory, so that one setsid cannot exec is reported as a + /// start failure, as without setsid. + /// + private static string Executable(string name) + { + if (name.Contains('/', StringComparison.Ordinal)) + { + if (IsExecutable(name)) + { + return name; + } + } + else + { + string path = Environment.GetEnvironmentVariable("PATH") ?? string.Empty; + foreach (string directory in path.Split(':')) + { + string candidate = Path.Combine(directory.Length == 0 ? "." : directory, name); + if (IsExecutable(candidate)) + { + return candidate; + } + } + } + + throw CarrierException.Loss("could not start " + Identities.Excerpt(name) + " (not an executable file)"); + } + + /// A command that exited. + /// The exit status; .NET reports a signal N as 128 + N. + /// At most the requested limit of stdout. + /// Whether stdout was longer than its limit. + /// At most the requested limit of stderr. + internal sealed record Output(int ExitStatus, byte[] Stdout, bool StdoutOverflow, byte[] Stderr); + + /// A bounded prefix of one stream, read only when poll reports it ready. + private sealed class Capture + { + private readonly Stream stream; + private readonly int limit; + private readonly byte[] chunk = new byte[8192]; + private byte[] kept = Array.Empty(); + private int length; + + internal Capture(Stream stream, int fd, int limit) + { + this.stream = stream; + Fd = fd; + this.limit = limit; + } + + internal int Fd { get; } + + internal bool Overflow { get; private set; } + + internal bool Ended { get; private set; } + + /// Reads at most bytes already in the pipe; returns how many. Never blocks. + internal int Step(int budget) + { + int total = 0; + if (Ended) + { + return 0; + } + + try + { + Span fds = stackalloc Posix.PollFd[1]; + while (total < budget) + { + fds[0] = new Posix.PollFd { Fd = Fd, Events = Posix.PollIn }; + Posix.Poll(fds, 0); + if ((fds[0].Revents & Ready) == 0) + { + break; + } + + int n = stream.Read(chunk, 0, Math.Min(chunk.Length, budget - total)); + if (n <= 0) + { + Ended = true; + break; + } + + Keep(n); + total += n; + } + } + catch (Exception error) when (error is IOException or ObjectDisposedException or CarrierException) + { + Ended = true; + } + + return total; + } + + private void Keep(int n) + { + int room = limit - length; + if (n > room) + { + Overflow = true; + } + + int take = Math.Min(n, room); + if (take > 0) + { + if (length + take > kept.Length) + { + Array.Resize(ref kept, Math.Min(limit, Math.Max(length + take, kept.Length * 2))); + } + + Array.Copy(chunk, 0, kept, length, take); + length += take; + } + } + + internal byte[] Bytes() => kept.AsSpan(0, length).ToArray(); + } + + /// The command's descendants seen while it ran, by pid, with their start times (Linux). + private sealed class Recorded + { + private readonly Dictionary started = new(); + private long next = Stopwatch.GetTimestamp(); + + /// + /// Records the command's current descendants, at most once per + /// , reading at most the entry and byte + /// caps; a stop ends the record at the next entry, recording nothing + /// from it. The last record, made on a stop once the command is + /// frozen, runs whenever it is asked, within the caps, until + /// (checked before each entry); what it read by + /// then is recorded. + /// + internal void Sample(int root, long expires, CancellationToken cancellationToken, long end = long.MaxValue) + { + bool final = end != long.MaxValue; + long now = Stopwatch.GetTimestamp(); + if (!RecordDescendants || (now < next && !final) || !OperatingSystem.IsLinux()) + { + return; + } + + next = now + (Stopwatch.Frequency * SampleIntervalMs / 1000); + var children = new Dictionary>(); + int cap = ProcEntryCap; + int byteCap = ProcByteCap; + int entries = 0; + long bytes = 0; + foreach (string entry in ProcDirectories()) + { + if (final ? Now() >= end : Stopped(expires, cancellationToken) is not null) + { + if (!final) + { + return; + } + + break; + } + + if (++entries > cap || bytes >= byteCap) + { + break; + } + + // Never more than the bytes left, so a record reads at most its cap. + if (ReadStat(entry, ref bytes, (int)Math.Min(StatBytes, byteCap - bytes), stoppable: !final) is { } stat) + { + if (!children.TryGetValue(stat.Parent, out List<(int, string)>? list)) + { + list = new List<(int, string)>(); + children[stat.Parent] = list; + } + + list.Add((stat.Pid, stat.Start)); + } + } + + var queue = new Queue(); + var seen = new HashSet { root }; + queue.Enqueue(root); + while (queue.Count > 0) + { + if (!children.TryGetValue(queue.Dequeue(), out List<(int Pid, string Start)>? list)) + { + continue; + } + + foreach ((int pid, string start) in list) + { + if (seen.Add(pid)) + { + if (started.Count < ProcEntries) + { + started.TryAdd(pid, start); + } + + queue.Enqueue(pid); + } + } + } + + RecordedHook?.Invoke(new List(started.Keys)); + } + + /// + /// Kills every recorded descendant still alive with the start time it + /// was recorded with, so a reused pid is not killed (the start time is + /// read just before the kill, as the JDK's process handles do). It + /// ends at , checked before each descendant; + /// when that is unset (at the command's own exit) it runs to the end, + /// bounded by the number recorded, unless a stop lands, which lets it + /// go on for at most more. + /// + internal void KillAlive(long expires, CancellationToken cancellationToken, long end = long.MaxValue) + { + long bytes = 0; + bool afterStop = end != long.MaxValue; + foreach ((int pid, string start) in started) + { + if (!afterStop && Stopped(expires, cancellationToken) is not null) + { + afterStop = true; + end = After(Now(), StopKillMs); + } + + if (afterStop && Now() >= end) + { + return; + } + + if (ReadStat(Path.Combine(ProcRoot, pid.ToString(CultureInfo.InvariantCulture)), ref bytes, StatBytes, !afterStop) is { } stat && stat.Start == start && !stat.Zombie) + { + Unix.Kill(pid); + } + } + } + } + + private static IEnumerable ProcDirectories() + { + IEnumerable entries; + try + { + entries = Directory.EnumerateDirectories(ProcRoot); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + yield break; + } + + using IEnumerator cursor = entries.GetEnumerator(); + while (true) + { + try + { + if (!cursor.MoveNext()) + { + yield break; + } + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + yield break; + } + + string name = Path.GetFileName(cursor.Current); + if (name.Length > 0 && name.Length <= 10 && IsDigits(name)) + { + yield return cursor.Current; + } + } + } + + private static bool IsDigits(string text) + { + foreach (char c in text) + { + if (c < '0' || c > '9') + { + return false; + } + } + + return true; + } + + /// + /// A process's pid, parent, start time, and whether it is a zombie, from + /// at most of /proc/<pid>/stat, + /// read after the last ) (the command name may hold any byte), + /// and at most ; null when it cannot be read. + /// Adds the bytes read to . Only reads for a + /// record or a descendant's kill are passed to . + /// + private static (int Pid, int Parent, int Group, string Start, bool Zombie)? ReadStat(string directory, ref long bytes, int limit = StatBytes, bool stoppable = false, bool report = true) + { + try + { + string path = Path.Combine(directory, "stat"); + byte[] buffer = new byte[Math.Clamp(limit, 0, StatBytes)]; + int read = 0; + using (var stream = new FileStream(path, new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Share = FileShare.ReadWrite, BufferSize = 0 })) + { + int n; + while (read < buffer.Length && (n = stream.Read(buffer, read, buffer.Length - read)) > 0) + { + read += n; + } + } + + bytes += read; + if (report) + { + StatReadHook?.Invoke(path, read, stoppable); + } + + string stat = System.Text.Encoding.UTF8.GetString(buffer, 0, read); + int close = stat.LastIndexOf(')'); + if (close < 0 || !int.TryParse(Path.GetFileName(directory), NumberStyles.None, CultureInfo.InvariantCulture, out int pid)) + { + return null; + } + + // state ppid pgrp session tty_nr tpgid flags minflt cminflt majflt + // cmajflt utime stime cutime cstime priority nice num_threads + // itrealvalue starttime ... + // A field cut by the byte cap is not read: the start time must be followed by another field. + string[] fields = stat[(close + 1)..].Trim().Split(' '); + if (fields.Length < 21 + || !int.TryParse(fields[1], NumberStyles.None, CultureInfo.InvariantCulture, out int parent) + || !int.TryParse(fields[2], NumberStyles.None, CultureInfo.InvariantCulture, out int group)) + { + return null; + } + + return (pid, parent, group, fields[19], fields[0] is "Z" or "X"); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + return null; + } + } + + /// + /// Kills group , the pid of , + /// unless the leader has exited and a live process now holds that pid: + /// while a group has members its id cannot be reused, so a reaped + /// leader's pid held by a live process means the old group is empty. + /// + internal static void KillGroup(int group, Process leader) + { + long bytes = 0; + if (HasExited(leader) && ReadStat("/proc/" + group.ToString(CultureInfo.InvariantCulture), ref bytes, report: false) is { Zombie: false }) + { + return; + } + + Unix.KillGroup(group); + } + + private static bool HasExited(Process process) + { + try + { + return process.HasExited; + } + catch (Exception error) when (error is InvalidOperationException or Win32Exception) + { + return true; + } + } + + /// + /// Whether setsid itself failed to exec the command: it exits 126 + /// (cannot execute) or 127 (not found) and its message, unlike the + /// command's, starts with "setsid: " (util-linux and BusyBox both + /// print that prefix). The command never ran then. + /// + internal static bool SetsidFailed(Output output) => + output.ExitStatus is 126 or 127 && System.Text.Encoding.UTF8.GetString(output.Stderr).StartsWith("setsid: ", StringComparison.Ordinal); + + /// + /// Runs , adding + /// to the inherited environment, and returns once it has exited. + /// + /// The executable and its arguments. + /// Variables added to the inherited environment. + /// Bounds the whole run, from the start to the exit. + /// How much stdout is kept; the rest is read and counted as overflow. + /// How much stderr is kept. + /// Stops the command; it is then killed. + /// The setsid to run the command through where groups are used, in place of . + /// + /// A loss when the command cannot start; a timeout when it does not exit + /// in time; aborted when the token is cancelled. The command is killed in + /// the last two cases. + /// + internal static Output Run( + IReadOnlyList command, + IReadOnlyDictionary? environment, + TimeSpan timeout, + int stdoutLimit, + int stderrLimit, + CancellationToken cancellationToken, + string? setsid = null) + { + if (command.Count == 0 || command[0].Length == 0) + { + throw new ArgumentException("command must be a nonempty executable argument list", nameof(command)); + } + + if (timeout <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(timeout), "timeout must be positive"); + } + + long expires = Worker.Expires(Stopwatch.GetTimestamp(), timeout); + if (Stopped(expires, cancellationToken) is { } early) + { + throw early; + } + + bool grouped = Grouped; + var info = new ProcessStartInfo(grouped ? setsid ?? Setsid! : command[0]) + { + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + if (grouped) + { + info.ArgumentList.Add("--"); + info.ArgumentList.Add(Executable(command[0])); + } + + for (int i = 1; i < command.Count; i++) + { + info.ArgumentList.Add(command[i]); + } + + if (environment is not null) + { + foreach (KeyValuePair variable in environment) + { + info.Environment[variable.Key] = variable.Value; + } + } + + Process process; + try + { + process = Process.Start(info) ?? throw new InvalidOperationException("no process was started"); + } + catch (Exception error) when (error is Win32Exception or InvalidOperationException or IOException or PlatformNotSupportedException) + { + throw CarrierException.Loss("could not start " + Identities.Excerpt(command[0]) + " (" + error.Message + ")"); + } + + int pid = process.Id; + StartedHook?.Invoke(pid); + Recorded? recorded = grouped ? null : new Recorded(); + + // Read now so the SIGSTOP on a stop cannot reach a process that took a reused pid. + long unused = 0; + string? coreStart = recorded is not null && OperatingSystem.IsLinux() ? ReadStat("/proc/" + pid.ToString(CultureInfo.InvariantCulture), ref unused, report: false)?.Start : null; + Stream? stdout = null; + Stream? stderr = null; + bool cleaned = false; + try + { + try + { + // Its stdin reads end of file at once. + process.StandardInput.Close(); + } + catch (Exception error) when (error is IOException or ObjectDisposedException) + { + // The command already closed its end. + } + + stdout = process.StandardOutput.BaseStream; + stderr = process.StandardError.BaseStream; + var output = new Capture(stdout, Descriptor(stdout), stdoutLimit); + var errors = new Capture(stderr, Descriptor(stderr), stderrLimit); + Output result = Await(process, pid, grouped, recorded, output, errors, expires, cancellationToken, ref cleaned); + if (grouped && SetsidFailed(result)) + { + string detail = System.Text.Encoding.UTF8.GetString(result.Stderr).Trim(); + int line = detail.IndexOf('\n', StringComparison.Ordinal); + throw CarrierException.Loss("could not start " + Identities.Excerpt(command[0]) + " (" + (line < 0 ? detail : detail[..line]) + ")"); + } + + return result; + } + finally + { + if (!cleaned) + { + // A stop: everything below ends within StopBudgetMs of it. + long stop = Now(); + if (grouped) + { + // While the command is alive, or unreaped, its group id is reserved. + if (KillGroupOnStop) + { + KillGroup(pid, process); + } + + KillQuietly(process, entireTree: false); + AwaitExit(process, After(stop, StopBudgetMs)); + if (KillGroupOnStop) + { + // Members forked while the first kill ran. + KillGroup(pid, process); + } + } + else + { + // Not .NET's tree kill, which reads every process on the + // machine: the command is frozen at once, so it starts + // nothing more and keeps its children; a last record + // within its budget; the command killed; then every + // recorded descendant within theirs. + if (!HasExited(process)) + { + if (OperatingSystem.IsLinux()) + { + Signal(pid, coreStart, Unix.Sigstop); + recorded!.Sample(pid, expires, cancellationToken, After(stop, StopScanMs)); + SignalHook?.Invoke(pid, Unix.Sigkill); + } + + KillQuietly(process, entireTree: !OperatingSystem.IsLinux()); + } + + recorded!.KillAlive(expires, cancellationToken, After(stop, StopScanMs + StopKillMs)); + AwaitExit(process, After(stop, StopBudgetMs)); + } + } + + // A descendant still writing has no reader now, so its next write fails. + stdout?.Dispose(); + stderr?.Dispose(); + process.Dispose(); + } + } + + private static int Descriptor(Stream stream) => + stream is PipeStream pipe + ? (int)pipe.SafePipeHandle.DangerousGetHandle() + : throw CarrierException.Loss("the command's output is " + stream.GetType().Name + ", not a pipe"); + + private static Output Await( + Process process, + int pid, + bool grouped, + Recorded? recorded, + Capture stdout, + Capture stderr, + long expires, + CancellationToken cancellationToken, + ref bool cleaned) + { + int idle = 1; + Span fds = stackalloc Posix.PollFd[2]; + while (true) + { + // The exit is observed before the pipes are read, so everything + // the command itself wrote is read below. + bool exited = HasExited(process); + if (!exited) + { + recorded?.Sample(pid, expires, cancellationToken); + } + + int read = stdout.Step(Slice) + stderr.Step(Slice); + if (exited) + { + // The cleanup always runs to the end, as Rust kills the group + // when the command exits. + if (grouped) + { + if (KillGroupAtExit) + { + KillGroup(pid, process); + } + } + else + { + // A stop lets these kills go on for a bounded time; it is reported below. + recorded!.KillAlive(expires, cancellationToken); + } + + cleaned = true; + stdout.Step(ExitDrain); + stderr.Step(ExitDrain); + ExitedHook?.Invoke(); + + // The cleanup may have outlived the deadline or a stop. + if (Stopped(expires, cancellationToken) is { } late) + { + throw late; + } + + return new Output(process.ExitCode, stdout.Bytes(), stdout.Overflow, stderr.Bytes()); + } + + if (Stopped(expires, cancellationToken) is { } stopped) + { + throw stopped; + } + + if (read > 0) + { + idle = 1; + continue; + } + + int count = 0; + foreach (Capture capture in new[] { stdout, stderr }) + { + if (!capture.Ended) + { + fds[count++] = new Posix.PollFd { Fd = capture.Fd, Events = Posix.PollIn }; + } + } + + Posix.Poll(fds[..count], Math.Min(idle, Worker.RemainingMs(expires, Stopwatch.GetTimestamp()))); + idle = Math.Min(idle * 2, PollSliceMs); + } + } + + /// Why the run must stop now, or null. + private static CarrierException? Stopped(long expires, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + { + return CarrierException.Aborted("command aborted"); + } + + return Stopwatch.GetTimestamp() >= expires ? CarrierException.Timeout("command did not exit in time") : null; + } + + private static void KillQuietly(Process process, bool entireTree) + { + try + { + process.Kill(entireTree); + } + catch (Exception error) when (error is InvalidOperationException or Win32Exception or AggregateException or NotSupportedException) + { + // Already gone, or a descendant could not be signalled. + } + } + + /// + /// Waits until at most for a killed command to + /// exit, in short poll(2) sleeps, so the wait neither blocks on a + /// .NET wait handle nor consumes a . + /// + private static void AwaitExit(Process process, long end) + { + ExitWaitHook?.Invoke(end); + while (!HasExited(process) && Now() < end) + { + Posix.Poll(Span.Empty, 1); + } + } + + /// + /// Sends to the command + /// unless it is gone, a zombie, or no longer has the start time + /// it had when it started (its pid was reused); + /// nothing is sent when that start time could not be read. + /// + private static void Signal(int pid, string? start, int signal) + { + long bytes = 0; + if (start is not null && ReadStat("/proc/" + pid.ToString(CultureInfo.InvariantCulture), ref bytes, report: false) is { Zombie: false } stat && stat.Start == start) + { + SignalHook?.Invoke(pid, signal); + Unix.Signal(pid, signal); + } + } + + /// The time the budgets after a stop are measured with. + private static long Now() => ClockHook?.Invoke() ?? Stopwatch.GetTimestamp(); + + /// The time after . + private static long After(long start, int milliseconds) => start + (Stopwatch.Frequency * milliseconds / 1000); +} diff --git a/dotnet/src/JacquardHost/Sdk/Dynamic.cs b/dotnet/src/JacquardHost/Sdk/Dynamic.cs new file mode 100644 index 0000000..9607b77 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Dynamic.cs @@ -0,0 +1,481 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Globalization; +using System.Text; +using System.Threading; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// A native value with no application type: Jacquard data by public name. +/// Decode any result into it with or +/// . Equality is structural (a +/// compares its exact bits), and both it and +/// walk a value with an explicit stack, so a deep +/// value cannot exhaust the call stack; equality compares each pair of +/// shared sub-values once. +/// +public abstract class Dynamic : IEquatable +{ + private const int RenderLimit = 4096; + + /// How many places hold this value as a child; more than one means it is shared. + private int uses; + + private protected Dynamic() + { + } + + /// Test seam: how many pair sets made on this thread. + [ThreadStatic] + internal static int PairSetsMade; + + /// Whether this value is a child in more than one place, so a walk can reach it twice. + private bool Shared => Volatile.Read(ref uses) > 1; + + /// Counts each of as held once more. + private protected static void Adopt(IReadOnlyList children) + { + foreach (Dynamic child in children) + { + Interlocked.Increment(ref child.uses); + } + } + + /// Unit, the empty tuple. + public static Tuple Unit { get; } = new(Array.Empty()); + + /// The child values, in order: items, fields, or an option's value. + private protected virtual IReadOnlyList Children => Array.Empty(); + + /// An Int. + public static Dynamic Of(long value) => new Int(value); + + /// An Int. + public static Dynamic Of(int value) => new Int(value); + + /// An Int; a widens exactly. + public static Dynamic Of(uint value) => new Int(value); + + /// An Int. + public static Dynamic Of(short value) => new Int(value); + + /// An Int. + public static Dynamic Of(ushort value) => new Int(value); + + /// An Int. + public static Dynamic Of(sbyte value) => new Int(value); + + /// An Int. + public static Dynamic Of(byte value) => new Int(value); + + /// A Real. + public static Dynamic Of(double value) => new Real(value); + + /// A Text. + public static Dynamic Of(string value) => new Text(value); + + /// A Bool. + public static Dynamic Of(bool value) => new Bool(value); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Unsigned64, error: true)] + public static Dynamic Of(ulong value) => throw new ArgumentException(NoConversion.Unsigned64, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.UnsignedNative, error: true)] + public static Dynamic Of(nuint value) => throw new ArgumentException(NoConversion.UnsignedNative, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Character, error: true)] + public static Dynamic Of(char value) => throw new ArgumentException(NoConversion.Character, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Single, error: true)] + public static Dynamic Of(float value) => throw new ArgumentException(NoConversion.Single, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.HalfFloat, error: true)] + public static Dynamic Of(Half value) => throw new ArgumentException(NoConversion.HalfFloat, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Decimal, error: true)] + public static Dynamic Of(decimal value) => throw new ArgumentException(NoConversion.Decimal, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Signed128, error: true)] + public static Dynamic Of(Int128 value) => throw new ArgumentException(NoConversion.Signed128, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Unsigned128, error: true)] + public static Dynamic Of(UInt128 value) => throw new ArgumentException(NoConversion.Unsigned128, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Big, error: true)] + public static Dynamic Of(System.Numerics.BigInteger value) => throw new ArgumentException(NoConversion.Big, nameof(value)); + + /// A List. + /// More items than the v0 hard limit. + public static Dynamic ListOf(params Dynamic[] items) => new List(items); + + /// A tuple. + /// More items than the v0 hard limit. + public static Dynamic TupleOf(params Dynamic[] items) => new Tuple(items); + + /// Some(value). + public static Dynamic Some(Dynamic value) => new Option(value ?? throw new ArgumentNullException(nameof(value))); + + /// None. + public static Dynamic None() => new Option(null); + + /// A constructor value by public name; also checks that the constructor exists. + /// More fields than the v0 hard limit. + public static Dynamic VariantOf(string name, params Dynamic[] fields) => new Variant(name, fields); + + /// + /// A read-only copy of , refused with an + /// when it holds more items than any v0 + /// frame admits: counted first when the collection says its count, and + /// otherwise enumerated at most one item past the limit. + /// + private static ReadOnlyCollection Limited(IEnumerable items, string what) + { + ArgumentNullException.ThrowIfNull(items); + int limit = Limits.Hard.MaxCollectionItems; + int? known = items switch + { + IReadOnlyCollection sized => sized.Count, + ICollection sized => sized.Count, + _ => null, + }; + if (known > limit) + { + throw new ArgumentException(what + " of " + known.Value.ToString(CultureInfo.InvariantCulture) + + " items exceeds the v0 hard limit of " + limit.ToString(CultureInfo.InvariantCulture), nameof(items)); + } + + var copy = new List(known ?? 0); + foreach (Dynamic item in items) + { + if (copy.Count == limit) + { + throw new ArgumentException(what + " exceeds the v0 hard limit of " + limit.ToString(CultureInfo.InvariantCulture) + " items", nameof(items)); + } + + copy.Add(item ?? throw new ArgumentNullException(nameof(items), what + " holds null")); + } + + Adopt(copy); + return copy.AsReadOnly(); + } + + /// Whether this node, apart from its children, equals . + private protected abstract bool SameNode(Dynamic other); + + /// This node's hash, apart from its children. + private protected abstract int NodeHash(); + + /// This node's text before its children, such as Variant(Job. + private protected abstract string Open(); + + /// + public bool Equals(Dynamic? other) + { + if (other is null) + { + return false; + } + + // Each pair of nodes is compared once: values that share sub-values + // (as decoded values and caller-built graphs may) are compared in + // time proportional to their distinct pairs, not their paths. Only a + // pair with a shared side can be reached twice (an unshared node has + // one parent and one place in it), so only those pairs are kept, and + // the set is made at the first: unshared values need none. + var stack = new Stack<(Dynamic Left, Dynamic Right)>(); + HashSet<(Dynamic Left, Dynamic Right)>? compared = null; + stack.Push((this, other)); + while (stack.Count > 0) + { + (Dynamic left, Dynamic right) = stack.Pop(); + if (ReferenceEquals(left, right)) + { + continue; + } + + if (left.Shared || right.Shared) + { + if (compared is null) + { + compared = new HashSet<(Dynamic Left, Dynamic Right)>(PairComparer.Instance); + PairSetsMade++; + } + + if (!compared.Add((left, right))) + { + continue; + } + } + + if (left.GetType() != right.GetType() || !left.SameNode(right)) + { + return false; + } + + IReadOnlyList a = left.Children; + IReadOnlyList b = right.Children; + if (a.Count != b.Count) + { + return false; + } + + for (int i = 0; i < a.Count; i++) + { + stack.Push((a[i], b[i])); + } + } + + return true; + } + + /// + public override bool Equals(object? obj) => obj is Dynamic other && Equals(other); + + /// Pairs of nodes by reference. + private sealed class PairComparer : IEqualityComparer<(Dynamic Left, Dynamic Right)> + { + internal static readonly PairComparer Instance = new(); + + public bool Equals((Dynamic Left, Dynamic Right) x, (Dynamic Left, Dynamic Right) y) => + ReferenceEquals(x.Left, y.Left) && ReferenceEquals(x.Right, y.Right); + + public int GetHashCode((Dynamic Left, Dynamic Right) obj) => + HashCode.Combine(System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(obj.Left), System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(obj.Right)); + } + + /// A hash of this node and its children's count; equal values hash alike. + public override int GetHashCode() => HashCode.Combine(GetType(), NodeHash(), Children.Count); + + /// The value as text, cut at 4,096 chars. + public override string ToString() + { + var output = new StringBuilder(); + var stack = new Stack(); + stack.Push(this); + while (stack.Count > 0 && output.Length <= RenderLimit) + { + object next = stack.Pop(); + if (next is string text) + { + output.Append(text); + continue; + } + + var node = (Dynamic)next; + output.Append(node.Open()); + IReadOnlyList children = node.Children; + stack.Push(")"); + for (int i = children.Count - 1; i >= 0; i--) + { + stack.Push(children[i]); + if (i > 0 || node is Variant) + { + stack.Push(", "); + } + } + } + + return output.Length > RenderLimit ? output.ToString(0, RenderLimit) + "..." : output.ToString(); + } + + /// A Jacquard Int. + /// The integer. + public sealed class Int(long Value) : Dynamic + { + /// The integer. + public long Value { get; } = Value; + + private protected override bool SameNode(Dynamic other) => other is Int value && value.Value == Value; + + private protected override int NodeHash() => Value.GetHashCode(); + + private protected override string Open() => "Int(" + Value.ToString(CultureInfo.InvariantCulture); + } + + /// A Jacquard Real; equality compares the exact bits. + /// The double. + public sealed class Real(double Value) : Dynamic + { + /// The double. + public double Value { get; } = Value; + + private protected override bool SameNode(Dynamic other) => + other is Real value && BitConverter.DoubleToInt64Bits(value.Value) == BitConverter.DoubleToInt64Bits(Value); + + private protected override int NodeHash() => BitConverter.DoubleToInt64Bits(Value).GetHashCode(); + + private protected override string Open() => "Real(" + Value.ToString("R", CultureInfo.InvariantCulture); + } + + /// A Jacquard Text. + public sealed class Text : Dynamic + { + /// A text value. + public Text(string value) + { + Value = value ?? throw new ArgumentNullException(nameof(value)); + } + + /// The string. + public string Value { get; } + + private protected override bool SameNode(Dynamic other) => other is Text value && string.Equals(value.Value, Value, StringComparison.Ordinal); + + private protected override int NodeHash() => StringComparer.Ordinal.GetHashCode(Value); + + private protected override string Open() => "Text(" + Identities.Excerpt(Value); + } + + /// A Jacquard Bool. + /// The boolean. + public sealed class Bool(bool Value) : Dynamic + { + /// The boolean. + public bool Value { get; } = Value; + + private protected override bool SameNode(Dynamic other) => other is Bool value && value.Value == Value; + + private protected override int NodeHash() => Value ? 1 : 2; + + private protected override string Open() => Value ? "Bool(true" : "Bool(false"; + } + + /// A Jacquard Hash. + public sealed class HashValue : Dynamic + { + /// A hash value. + public HashValue(Hash value) + { + Value = value ?? throw new ArgumentNullException(nameof(value)); + } + + /// The hash. + public Hash Value { get; } + + private protected override bool SameNode(Dynamic other) => other is HashValue value && value.Value.Equals(Value); + + private protected override int NodeHash() => Value.GetHashCode(); + + private protected override string Open() => "Hash(" + Value.Hex; + } + + /// A positional product; the empty tuple is unit. + public sealed class Tuple : Dynamic + { + /// A tuple of a copy of . + /// More items than the v0 hard limit. + public Tuple(IEnumerable items) + { + Items = Limited(items, "tuple"); + } + + /// The items. + public IReadOnlyList Items { get; } + + private protected override IReadOnlyList Children => Items; + + private protected override bool SameNode(Dynamic other) => true; + + private protected override int NodeHash() => 3; + + private protected override string Open() => "Tuple("; + } + + /// A Jacquard List. + public sealed class List : Dynamic + { + /// A list of a copy of . + /// More items than the v0 hard limit. + public List(IEnumerable items) + { + Items = Limited(items, "list"); + } + + /// The items. + public IReadOnlyList Items { get; } + + private protected override IReadOnlyList Children => Items; + + private protected override bool SameNode(Dynamic other) => true; + + private protected override int NodeHash() => 4; + + private protected override string Open() => "List("; + } + + /// A Jacquard Option: is null for None. + public sealed class Option : Dynamic + { + /// Some(value), or None for null. + public Option(Dynamic? value) + { + Value = value; + if (value is not null) + { + Adopt(new[] { value }); + } + + } + + /// The value of Some, or null for None. + public Dynamic? Value { get; } + + private protected override IReadOnlyList Children => Value is null ? Array.Empty() : new[] { Value }; + + private protected override bool SameNode(Dynamic other) => other is Option option && (option.Value is null) == (Value is null); + + private protected override int NodeHash() => Value is null ? 5 : 6; + + private protected override string Open() => Value is null ? "None(" : "Some("; + } + + /// + /// A constructor by public name (store spelling such as stock, or + /// display spelling such as Stock) and its positional fields. + /// + public sealed class Variant : Dynamic + { + /// A variant of a copy of . + /// More fields than the v0 hard limit. + public Variant(string name, IEnumerable fields) + { + Name = name ?? throw new ArgumentNullException(nameof(name)); + Fields = Limited(fields, "constructor"); + } + + /// The constructor's public name. + public string Name { get; } + + /// The positional fields. + public IReadOnlyList Fields { get; } + + private protected override IReadOnlyList Children => Fields; + + private protected override bool SameNode(Dynamic other) => other is Variant variant && string.Equals(variant.Name, Name, StringComparison.Ordinal); + + private protected override int NodeHash() => StringComparer.Ordinal.GetHashCode(Name); + + private protected override string Open() => "Variant(" + Identities.Excerpt(Name); + } +} diff --git a/dotnet/src/JacquardHost/Sdk/Encoder.cs b/dotnet/src/JacquardHost/Sdk/Encoder.cs new file mode 100644 index 0000000..eeb40b9 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Encoder.cs @@ -0,0 +1,380 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// Encodes one value at a location. Every encoding is bounded by the +/// selected limits as it goes, so no value, however it is built (one long +/// text referenced many times, say), costs more than about one frame of work +/// and memory: the JSON depth Core's wire form will need, +/// max_collection_items for each list, tuple, and constructor, +/// max_value_nodes values, and max_text_bytes for each text (in +/// UTF-8 bytes) and max_frame_bytes for all of them together. A text +/// longer in chars than either is refused before it is scanned. Nesting is +/// checked before each nested encoder is made, so no value or codec nesting +/// can recurse past the depth limit. +/// +/// +/// A custom codec may return a it built itself rather +/// than through an encoder. Such a value is not unchecked: before its type is +/// checked, and so before any worker starts, a function call measures every +/// argument value again under one shared budget (nodes, JSON depth, +/// collection items, and text bytes, alone and together), whatever codec +/// built it; the invoke frame is then measured as it is rendered, and the +/// host measures a handler's result the same way before sending it. A value +/// returned by is not measured again. +/// +public sealed class Encoder +{ + private readonly int depth; + private readonly Budget budget; + + private Encoder(ProgramData data, string path, int depth, Budget budget) + { + Data = data; + Path = path; + this.depth = depth; + this.budget = budget; + } + + /// The program's data. + public ProgramData Data { get; } + + /// The location being encoded, such as triage.batch.argument[1][0].service. + public string Path { get; } + + /// What one encoding has used, shared by every location in it. + internal sealed class Budget + { + internal int Nodes { get; set; } + + internal long TextBytes { get; set; } + } + + /// A root encoder at , sharing (as one frame's arguments do) or with a fresh one. + internal static Encoder Root(ProgramData data, string path, Budget? budget = null) + { + var encoder = new Encoder(data, path, 3, budget ?? new Budget()); + encoder.CheckDepth(encoder.depth); + return encoder; + } + + /// An invalid-argument failure at this location. + public JacquardException Invalid(string message) => JacquardException.InvalidAt(Path, message); + + /// An invalid-argument failure at this location, caused by . + internal JacquardException Invalid(string message, Exception cause) => JacquardException.InvalidAt(Path, message, cause); + + private void CheckDepth(long needed) + { + if (needed > Data.Limits.MaxJsonDepth) + { + throw Invalid("value exceeds the v0 JSON depth; use smaller independent batches"); + } + } + + private Encoder Child(string segment, int extra) + { + // Checked before the child exists, so nesting stops at the depth limit. + CheckDepth((long)depth + extra); + return new Encoder(Data, Path + segment, depth + extra, budget); + } + + /// Charges one value node. + private Value Node(Value value) + { + if (budget.Nodes >= Data.Limits.MaxValueNodes) + { + throw Invalid("value exceeds max_value_nodes; use smaller independent batches"); + } + + budget.Nodes++; + return value; + } + + /// Encodes with here. + internal Value Encode(Codec codec, T value) + { + ArgumentNullException.ThrowIfNull(codec); + if (value is null && !codec.AcceptsNull) + { + throw Invalid("null has no Jacquard value; use Option.None (or a nullable codec) for None, and Unit.Value for unit"); + } + + return codec.Encode(this, value); + } + + /// Encodes a named field of a record or variant one level deeper. + /// An invalid argument. + public Value Field(string name, Codec codec, T value) + { + ArgumentNullException.ThrowIfNull(name); + return Child("." + Identities.Excerpt(name), 2).Encode(codec, value); + } + + /// Encodes a positional item of a tuple one level deeper. + /// An invalid argument. + public Value Item(int index, Codec codec, T value) => + Child("[" + index.ToString(CultureInfo.InvariantCulture) + "]", 2).Encode(codec, value); + + /// + /// A constructor value by public name, such as "Job" or "job", + /// of fields already encoded (through ). Bool, + /// List, and Option constructors are refused: use , + /// , and . + /// + /// An invalid argument: an unknown constructor, or too many fields. + public Value Constructor(string name, IReadOnlyList fields) + { + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(fields); + + if (name.Length > Identities.NameLimit) + { + throw Invalid(Identities.TooLong("constructor", name)); + } + + // Counted before the fields are copied into the value. + CheckCount(fields.Count, "constructor"); + if (ProgramData.Builtins.Contains(Identities.DisplayToStore(name))) + { + throw Invalid("use the native form of builtin constructor " + Identities.Excerpt(name)); + } + + return Builtin(name, fields); + } + + private Value Builtin(string name, IReadOnlyList fields) + { + Hash identity = Data.ConstructorId(name) ?? throw Invalid("unknown constructor " + Identities.Excerpt(name)); + return Node(new ConstructorValue(identity, fields)); + } + + /// A tuple of already encoded items. + internal Value Tuple(IReadOnlyList items) => Node(new TupleValue(items)); + + /// Refuses more than max_collection_items items, before any is encoded. + internal void CheckCount(int count, string what) + { + if (count > Data.Limits.MaxCollectionItems) + { + throw Invalid(what + " exceeds max_collection_items"); + } + } + + internal Value Bool(bool value) => Builtin(value ? "true" : "false", Array.Empty()); + + internal Value Integer(long value) + { + if (value < Json.IntMin || value > Json.IntMax) + { + throw Invalid("integer is outside Core's 63-bit Int range"); + } + + return Node(new IntValue(value)); + } + + internal Value Real(double value) => Node(RealValue.Of(value)); + + internal Value Text(string text) + { + ChargeText(text); + return Node(new TextValue(text)); + } + + /// Charges a text by its UTF-8 bytes, refusing before it is scanned when its length in chars already exceeds a limit. + private void ChargeText(string text) + { + int limit = Data.Limits.MaxTextBytes; + + // A text's UTF-8 length is at least its length in chars, so these + // refuse before it is scanned. + if (text.Length > limit) + { + throw Invalid("text exceeds max_text_bytes"); + } + + if (budget.TextBytes + text.Length > Data.Limits.MaxFrameBytes) + { + throw Invalid("value exceeds max_frame_bytes; use smaller independent batches"); + } + + if (!JsonCodec.IsScalar(text)) + { + throw Invalid("text contains a lone surrogate, which has no Jacquard encoding"); + } + + long length = JsonCodec.Utf8Length(text); + if (length > limit) + { + throw Invalid("text exceeds max_text_bytes"); + } + + budget.TextBytes += length; + if (budget.TextBytes > Data.Limits.MaxFrameBytes) + { + throw Invalid("value exceeds max_frame_bytes; use smaller independent batches"); + } + } + + internal Value Hash(Hash hash) => Node(new HashValue(hash)); + + /// + /// A List of items of + /// from : one cons per item, nested on the + /// wire. The count is checked before any item is read. + /// + internal Value List(Codec codec, IReadOnlyList items, int start, int count) + { + CheckCount(count, "list"); + + // A linked list nests one constructor per element on the wire. + CheckDepth((long)depth + (2L * count)); + var encoded = new Value[count]; + for (int index = count - 1; index >= 0; index--) + { + encoded[index] = Child("[" + index.ToString(CultureInfo.InvariantCulture) + "]", 2 * (index + 1)).Encode(codec, items[start + index]); + } + + Value result = Builtin("nil", Array.Empty()); + for (int index = count - 1; index >= 0; index--) + { + result = Builtin("cons", new[] { encoded[index], result }); + } + + return result; + } + + /// None, or Some of one level deeper. + internal Value Option(Codec codec, bool present, T value) + { + if (!present) + { + return Builtin("none", Array.Empty()); + } + + Value inner = Child(string.Empty, 2).Encode(codec, value); + return Builtin("some", new[] { inner }); + } + + /// + /// Charges a raw boundary value passed through unchanged, walking it + /// within the same bounds; a value shared many times is counted each time + /// it occurs. The walk keeps an explicit stack, and the depth is checked + /// before each level is entered. + /// + internal Value Raw(Value value) + { + ArgumentNullException.ThrowIfNull(value); + var stack = new Stack<(Value Value, int Level)>(); + stack.Push((value, 0)); + while (stack.Count > 0) + { + (Value current, int level) = stack.Pop(); + if (current is null) + { + throw Invalid("null is not a boundary value"); + } + + if ((long)depth + (2L * level) > Data.Limits.MaxJsonDepth) + { + throw Invalid("value exceeds the v0 JSON depth; use smaller independent batches"); + } + + Node(current); + switch (current) + { + case TextValue text: + ChargeText(text.Value); + break; + case TupleValue tuple: + CheckCount(tuple.Items.Count, "tuple"); + Pending(stack.Count, tuple.Items.Count); + foreach (Value item in tuple.Items) + { + stack.Push((item, level + 1)); + } + + break; + case ConstructorValue constructor: + CheckCount(constructor.Arguments.Count, "constructor"); + Pending(stack.Count, constructor.Arguments.Count); + foreach (Value argument in constructor.Arguments) + { + stack.Push((argument, level + 1)); + } + + break; + case IntValue number when number.Value < Json.IntMin || number.Value > Json.IntMax: + throw Invalid("integer is outside Core's 63-bit Int range"); + default: + // Real and Hash are one node each. + break; + } + } + + return value; + } + + /// + /// Refuses before more values are stacked when + /// the values already charged and stacked could not all be charged, so + /// the walk's stack never holds more than max_value_nodes values. + /// + private void Pending(int stacked, int children) + { + if ((long)budget.Nodes + stacked + children > Data.Limits.MaxValueNodes) + { + throw Invalid("value exceeds max_value_nodes; use smaller independent batches"); + } + } +} + +/// Decodes boundary values with the program's data. +public sealed class Decoder +{ + internal Decoder(ProgramData data) + { + Data = data; + } + + /// The program's data. + public ProgramData Data { get; } + + /// Decodes a nested value. + /// A binding failure when the value does not map. + public T Decode(Codec codec, Value value) + { + ArgumentNullException.ThrowIfNull(codec); + return codec.Decode(this, value); + } + + /// Whether is built by the named constructor. + public bool Is(Value value, string constructor) + { + ArgumentNullException.ThrowIfNull(constructor); + return Data.ConstructorId(constructor) is { } expected && value is ConstructorValue built && built.Identity.Equals(expected); + } + + /// The fields of a value built by the named constructor with fields. + /// A mapping failure for another constructor or count. + public IReadOnlyList Fields(Value value, string constructor, int count) + { + if (Is(value, constructor) && value is ConstructorValue built && built.Arguments.Count == count) + { + return built.Arguments; + } + + throw Mapping(Identities.Excerpt(constructor) + " result has the wrong constructor or field count"); + } + + /// A mapping failure for a value that is none of a type's constructors. + public JacquardException Mismatch(string typeName) => Mapping("unknown " + Identities.Excerpt(typeName) + " case"); + + /// A mapping failure: a boundary value that does not decode into the requested type. + [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1822", Justification = "Part of the decoder a custom codec is given, beside its other failures.")] + public JacquardException Mapping(string? message) => JacquardException.Binding(message ?? "null"); +} diff --git a/dotnet/src/JacquardHost/Sdk/Function.cs b/dotnet/src/JacquardHost/Sdk/Function.cs new file mode 100644 index 0000000..bb14cfe --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Function.cs @@ -0,0 +1,317 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// A bound checked function. It is immutable and thread-safe; each call runs +/// one fresh, serial worker. Arguments are s: plain +/// values convert implicitly, and anything else is passed as +/// codec.Arg(value). Before any worker starts they are checked +/// against the function's checked signature, the selected limits, and the +/// frame size. +/// +public sealed class Function +{ + private readonly Hash target; + private readonly OperationRegistry registry; + private readonly Host host; + private readonly Action? rootCheck; + + internal Function(string name, CallInterface signature, Hash target, OperationRegistry registry, Host host, ProgramData data, Action? rootCheck = null) + { + this.rootCheck = rootCheck; + Name = name; + Signature = signature; + this.target = target; + this.registry = registry; + this.host = host; + Data = data; + } + + /// Test seam: runs after the arguments are measured, just before their types are checked. Null outside tests. + internal static volatile Action? ConformHook; + + /// The public name. + public string Name { get; } + + /// The checked signature. + public CallInterface Signature { get; } + + internal ProgramData Data { get; } + + /// + /// Refuses a call with the wrong number of arguments from the count + /// alone, before anything is copied or encoded, so the work below is + /// bounded by the checked signature, not by what the caller passed. + /// + private void Arity(long received) + { + int expected = Signature.Parameters.Count; + if (received != expected) + { + throw JacquardException.Invalid(Name + ": expected " + expected.ToString(CultureInfo.InvariantCulture) + " arguments, received " + + received.ToString(CultureInfo.InvariantCulture)); + } + } + + private string ArgumentPath(int index) => Name + ".argument[" + index.ToString(CultureInfo.InvariantCulture) + "]"; + + /// Encodes , one frame's worth, with one shared budget. + private List Encode(IReadOnlyList args) + { + Arity(args.Count); + var budget = new Encoder.Budget(); + var values = new List(args.Count); + for (int i = 0; i < args.Count; i++) + { + string path = ArgumentPath(i); + Encoder encoder = Encoder.Root(Data, path, budget); + if (args[i] is not { } argument) + { + throw encoder.Invalid("null has no Jacquard value; pass codec.Arg(value)"); + } + + try + { + values.Add(argument.Encode(encoder)); + } + catch (Exception error) when (Caller.Wraps(error)) + { + // A caller's codec failed; no worker starts, as for any refusal. + throw ProgramData.CodecFailed(path, error); + } + } + + return values; + } + + /// + /// Validates encoded arguments against the signature and the frame + /// limits, before any worker starts. Every value is first measured under + /// one shared budget (nodes, depth, text bytes, collection items), + /// whatever codec built it, so the type check below walks at most one + /// frame's worth of values. + /// + private List Check(List values) + { + Arity(values.Count); + var measured = new Encoder.Budget(); + for (int i = 0; i < values.Count; i++) + { + Encoder.Root(Data, ArgumentPath(i), measured).Raw(values[i]); + } + + ConformHook?.Invoke(); + for (int i = 0; i < values.Count; i++) + { + Data.Conform(values[i], Signature.Parameters[i], ArgumentPath(i)); + } + + Limits limits = host.Limits; + try + { + JsonObject frame = Protocol.Codec.InvokeFrame(target, Signature, values, registry.Entries(), limits); + JsonCodec.FrameBytes(frame, limits.MaxFrameBytes, limits.MaxJsonDepth); + } + catch (ProtocolException error) + { + throw JacquardException.Invalid(Name + ": " + error.Message + "; reduce input size or use smaller independent batches"); + } + + return values; + } + + /// + /// Runs one worker on the prepared store, by its path, once the cache + /// root is checked to be the directory that was prepared (a cancelled + /// call starts no worker and is reported as aborted, as before). + /// + private Invocation Run(List values, CancellationToken cancellationToken) + { + if (!cancellationToken.IsCancellationRequested) + { + rootCheck?.Invoke(); + } + + return new(host.Invoke(target, Signature, values, registry, cancellationToken), this); + } + + /// + /// Validates the arguments locally, then runs one worker. The exception + /// is only a local refusal; Core and carrier failures are in the + /// . Nothing is retried. + /// + /// An invalid argument; no worker started. + public Invocation Invoke(params Argument[] args) => Invoke((IReadOnlyList)args, default); + + /// + /// ; cancelling + /// kills the worker. + /// + /// An invalid argument; no worker started. + public Invocation Invoke(IReadOnlyList args, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(args); + return Run(Check(Encode(args)), cancellationToken); + } + + /// Invokes the function and decodes its result. + /// + /// An invalid argument, a failed invocation, or a result that does not + /// decode (which keeps the successful raw result). + /// + public TResult Call(Codec result, params Argument[] args) => Call(result, (IReadOnlyList)args, default); + + /// ; cancelling the token kills the worker. + /// An invalid argument, a failed invocation, or a result that does not decode. + public TResult Call(Codec result, IReadOnlyList args, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(result); + return Invoke(args, cancellationToken).Decode(result); + } + + /// + /// Runs a pure function whose final parameter and result are Lists over + /// explicit batches of , with + /// as the other arguments, and concatenates the + /// decoded results. Every batch is validated before the first runs, one + /// at a time: validation returns nothing, so each batch's encoding is + /// unreachable once it is checked, and the batch is encoded and checked + /// again just before it runs. Before each batch, and once more after the + /// last, the validation observes the token. Validation has no deadline of + /// its own, as in the other SDKs. Nothing is retried. The number of rows + /// is read once; a list whose count differs before a later batch is + /// refused. + /// + /// + /// A refusal before any worker started ( + /// is -1): an invalid batch, or a cancellation, which is reported as the + /// host's aborted invocation without a worker; or the failure of one + /// batch, with the decoded values of the batches before it in + /// . + /// + public IReadOnlyList MapBatches( + Codec item, + Codec row, + IReadOnlyList rows, + int batchSize, + IReadOnlyList? leading = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(item); + ArgumentNullException.ThrowIfNull(row); + ArgumentNullException.ThrowIfNull(rows); + leading ??= Array.Empty(); + if (batchSize < 1) + { + throw JacquardException.Invalid("batch size must be positive"); + } + + Hash? list = Data.Name("list"); + IReadOnlyList parameters = Signature.Parameters; + if (Signature.Effects.Count != 0 || parameters.Count == 0 || !IsList(parameters[^1], list) || !IsList(Signature.Result, list)) + { + throw JacquardException.Binding("MapBatches needs a pure function with a final List parameter and a List result"); + } + + // From the count alone, before any leading argument is copied. + Arity(leading.Count + 1L); + List empty = Batch(leading, row, rows, 0, 0); + + // Read once: every batch's range comes from this count, so a list + // that shrinks cannot make a negative batch. + int total = rows.Count; + int count = (int)((total + (long)batchSize - 1) / batchSize); + for (int index = 0; index < count; index++) + { + RefuseIfStopped(empty, cancellationToken); + Unchanged(rows, total); + Validate(leading, row, rows, total, index, batchSize); + } + + // A cancellation during the last batch's validation starts no worker. + RefuseIfStopped(empty, cancellationToken); + var completed = new List(); + Codec> results = Codecs.List(item); + for (int index = 0; index < count; index++) + { + try + { + Unchanged(rows, total); + (int start, int size) = Range(total, index, batchSize); + completed.AddRange(Run(Batch(leading, row, rows, start, size), cancellationToken).Decode(results)); + } + catch (JacquardException error) + { + var done = new object?[completed.Count]; + for (int i = 0; i < done.Length; i++) + { + done[i] = completed[i]; + } + + throw JacquardException.Batch(index, done, error); + } + } + + return completed; + } + + /// Throws the host's aborted invocation, which starts no worker, once the token is cancelled. + private void RefuseIfStopped(List empty, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + { + // The host observes the cancellation before it spawns a worker. + Invocation stopped = Run(empty, cancellationToken); + if (!stopped.Ok) + { + throw JacquardException.Invocation(stopped.Raw); + } + } + } + + /// Encodes and checks one batch, keeping nothing: the encoding is unreachable once this returns. + private void Validate(IReadOnlyList leading, Codec row, IReadOnlyList rows, int total, int index, int batchSize) + { + (int start, int size) = Range(total, index, batchSize); + Batch(leading, row, rows, start, size); + } + + /// Refuses rows whose count is no longer . + private static void Unchanged(IReadOnlyList rows, int total) + { + int now = rows.Count; + if (now != total) + { + throw JacquardException.Invalid("rows changed from " + total.ToString(CultureInfo.InvariantCulture) + " to " + + now.ToString(CultureInfo.InvariantCulture) + " during MapBatches"); + } + } + + private static (int Start, int Size) Range(int total, int index, int batchSize) + { + long start = (long)index * batchSize; + return ((int)start, (int)Math.Min(total - start, batchSize)); + } + + private List Batch(IReadOnlyList leading, Codec row, IReadOnlyList rows, int start, int size) + { + var args = new Argument[leading.Count + 1]; + for (int i = 0; i < leading.Count; i++) + { + args[i] = leading[i]; + } + + args[^1] = new Rows(row, rows, start, size); + return Check(Encode(args)); + } + + private static bool IsList(BoundaryType type, Hash? list) => list is not null && type is NominalType nominal && nominal.Identity.Equals(list); + + /// + public override string ToString() => "Function[" + Name + ", " + target.Hex + "]"; +} diff --git a/dotnet/src/JacquardHost/Sdk/Identities.cs b/dotnet/src/JacquardHost/Sdk/Identities.cs new file mode 100644 index 0000000..853c955 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Identities.cs @@ -0,0 +1,508 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// Public names and identities printed by Core's hash and check --print-sigs commands, and the SDK's text bounds. +internal static class Identities +{ + /// The longest name, in chars, the SDK looks up; a longer caller name is refused before it is looked up, hashed, or converted. + internal const int NameLimit = 4096; + + /// The longest excerpt of caller text quoted in a message, in chars. + internal const int ExcerptLimit = 64; + + /// + /// The deepest type a signature may nest (a type with no arguments has + /// depth 1): the protocol's value-depth limit (max_json_depth, 64), + /// since no deeper value could be passed. + /// + internal const int SignatureDepth = 64; + + /// The most types one signature may name. + internal const int SignatureNodes = 4096; + + /// The most tokens one signature may have, which bounds the parser's work before any type is counted. + internal const int SignatureTokens = 4 * SignatureNodes; + + /// One public name and identity printed by jacquard hash. + internal sealed record Entry(string Name, Hash Hash); + + /// for a message: whole when short, otherwise its first 64 chars and its length. + internal static string Excerpt(string? text) + { + if (text is null) + { + return "null"; + } + + if (text.Length <= ExcerptLimit) + { + return text; + } + + int end = char.IsHighSurrogate(text[ExcerptLimit - 1]) ? ExcerptLimit - 1 : ExcerptLimit; + return string.Concat(text.AsSpan(0, end), "... (" + text.Length.ToString(CultureInfo.InvariantCulture) + " chars)"); + } + + /// + /// for a message: its type and an excerpt of its + /// message, never its whole text. Reading a caller's exception message + /// runs the caller's code; a failure there is described by type alone. + /// + [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1031", Justification = "A caller's Message property may throw anything; the description falls back to the type.")] + internal static string Describe(Exception error) + { + string? message; + try + { + message = error.Message; + } + catch (Exception) + { + message = null; + } + + string type = error.GetType().FullName ?? error.GetType().Name; + return message is null ? type : type + ": " + Excerpt(message); + } + + /// The refusal of a caller's name longer than , quoting an excerpt. + internal static string TooLong(string what, string name) => + what + " name " + Excerpt(name) + " exceeds " + NameLimit.ToString(CultureInfo.InvariantCulture) + " chars"; + + /// , refused when it is longer than , before it is looked up or hashed. + /// A binding failure quoting an excerpt. + internal static string Bounded(string name, string what) => + name.Length > NameLimit ? throw JacquardException.Binding(TooLong(what, name)) : name; + + /// + /// Converts a display name such as WorkToday to its store + /// spelling, work-today; lowercase names are unchanged. A name + /// longer than is refused, never expanded. + /// + /// A binding failure for a name over the limit. + internal static string DisplayToStore(string name) + { + Bounded(name, "a"); + if (name.Length == 0) + { + return name; + } + + // A lone surrogate decodes as U+FFFD, which is not upper case. + _ = Rune.DecodeFromUtf16(name, out Rune first, out _); + if (!Rune.IsUpper(first)) + { + return name; + } + + var output = new StringBuilder(name.Length + 8); + int index = 0; + foreach (Rune rune in name.EnumerateRunes()) + { + if (index > 0 && Rune.IsUpper(rune)) + { + output.Append('-'); + } + + output.Append(Rune.ToLowerInvariant(rune).ToString()); + index++; + } + + return output.ToString(); + } + + /// The form used to look up effects and operations: store spelling, with _ accepted for -. + /// A binding failure for a name over the limit. + internal static string Normal(string name, string what) => DisplayToStore(Bounded(name, what)).Replace('_', '-'); + + /// Parses one N:name hash line, or returns null. + private static (string Index, Entry Entry)? HashLine(string line) + { + int colon = line.IndexOf(':', StringComparison.Ordinal); + if (colon <= 0) + { + return null; + } + + string index = line[..colon]; + string rest = line[(colon + 1)..]; + int space = rest.IndexOf(' ', StringComparison.Ordinal); + if (space < 0) + { + return null; + } + + string name = rest[..space]; + foreach (char c in index) + { + if (c < '0' || c > '9') + { + return null; + } + } + + if (name.Length == 0) + { + return null; + } + + foreach (Rune rune in name.EnumerateRunes()) + { + if (Rune.IsWhiteSpace(rune)) + { + return null; + } + } + + string hex = rest[(space + 1)..]; + return Hash.LowerHex(hex, 64) ? (index, new Entry(name, new Hash(hex))) : null; + } + + /// + /// Parses jacquard hash output into declaration groups: a type or + /// effect identity followed by its constructors or operations, in Core's + /// printed order. + /// + /// A binding failure when the output names no identity. + internal static List> HashGroups(string output) + { + var order = new List(); + var groups = new Dictionary>(StringComparer.Ordinal); + foreach (string line in output.Split('\n')) + { + (string Index, Entry Entry)? parsed = HashLine(line.Trim()); + if (parsed is { } found) + { + if (!groups.TryGetValue(found.Index, out List? group)) + { + group = new List(); + groups[found.Index] = group; + order.Add(found.Index); + } + + group.Add(found.Entry); + } + } + + if (order.Count == 0) + { + throw JacquardException.Binding("Core hash command returned no public identities"); + } + + var result = new List>(order.Count); + foreach (string index in order) + { + result.Add(groups[index]); + } + + return result; + } + + /// Refuses a public name that denotes different identities in different declaration groups. + /// A binding failure naming the ambiguous name. + internal static void CheckGroups(List> groups) + { + var seen = new Dictionary>(StringComparer.Ordinal); + foreach (List group in groups) + { + var current = new Dictionary>(StringComparer.Ordinal); + var names = new List(); + foreach (Entry entry in group) + { + if (!current.TryGetValue(entry.Name, out HashSet? set)) + { + set = new HashSet(); + current[entry.Name] = set; + names.Add(entry.Name); + } + + set.Add(entry.Hash); + } + + foreach (string name in names) + { + if (!seen.TryGetValue(name, out HashSet? previous)) + { + previous = new HashSet(); + seen[name] = previous; + } + + if (previous.Count > 0 && !previous.IsSupersetOf(current[name])) + { + throw JacquardException.Binding("ambiguous public name " + Excerpt(name) + + " occurs in different declaration groups; use distinct names or explicit low-level identities"); + } + + previous.UnionWith(current[name]); + } + } + } + + private static JacquardException TooDeep() => + JacquardException.Binding("signature nests types deeper than " + SignatureDepth.ToString(CultureInfo.InvariantCulture) + + " levels, the protocol's value-depth limit (max_json_depth)"); + + /// Tokens of a printed signature, at most a few per type, so the token count is bounded too. + private static List Tokens(string text) + { + var tokens = new List(); + int at = 0; + int end = text.Length; + while (true) + { + while (at < end && char.IsWhiteSpace(text[at])) + { + at++; + } + + if (at == end) + { + return tokens; + } + + if (tokens.Count >= SignatureTokens) + { + throw JacquardException.Binding("signature is longer than " + SignatureTokens.ToString(CultureInfo.InvariantCulture) + " tokens"); + } + + int length; + char first = text[at]; + if (string.CompareOrdinal(text, at, "->{", 0, 3) == 0) + { + length = 3; + } + else if ("(),{}".Contains(first, StringComparison.Ordinal)) + { + length = 1; + } + else if ((first >= 'a' && first <= 'z') || (first >= 'A' && first <= 'Z')) + { + length = 1; + while (at + length < end) + { + char c = text[at + length]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || ".?!-".Contains(c, StringComparison.Ordinal))) + { + break; + } + + length++; + } + } + else + { + throw JacquardException.Binding("unsupported signature near " + Excerpt(text.Substring(at, Math.Min(12, end - at)))); + } + + tokens.Add(text.Substring(at, length)); + at += length; + } + } + + /// + /// Reads a closed v0 signature printed by check --print-sigs, such as + /// (List Line, Int) ->{Inventory} Option Text. Polymorphic and + /// effect-polymorphic signatures are refused, and so is one with more than + /// tokens, one whose types nest deeper than + /// levels, or one with more than + /// types. The nesting is refused before the + /// parser recurses past it, so no signature can exhaust the stack. + /// + /// A binding failure naming why the signature has no binding. + internal static CallInterface ParseSignature(string text, IReadOnlyDictionary names) => new Parser(Tokens(text), names).Parse(); + + /// + /// Refuses a type deeper than or, with + /// , more than types. + /// It walks with an explicit stack, so a deep type cannot exhaust it. + /// + private static void Measure(BoundaryType type, ref int nodes) + { + var stack = new Stack<(BoundaryType Type, int Level)>(); + stack.Push((type, 1)); + while (stack.Count > 0) + { + (BoundaryType current, int level) = stack.Pop(); + if (level > SignatureDepth) + { + throw TooDeep(); + } + + if (++nodes > SignatureNodes) + { + throw JacquardException.Binding("signature names more than " + SignatureNodes.ToString(CultureInfo.InvariantCulture) + " types"); + } + + IReadOnlyList children = current is NominalType nominal ? nominal.Arguments : ((TupleType)current).Items; + foreach (BoundaryType child in children) + { + stack.Push((child, level + 1)); + } + } + } + + private sealed class Parser + { + private static readonly HashSet Ends = new(StringComparer.Ordinal) { string.Empty, ",", ")", "}", "->{" }; + + private readonly List tokens; + private readonly IReadOnlyDictionary names; + private int index; + private int frames; + + internal Parser(List tokens, IReadOnlyDictionary names) + { + this.tokens = tokens; + this.names = names; + } + + private string Peek() => index < tokens.Count ? tokens[index] : string.Empty; + + private string Take(string expected) + { + string token = Peek(); + if (token.Length == 0 || (expected.Length > 0 && !string.Equals(token, expected, StringComparison.Ordinal))) + { + throw JacquardException.Binding("expected \"" + expected + "\", found \"" + Excerpt(token) + "\""); + } + + index++; + return token; + } + + private Hash Identity(string display) + { + if (names.TryGetValue(DisplayToStore(display), out Hash? hash) || names.TryGetValue(display, out hash)) + { + return hash; + } + + throw JacquardException.Binding("no identity for " + Excerpt(display)); + } + + private List Items(string close) + { + var output = new List(); + if (string.Equals(Peek(), close, StringComparison.Ordinal)) + { + index++; + return output; + } + + while (true) + { + output.Add(Type()); + if (string.Equals(Peek(), ",", StringComparison.Ordinal)) + { + index++; + if (string.Equals(Peek(), close, StringComparison.Ordinal)) + { + index++; + return output; + } + + continue; + } + + Take(close); + return output; + } + } + + private BoundaryType Atom() + { + string token = Take(string.Empty); + if (token == "(") + { + List items = Items(")"); + if (items.Count == 1 && !string.Equals(tokens[index - 2], ",", StringComparison.Ordinal)) + { + return items[0]; + } + + return new TupleType(items); + } + + if (!char.IsLetter(token[0])) + { + throw JacquardException.Binding("unsupported type \"" + Excerpt(token) + "\""); + } + + return new NominalType(Identity(token)); + } + + private BoundaryType Type() + { + // Each nested parse is inside a type one level shallower (Core + // prints no redundant parentheses), so this bounds the recursion + // before it goes further. + if (++frames > SignatureDepth) + { + throw TooDeep(); + } + + BoundaryType head = Atom(); + if (head is NominalType nominal) + { + var arguments = new List(); + while (!Ends.Contains(Peek())) + { + arguments.Add(Atom()); + } + + head = new NominalType(nominal.Identity, arguments); + } + + frames--; + return head; + } + + internal CallInterface Parse() + { + Take("("); + List parameters = Items(")"); + Take("->{"); + var effects = new List(); + if (!string.Equals(Peek(), "}", StringComparison.Ordinal)) + { + while (true) + { + Hash identity = Identity(Take(string.Empty)); + if (!effects.Contains(identity)) + { + effects.Add(identity); + } + + if (!string.Equals(Peek(), ",", StringComparison.Ordinal)) + { + break; + } + + index++; + } + } + + Take("}"); + BoundaryType result = Type(); + if (Peek().Length != 0) + { + throw JacquardException.Binding("trailing signature tokens"); + } + + effects.Sort(); + int nodes = 0; + foreach (BoundaryType parameter in parameters) + { + Measure(parameter, ref nodes); + } + + Measure(result, ref nodes); + return new CallInterface(parameters, result, effects); + } + } +} diff --git a/dotnet/src/JacquardHost/Sdk/Invocation.cs b/dotnet/src/JacquardHost/Sdk/Invocation.cs new file mode 100644 index 0000000..657c526 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Invocation.cs @@ -0,0 +1,292 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// One completed invocation with its raw evidence. +public sealed class Invocation +{ + private readonly Function function; + + internal Invocation(InvocationResult raw, Function function) + { + Raw = raw; + this.function = function; + } + + /// The host's result: Core's frames, the host's frames, and evidence. + public InvocationResult Raw { get; } + + /// Whether Core reported success. + public bool Ok => Raw.Ok; + + /// The primary diagnostic or host code. + public string Code => Raw.Code; + + /// A summary for people. + public string Message => Raw.Message; + + /// The classified failure, or null on success. + public JacquardException? Error => Ok ? null : JacquardException.Invocation(Raw); + + /// Decodes the result. + /// + /// The invocation's failure, or a binding failure keeping the successful + /// raw result when it does not decode; when the caller's codec or record + /// constructor threw, that exception is the inner exception. + /// + public T Decode(Codec codec) + { + ArgumentNullException.ThrowIfNull(codec); + if (Error is { } error) + { + throw error; + } + + Value value = Raw.Value ?? throw JacquardException.Binding(function.Name + ": Core completed without a value", Raw, null); + try + { + return function.Data.Decode(codec, value); + } + catch (JacquardException mapping) + { + // Already bounded, and the SDK's own words. + throw JacquardException.Binding( + JacquardException.Joined(function.Name + ": Core completed, but result mapping failed: ", mapping.Message), + Raw, + mapping.InnerException ?? mapping); + } + } + + /// Decodes the result into . + /// As for . + public Dynamic ToDynamic() => Decode(Codecs.Dynamic); + + /// Lists the outcome and each effect request by operation name. + public string Render() + { + var lines = new List { function.Name + ": " + (Ok ? "ok" : Message) }; + foreach (EffectRequest request in Raw.Requests) + { + lines.Add(" " + request.Ordinal.ToString(CultureInfo.InvariantCulture) + ". " + (function.Data.Label(request.Operation) ?? request.Operation.Hex)); + } + + return string.Join("\n", lines); + } + + /// + /// Test seam: runs with the temporary file's path after it is written + /// and flushed, before it is checked and renamed over the target. Null + /// outside tests. + /// + internal static volatile Action? BeforeRenameHook; + + /// + /// Test seam: reads the status of the temporary file's handle in place + /// of . Null outside tests. + /// + internal static volatile Func? HandleStatHook; + + /// + /// Writes the invocation's frames and host evidence. The record contains + /// argument and result values; it is written only when called. Where the + /// file system has POSIX permissions, the target's directory and every + /// directory above it must be owned by this user or root and not writable + /// by its group or others, unless it is sticky (such as /tmp) or + /// the group is this user's private group, as for the cache's + /// directories (the password database listing that rule needs runs at + /// most once per Save, limited to 5 seconds and then stopped within + /// 750 ms, so at most about 5.75 seconds; Save takes no token, so it is + /// not stopped earlier); otherwise the save is refused, since another user could + /// replace the file before or after it is renamed. The record is written + /// through one handle to a new file, created exclusively (never through + /// a link) in the target's directory, owned by this process's user and + /// owner-only (read and write for its owner), and flushed to disk; the + /// file at its path is checked to be the one written (the same device + /// and inode), and only then renamed over . + /// Whatever was at the path is replaced, never written through: an + /// existing file's permissions and owner are not kept, and a symbolic + /// link there is replaced, not followed. Where the directory is sticky, + /// a file another user owns cannot be replaced and the save fails. + /// + /// The file cannot be written, the target's directory is not safe from other users, the temporary file was replaced before its rename, or the path is longer than any path. + /// The file may not be written. + public void Save(string path) + { + ArgumentNullException.ThrowIfNull(path); + if (path.Length == 0 || path.Contains('\0', StringComparison.Ordinal) || path.Length > Core.PathLimit || Core.AbsoluteLength(path) > Core.PathLimit) + { + throw new IOException(Identities.Excerpt(path) + " is not a path or is longer than any path"); + } + + JsonObject host = Json.Object( + ("evidence", Raw.HostEvidence), + ("exit_status", Raw.ExitStatus is { } status ? Json.Of(status) : Json.Null), + ("signal", Raw.Signal is { } signal ? Json.Of(signal) : Json.Null), + ("stderr_base64", Json.Of(Convert.ToBase64String(Raw.Stderr.ToArray())))); + JsonObject record = Json.Object( + ("core_frames", Json.Of(Raw.CoreFrames)), + ("function", Json.Of(function.Name)), + ("host", host), + ("host_frames", Json.Of(Raw.HostFrames)), + ("schema", Json.Of("jacquard-host-inspection-v1"))); + string text; + try + { + text = JsonCodec.Stringify(record, 128); + } + catch (ProtocolException error) + { + throw new IOException(error.Message, error); + } + + byte[] bytes = Encoding.UTF8.GetBytes(text + "\n"); + string full = Path.GetFullPath(path); + string directory = Path.GetDirectoryName(full) ?? throw new IOException(Identities.Excerpt(path) + " names no file"); + string name = Path.GetFileName(full); + if (name.Length == 0) + { + throw new IOException(Identities.Excerpt(path) + " names no file"); + } + + if (Unix.Supported) + { + // The directory is resolved once and used by its real path, so + // the directories checked are the ones written. + directory = Unix.RealPath(directory); + full = Path.Combine(directory, name); + string? problem; + try + { + // Save takes no token; the password database listing is bounded by its own time limit. + problem = Core.UnsafeDirectories(directory, "save target " + full, new Core.PasswordListing(), CancellationToken.None); + } + catch (JacquardException error) + { + throw new IOException(error.Message, error); + } + + if (problem is not null) + { + throw new IOException(JacquardException.Bounded(problem)); + } + } + + string temporary = Path.Combine(directory, string.Format( + CultureInfo.InvariantCulture, + ".jacquard-save-{0}-{1}", + Environment.ProcessId, + Convert.ToHexString(System.Security.Cryptography.RandomNumberGenerator.GetBytes(8)).ToLowerInvariant())); + bool renamed = false; + Unix.Status written = default; + try + { + // FileMode.CreateNew is O_CREAT|O_EXCL, which never follows a + // link at the path. The one handle is written, flushed, and, + // where there are POSIX permissions, held open until the rename + // (elsewhere an open file cannot be renamed). + var stream = new FileStream(temporary, Core.OwnerOnly(FileMode.CreateNew)); + try + { + if (Unix.Supported) + { + try + { + written = HandleStatHook is { } stat ? stat(stream.SafeFileHandle) : Unix.Stat(stream.SafeFileHandle); + } + catch + { + // The handle's identity is unknown; the path's stands + // in only while it is still an empty regular file of + // this user's, as the one just created exclusively is, + // so that file is removed. + written = JustCreated(temporary); + throw; + } + } + + if (!OperatingSystem.IsWindows()) + { + // Exactly owner-only, whatever the umask left. + File.SetUnixFileMode(stream.SafeFileHandle, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + + stream.Write(bytes); + stream.Flush(flushToDisk: true); + BeforeRenameHook?.Invoke(temporary); + if (Unix.Supported && !Unix.Stat(temporary, follow: false).SameInode(written)) + { + // Not this call's file: it is neither renamed nor removed. + written = default; + throw new IOException("the temporary file for " + Identities.Excerpt(full) + " was replaced before it was renamed; nothing was saved"); + } + + if (!Unix.Supported) + { + stream.Dispose(); + } + + File.Move(temporary, full, overwrite: true); + renamed = true; + } + finally + { + stream.Dispose(); + } + } + finally + { + if (!renamed) + { + RemoveTemporary(temporary, written); + } + } + } + + /// + /// The status of when it is an empty + /// regular file this user owns, as the file just created exclusively + /// there is; default (nothing to remove) otherwise or when it cannot be read. + /// + private static Unix.Status JustCreated(string temporary) + { + try + { + Unix.Status status = Unix.Stat(temporary, follow: false); + return status.Kind == Unix.Kind.Regular && status.Size == 0 && status.Owner == Unix.EffectiveUser() ? status : default; + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + return default; + } + } + + /// + /// Removes the temporary file this call created exclusively, and only + /// that: where it was replaced, or its identity is unknown, whatever is + /// at the path is left behind under its distinctive name. + /// + private static void RemoveTemporary(string temporary, Unix.Status written) + { + try + { + if (!Unix.Supported || (written.Kind == Unix.Kind.Regular && Unix.Stat(temporary, follow: false).SameInode(written))) + { + File.Delete(temporary); + } + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + // Left behind under its distinctive name. + } + } + + /// + public override string ToString() => "Invocation[" + function.Name + ", " + Raw + "]"; +} diff --git a/dotnet/src/JacquardHost/Sdk/JacquardException.cs b/dotnet/src/JacquardHost/Sdk/JacquardException.cs new file mode 100644 index 0000000..6791dfb --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/JacquardException.cs @@ -0,0 +1,191 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Globalization; + +namespace JacquardLang.Host.Sdk; + +/// What an SDK failure is about. +public enum JacquardErrorKind +{ + /// The program could not be checked, hashed, installed, or loaded from its cache. + Preparation, + + /// + /// A mapping or binding problem: an unknown name, a missing handler, or a + /// successful Core result that does not decode into the requested type + /// ( is then set). + /// + Binding, + + /// A local input that cannot be sent; no worker started. + InvalidArgument, + + /// The invocation ran and did not succeed; classifies it. + Invocation, +} + +/// +/// Every SDK failure. says what failed; an invocation that +/// ran keeps its raw . A failure caused by the caller's +/// own code (a codec, a record accessor or constructor) keeps that exception +/// as . The message is at most +/// chars: a longer one keeps its start and notes +/// its whole length. +/// +public sealed class JacquardException : Exception +{ + /// The longest message kept, in chars, its noted length included. + public const int MessageLimit = 8192; + + private static readonly IReadOnlyList Nothing = new ReadOnlyCollection(Array.Empty()); + + private JacquardException(JacquardErrorKind kind, string message, InvocationResult? result, Exception? cause, int batchIndex, IReadOnlyList completed) + : base(Bounded(message), cause) + { + Kind = kind; + Result = result; + BatchIndex = batchIndex; + Completed = completed; + } + + /// What failed. + public JacquardErrorKind Kind { get; } + + /// + /// The raw result, when a worker ran: always for + /// , and for a successful result + /// that did not decode; otherwise null. + /// + public InvocationResult? Result { get; } + + /// How the invocation failed, for ; otherwise null. + public Failure? Failure => InnerException is InvocationException error ? error.Failure : null; + + /// + /// For Function.MapBatches: + /// the index of the batch that failed, or -1 when a batch was refused + /// before any worker started (and for every other failure). + /// + public int BatchIndex { get; } + + /// + /// For MapBatches: the decoded values of the batches that completed + /// before the failed one, in order, read-only. Nothing is retried. + /// + public IReadOnlyList Completed { get; } + + /// within chars, its start kept and its whole length noted. + internal static string Bounded(string? message) => Joined(string.Empty, message); + + /// + /// followed by , in at + /// most chars, noting the whole length once. A + /// message that was already cut keeps its own noted length, plus the + /// prefix, so it is never cut twice or noted wrongly. Nothing longer than + /// the limit is copied. + /// + internal static string Joined(string prefix, string? message) + { + message ??= "null"; + long noted = NotedLength(message); + long total = prefix.Length + (noted < 0 ? message.Length : noted); + if (noted < 0 && total <= MessageLimit) + { + return prefix + message; + } + + int bodyLength = noted < 0 ? message.Length : message.LastIndexOf("... (", StringComparison.Ordinal); + string suffix = "... (" + total.ToString(CultureInfo.InvariantCulture) + " chars)"; + int room = MessageLimit - suffix.Length - prefix.Length; + if (room <= 0) + { + // A prefix past the limit (not one the SDK builds): keep its start. + int keep = MessageLimit - suffix.Length; + if (char.IsHighSurrogate(prefix[keep - 1])) + { + keep--; + } + + return string.Concat(prefix.AsSpan(0, keep), suffix); + } + + int end = Math.Min(room, bodyLength); + if (end > 0 && end < bodyLength && char.IsHighSurrogate(message[end - 1])) + { + end--; + } + + return string.Concat(prefix, message.AsSpan(0, end), suffix); + } + + /// The noted length of a message already cut by , or -1. + private static long NotedLength(string message) + { + // A cut that dropped a high surrogate is one char shorter. + if (message.Length < MessageLimit - 1 || message.Length > MessageLimit || !message.EndsWith(" chars)", StringComparison.Ordinal)) + { + return -1; + } + + int open = message.LastIndexOf("... (", StringComparison.Ordinal); + if (open < 0) + { + return -1; + } + + ReadOnlySpan digits = message.AsSpan(open + 5, message.Length - 7 - (open + 5)); + if (!long.TryParse(digits, NumberStyles.None, CultureInfo.InvariantCulture, out long noted)) + { + return -1; + } + + // No string is longer than int.MaxValue, so a larger note is not the SDK's. + return noted < message.Length || noted > int.MaxValue ? -1 : noted; + } + + internal static JacquardException Preparation(string message) => new(JacquardErrorKind.Preparation, message, null, null, -1, Nothing); + + /// A preparation failure caused by , such as a caller's source list that threw. + internal static JacquardException Preparation(string message, Exception? cause) => new(JacquardErrorKind.Preparation, message, null, cause, -1, Nothing); + + internal static JacquardException Binding(string message) => new(JacquardErrorKind.Binding, message, null, null, -1, Nothing); + + /// A binding failure caused by , such as a caller's codec that threw. + internal static JacquardException Binding(string message, Exception? cause) => new(JacquardErrorKind.Binding, message, null, cause, -1, Nothing); + + /// A successful Core result that could not be mapped; the result is kept. + internal static JacquardException Binding(string message, InvocationResult result, Exception? cause) => + new(JacquardErrorKind.Binding, message, result, cause, -1, Nothing); + + internal static JacquardException Invalid(string message) => new(JacquardErrorKind.InvalidArgument, message, null, null, -1, Nothing); + + /// An invalid argument at a value's location, such as shipping-cost.argument[0]. + internal static JacquardException InvalidAt(string path, string? message, Exception? cause = null) => + new(JacquardErrorKind.InvalidArgument, Joined(path + ": ", message), null, cause, -1, Nothing); + + /// A failed invocation; is not ok. + internal static JacquardException Invocation(InvocationResult result) => + new(JacquardErrorKind.Invocation, result.Message, result, new InvocationException(result), -1, Nothing); + + /// + /// as the failure of batch , + /// after the decoded values of the batches before it, copied read-only. + /// + internal static JacquardException Batch(int index, IReadOnlyList completed, JacquardException error) + { + var copy = new object?[completed.Count]; + for (int i = 0; i < copy.Length; i++) + { + copy[i] = completed[i]; + } + + return new JacquardException( + error.Kind, + Joined("batch " + index.ToString(CultureInfo.InvariantCulture) + ": ", error.Message), + error.Result, + error.InnerException, + index, + new ReadOnlyCollection(copy)); + } +} diff --git a/dotnet/src/JacquardHost/Sdk/Operations.cs b/dotnet/src/JacquardHost/Sdk/Operations.cs new file mode 100644 index 0000000..db01f40 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Operations.cs @@ -0,0 +1,378 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// One effect's service: operation names mapped to .NET handlers. Names are +/// store spellings such as "lookup-stock", or display spellings; +/// _ is accepted for -. Each handler declares a codec per +/// parameter and one for its result, and may take the +/// first, whose +/// is cancelled when the invocation stops or ends: +/// +/// var inventory = new Operations() +/// .On("lookup-stock", Codecs.Text, Codecs.Int, sku => sku == "sensor" ? 20L : 0L) +/// .On("reserve", Codecs.Text, Codecs.Int, Codecs.Unit, (context, sku, units) => +/// { +/// context.CancellationToken.ThrowIfCancellationRequested(); +/// return Unit.Value; +/// }); +/// +/// +/// +/// A request whose arguments do not decode into the parameters, or whose +/// argument count differs, is reported as an outside failure and the handler +/// does not run. Nor does it run once the invocation is cancelled or its +/// lifetime passed while the arguments were decoded. A +/// is reported as that failure; any other +/// exception is reported as completion unknown, without its message. A +/// result with no encoding within the selected limits is reported as an +/// outside failure, after the handler returned. The service is thread-safe. +/// +public sealed class Operations +{ + private readonly SortedDictionary handlers = new(StringComparer.Ordinal); + private readonly object sync = new(); + + /// + /// Test seam: runs on the handler thread after the arguments are decoded + /// and before the stop check that precedes the handler. Null outside tests. + /// + internal static volatile Action? AfterDecodeHook; + + /// A typed handler with its codecs applied. + internal delegate Value Handler(ProgramData data, HandlerContext context, IReadOnlyList arguments); + + /// One parameter's codec, decoding to a boxed value. + private interface ICodecBox + { + object? Decode(Decoder decoder, Value value); + } + + /// How many operations this service binds, without copying them. + internal int Count + { + get + { + lock (sync) + { + return handlers.Count; + } + } + } + + /// + /// The handlers by store-spelled operation name, or null when there are + /// more than : the count is checked before + /// anything is copied, under the same lock as the copy. + /// + internal IReadOnlyDictionary? Snapshot(int limit) + { + lock (sync) + { + return handlers.Count > limit ? null : new ReadOnlyDictionary(new Dictionary(handlers, StringComparer.Ordinal)); + } + } + + private sealed class CodecBox : ICodecBox + { + private readonly Codec codec; + + internal CodecBox(Codec codec) + { + this.codec = codec; + } + + public object? Decode(Decoder decoder, Value value) => codec.Decode(decoder, value); + } + + private static CodecBox Box(Codec codec) => new CodecBox(codec ?? throw new ArgumentNullException(nameof(codec))); + + private Operations Bind(string operation, ICodecBox[] parameters, Codec result, Func invoke) + { + ArgumentNullException.ThrowIfNull(operation); + ArgumentNullException.ThrowIfNull(result); + Handler handler = (data, context, arguments) => + { + if (arguments.Count != parameters.Length) + { + throw HandlerException.OutsideFailed("handler parameter count differs from the request"); + } + + var decoder = new Decoder(data); + object?[] values = new object?[parameters.Length]; + for (int i = 0; i < values.Length; i++) + { + try + { + values[i] = parameters[i].Decode(decoder, arguments[i]); + } + catch (Exception error) when (error is not OutOfMemoryException) + { + throw HandlerException.OutsideFailed("handler argument cannot be mapped"); + } + } + + AfterDecodeHook?.Invoke(); + + // Decoding takes time; no outside action begins once the call stopped. + if (context.IsCancelled) + { + throw HandlerException.OutsideFailed("handler was not started"); + } + + TResult value = invoke(context, values); + try + { + return Encoder.Root(data, "result").Encode(result, value); + } + catch (Exception error) when (error is not OutOfMemoryException) + { + throw HandlerException.OutsideFailed("handler result cannot be represented"); + } + }; + string key = Identities.Normal(operation, "operation"); + lock (sync) + { + handlers[key] = handler; + } + + return this; + } + + /// + /// Serves with a handler of 0 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, Array.Empty(), result, (context, values) => handler()); + } + + /// + /// Serves with a handler that takes the first, of 0 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, Array.Empty(), result, (context, values) => handler(context)); + } + + /// + /// Serves with a handler of 1 decoded parameter, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first) }, result, (context, values) => handler((TA)values[0]!)); + } + + /// + /// Serves with a handler that takes the first, of 1 decoded parameter, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first) }, result, (context, values) => handler(context, (TA)values[0]!)); + } + + /// + /// Serves with a handler of 2 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second) }, result, (context, values) => handler((TA)values[0]!, (TB)values[1]!)); + } + + /// + /// Serves with a handler that takes the first, of 2 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second) }, result, (context, values) => handler(context, (TA)values[0]!, (TB)values[1]!)); + } + + /// + /// Serves with a handler of 3 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third) }, result, (context, values) => handler((TA)values[0]!, (TB)values[1]!, (TC)values[2]!)); + } + + /// + /// Serves with a handler that takes the first, of 3 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third) }, result, (context, values) => handler(context, (TA)values[0]!, (TB)values[1]!, (TC)values[2]!)); + } + + /// + /// Serves with a handler of 4 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Decodes parameter 3. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec fourth, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third), Box(fourth) }, result, (context, values) => handler((TA)values[0]!, (TB)values[1]!, (TC)values[2]!, (TD)values[3]!)); + } + + /// + /// Serves with a handler that takes the first, of 4 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Decodes parameter 3. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec fourth, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third), Box(fourth) }, result, (context, values) => handler(context, (TA)values[0]!, (TB)values[1]!, (TC)values[2]!, (TD)values[3]!)); + } + + /// + /// Serves with a handler of 5 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Decodes parameter 3. + /// Decodes parameter 4. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec fourth, Codec fifth, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third), Box(fourth), Box(fifth) }, result, (context, values) => handler((TA)values[0]!, (TB)values[1]!, (TC)values[2]!, (TD)values[3]!, (TE)values[4]!)); + } + + /// + /// Serves with a handler that takes the first, of 5 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Decodes parameter 3. + /// Decodes parameter 4. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec fourth, Codec fifth, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third), Box(fourth), Box(fifth) }, result, (context, values) => handler(context, (TA)values[0]!, (TB)values[1]!, (TC)values[2]!, (TD)values[3]!, (TE)values[4]!)); + } + + /// + /// Serves with a handler of 6 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Decodes parameter 3. + /// Decodes parameter 4. + /// Decodes parameter 5. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec fourth, Codec fifth, Codec sixth, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third), Box(fourth), Box(fifth), Box(sixth) }, result, (context, values) => handler((TA)values[0]!, (TB)values[1]!, (TC)values[2]!, (TD)values[3]!, (TE)values[4]!, (TF)values[5]!)); + } + + /// + /// Serves with a handler that takes the first, of 6 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Decodes parameter 3. + /// Decodes parameter 4. + /// Decodes parameter 5. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec fourth, Codec fifth, Codec sixth, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third), Box(fourth), Box(fifth), Box(sixth) }, result, (context, values) => handler(context, (TA)values[0]!, (TB)values[1]!, (TC)values[2]!, (TD)values[3]!, (TE)values[4]!, (TF)values[5]!)); + } + + /// + public override string ToString() + { + lock (sync) + { + return "Operations[" + string.Join(", ", handlers.Keys) + "]"; + } + } +} diff --git a/dotnet/src/JacquardHost/Sdk/Options.cs b/dotnet/src/JacquardHost/Sdk/Options.cs new file mode 100644 index 0000000..7b4e75d --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Options.cs @@ -0,0 +1,33 @@ +using System; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// Options for Core discovery and invocation bounds. A null or empty value means unset. +public sealed record RuntimeOptions +{ + /// Every option unset. + public static RuntimeOptions Defaults { get; } = new(); + + /// The Core executable; defaults to $JACQUARD_BIN, then jacquard on the PATH. + public string? Binary { get; init; } + + /// Core's prelude directory; defaults to $JACQUARD_PRELUDE. + public string? Prelude { get; init; } + + /// Bounds each invocation's worker lifetime; null means the host's default (30 seconds). + public TimeSpan? Deadline { get; init; } + + /// The session limits; null selects the hard limits. + public Limits? Limits { get; init; } +} + +/// Options for one preparation. +public sealed record PrepareOptions +{ + /// The default cache directory. + public static PrepareOptions Defaults { get; } = new(); + + /// Holds verified prepared programs; null means . + public string? CacheDir { get; init; } +} diff --git a/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs b/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs new file mode 100644 index 0000000..2a2679b --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs @@ -0,0 +1,337 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using JacquardLang.Host.Carrier; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// A prepared, verified program. Bind its checked functions with +/// . +/// changes the services of functions bound afterwards; +/// bound functions are immutable and thread-safe. +/// +public sealed class PreparedProgram +{ + private readonly object sync = new(); + private readonly SortedDictionary interfaces = new(StringComparer.Ordinal); + private readonly SortedDictionary effects = new(StringComparer.Ordinal); + private readonly SortedDictionary services = new(StringComparer.Ordinal); + + /// Why each checked function without a binding has none. + private readonly SortedDictionary unsupported = new(StringComparer.Ordinal); + + private Host host; + + [SuppressMessage("Design", "CA1031", Justification = "No exception from reading one signature may fail the preparation; the function is left unbound with the reason.")] + internal PreparedProgram(Runtime runtime, string directory, Runtime.Bindings bindings, string executable, Action rootCheck) + { + RootCheck = rootCheck; + Runtime = runtime; + Directory = directory; + var names = new Dictionary(StringComparer.Ordinal); + foreach (Identities.Entry entry in bindings.Names) + { + names[entry.Name] = entry.Hash; + } + + var constructors = new Dictionary(StringComparer.Ordinal); + foreach (Identities.Entry entry in bindings.Constructors) + { + constructors[entry.Name] = entry.Hash; + } + + var environment = new Dictionary(StringComparer.Ordinal); + foreach (DictionaryEntry variable in Environment.GetEnvironmentVariables()) + { + if (variable.Key is string key && variable.Value is string value) + { + environment[key] = value; + } + } + + environment["JACQUARD_PRELUDE"] = runtime.Prelude; + var carrier = new WorkerOptions { Command = new[] { executable }, Environment = environment, Deadline = runtime.Deadline }; + try + { + host = new Host(Path.Combine(directory, "store"), new HostOptions { Carrier = carrier, Limits = runtime.Limits }); + } + catch (ArgumentException error) + { + throw JacquardException.Binding(error.Message); + } + + var used = new HashSet(); + foreach (KeyValuePair signature in bindings.Signatures) + { + try + { + CallInterface parsed = Identities.ParseSignature(signature.Value, names); + used.UnionWith(parsed.Effects); + interfaces[signature.Key] = parsed; + } + catch (JacquardException reason) + { + // Polymorphic, too deep, and other unsupported signatures + // have no binding; the other functions keep theirs. + unsupported[signature.Key] = reason.Message; + } + catch (Exception unexpected) when (unexpected is not OutOfMemoryException) + { + // The parser's bounds make this unreachable; nothing escapes preparation either way. + unsupported[signature.Key] = "signature cannot be read (" + Identities.Describe(unexpected) + ")"; + } + } + + foreach (IReadOnlyList group in bindings.Groups) + { + foreach (Identities.Entry entry in group) + { + if (used.Contains(entry.Hash)) + { + effects[entry.Name] = new EffectGroup(entry.Hash, group.Where(other => !other.Hash.Equals(entry.Hash)).ToList()); + } + } + } + + Data = new ProgramData(names, constructors, bindings.Groups, runtime.Limits); + } + + /// Refuses a call once the cache root is no longer the directory that was checked. + internal Action RootCheck { get; } + + private sealed record EffectGroup(Hash Identity, IReadOnlyList Operations); + + /// The runtime that prepared this program. + public Runtime Runtime { get; } + + /// The verified cache entry. + public string Directory { get; } + + /// The program's names and identities. + public ProgramData Data { get; } + + /// The checked functions that have a binding, by public name. + public IReadOnlyList Functions + { + get + { + lock (sync) + { + return interfaces.Keys.ToList(); + } + } + } + + /// Test seam: whether the calling thread holds the program's lock. + internal bool LockHeld => System.Threading.Monitor.IsEntered(sync); + + /// Test seam: replaces the worker command for functions bound afterwards. + internal void SetWorkerCommand(IReadOnlyList command, IReadOnlyDictionary? environment = null) + { + lock (sync) + { + host = new Host(host.Store, new HostOptions + { + Carrier = new WorkerOptions { Command = command, Environment = environment, Deadline = Runtime.Deadline }, + Limits = host.Limits, + }); + } + } + + /// + /// Configures an effect's service, by effect name such as + /// "Inventory", for every function bound afterwards. + /// + /// A binding failure for an effect no checked function uses. + public PreparedProgram Provide(string effect, Operations operations) + { + ArgumentNullException.ThrowIfNull(effect); + ArgumentNullException.ThrowIfNull(operations); + lock (sync) + { + string key = Identities.Normal(effect, "effect"); + if (!effects.ContainsKey(key)) + { + throw JacquardException.Binding("unknown effect " + Identities.Excerpt(effect) + (unsupported.Count == 0 ? string.Empty + : "; the effects of functions with unsupported signatures (" + string.Join(", ", unsupported.Keys.Select(Identities.Excerpt)) + ") are not bound")); + } + + services[key] = operations; + return this; + } + } + + /// + /// Binds a checked public function with services for its effects, by + /// effect name, overriding . Every operation of each + /// checked root effect must have a handler, and every name a service + /// binds must be an operation of the program. + /// + /// + /// A binding failure for an unknown function or effect, a missing + /// handler, or a name that is no operation. + /// + public Function Function(string name, IReadOnlyDictionary? bindings = null) + { + ArgumentNullException.ThrowIfNull(name); + Identities.Bounded(name, "function"); + bindings ??= new Dictionary(); + + // The caller's dictionary is read here, before the program's lock is + // taken, so its code cannot hold that lock: its count once, and its + // entries only when they are no more than the program's effects, at + // most one past them. The effects are fixed when the program is made. + // What it throws is wrapped with the cause, as for other caller code. + int declared; + var given = new List>(); + try + { + declared = bindings.Count; + if (declared <= effects.Count) + { + foreach (KeyValuePair binding in bindings) + { + given.Add(binding); + if (given.Count > effects.Count) + { + break; + } + } + } + } + catch (Exception error) when (Caller.Wraps(error)) + { + throw JacquardException.Binding(Identities.Excerpt(name) + ": reading the bindings failed: " + Identities.Describe(error), error); + } + + lock (sync) + { + if (!interfaces.TryGetValue(name, out CallInterface? signature)) + { + string? reason = unsupported.TryGetValue(name, out string? why) ? why : null; + throw JacquardException.Binding(Identities.Excerpt(name) + ": absent or unsupported closed v0 signature" + (reason is null ? string.Empty : ": " + reason)); + } + + Hash target = Data.Name(name) ?? throw JacquardException.Binding(Identities.Excerpt(name) + ": no public identity"); + var selected = new SortedDictionary(StringComparer.Ordinal); + foreach (KeyValuePair effect in effects) + { + if (signature.Effects.Contains(effect.Value.Identity)) + { + selected[effect.Key] = effect.Value; + } + } + + if (selected.Count != signature.Effects.Count) + { + throw JacquardException.Binding(name + ": effect metadata is unavailable; use the low-level host package"); + } + + // Each binding must name one of the function's effects, so more + // bindings than effects are refused from the count, before any is read. + if (declared > selected.Count) + { + throw JacquardException.Binding(name + ": " + declared.ToString(CultureInfo.InvariantCulture) + " bindings given, but it uses " + + selected.Count.ToString(CultureInfo.InvariantCulture) + " effect" + (selected.Count == 1 ? string.Empty : "s") + + (selected.Count == 0 ? string.Empty : ": " + string.Join(", ", selected.Keys))); + } + + var chosen = new SortedDictionary(services, StringComparer.Ordinal); + int read = 0; + foreach (KeyValuePair binding in given) + { + // A dictionary whose count understated its entries was not read past the bound. + if (++read > selected.Count) + { + throw JacquardException.Binding(name + ": more bindings than effects"); + } + + ArgumentNullException.ThrowIfNull(binding.Key, nameof(bindings)); + string key = Identities.Normal(binding.Key, "effect"); + if (!selected.ContainsKey(key)) + { + throw JacquardException.Binding(name + " does not use " + Identities.Excerpt(binding.Key)); + } + + chosen[key] = binding.Value ?? throw new ArgumentNullException(nameof(bindings), "operations"); + } + + var operationNames = new HashSet(StringComparer.Ordinal); + foreach (EffectGroup effect in effects.Values) + { + foreach (Identities.Entry operation in effect.Operations) + { + operationNames.Add(operation.Name); + } + } + + var registry = new OperationRegistry(); + var missing = new List(); + foreach (KeyValuePair effect in selected) + { + IReadOnlyDictionary handlers = new Dictionary(); + if (chosen.TryGetValue(effect.Key, out Operations? service)) + { + // Every name a service binds must be an operation of the + // program, so a service binding more is refused from its + // count, before any is copied. + handlers = service.Snapshot(operationNames.Count) + ?? throw JacquardException.Binding(name + ": the service for " + effect.Key + " binds more operations than the program's " + + operationNames.Count.ToString(CultureInfo.InvariantCulture)); + } + + // One service may serve several effects, but every name it + // binds must be some effect's operation. + foreach (string key in handlers.Keys.OrderBy(key => key, StringComparer.Ordinal)) + { + if (!operationNames.Contains(key)) + { + throw JacquardException.Binding(name + ": " + effect.Key + " has no operation " + Identities.Excerpt(key)); + } + } + + foreach (Identities.Entry operation in effect.Value.Operations) + { + if (!handlers.TryGetValue(operation.Name, out Operations.Handler? handler)) + { + missing.Add(effect.Key + "." + operation.Name); + continue; + } + + ProgramData data = Data; + try + { + registry.Bind(effect.Value.Identity, operation.Hash, (context, arguments) => handler(data, context, arguments)); + } + catch (ArgumentException error) + { + throw JacquardException.Binding(name + ": " + error.Message); + } + } + } + + if (missing.Count != 0) + { + throw JacquardException.Binding(name + ": missing handlers: " + string.Join(", ", missing) + + ". Bind every operation in each declared effect before calling"); + } + + return new Function(name, signature, target, registry, host, Data, RootCheck); + } + } + + /// + public override string ToString() + { + lock (sync) + { + return "PreparedProgram[" + Directory + ", functions=" + string.Join(", ", interfaces.Keys) + "]"; + } + } +} diff --git a/dotnet/src/JacquardHost/Sdk/ProgramData.cs b/dotnet/src/JacquardHost/Sdk/ProgramData.cs new file mode 100644 index 0000000..0e55590 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/ProgramData.cs @@ -0,0 +1,375 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// The prepared program's public names and identities, used to map native +/// values. It is immutable once the program is prepared, and thread-safe. +/// +public sealed class ProgramData +{ + internal static readonly ImmutableHashSet Builtins = ImmutableHashSet.Create(StringComparer.Ordinal, "true", "false", "some", "none", "cons", "nil"); + + private readonly Dictionary names; + private readonly Dictionary constructors; + private readonly Dictionary labels = new(); + private readonly Dictionary primitives = new(); + + /// Each declaration group's head identity and its member identities. + private readonly Dictionary> members = new(); + + internal ProgramData(IReadOnlyDictionary names, IReadOnlyDictionary constructors, IReadOnlyList> groups, Limits limits) + { + this.names = new Dictionary(names, StringComparer.Ordinal); + this.constructors = new Dictionary(constructors, StringComparer.Ordinal); + Limits = limits; + foreach (KeyValuePair entry in constructors.OrderBy(pair => pair.Key, StringComparer.Ordinal)) + { + labels[entry.Value] = entry.Key; + } + + foreach (string name in new[] { "int", "real", "text", "bool", "hash", "list", "option" }) + { + if (names.TryGetValue(name, out Hash? identity)) + { + primitives[identity] = name; + } + } + + foreach (IReadOnlyList group in groups) + { + if (group.Count > 0) + { + members[group[0].Hash] = new HashSet(group.Skip(1).Select(entry => entry.Hash)); + } + } + } + + /// The session limits that bound mapped values. + public Limits Limits { get; } + + /// + /// The exact identity of a public type, constructor, effect, or term name + /// in the prepared program and its discovered prelude. + /// + /// A binding failure for an unknown name. + public Hash Identity(string name) + { + ArgumentNullException.ThrowIfNull(name); + return names.TryGetValue(Identities.DisplayToStore(Identities.Bounded(name, "public")), out Hash? identity) + ? identity + : throw JacquardException.Binding("unknown name " + Identities.Excerpt(name)); + } + + /// The public name of a constructor or operation identity, or null. + public string? Label(Hash identity) + { + ArgumentNullException.ThrowIfNull(identity); + return labels.TryGetValue(identity, out string? label) ? label : null; + } + + internal Hash? Name(string name) => names.TryGetValue(name, out Hash? identity) ? identity : null; + + /// A binding failure for a name over the limit. + internal Hash? ConstructorId(string name) => constructors.TryGetValue(Identities.DisplayToStore(Identities.Bounded(name, "constructor")), out Hash? identity) ? identity : null; + + /// A after checking that the constructor exists. + /// + /// A binding failure for an unknown constructor, or for more fields than + /// the v0 hard limit of collection items. + /// + public Dynamic Make(string name, params Dynamic[] fields) + { + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(fields); + if (fields.Length > Limits.Hard.MaxCollectionItems) + { + throw JacquardException.Binding("constructor of " + fields.Length.ToString(CultureInfo.InvariantCulture) + + " fields exceeds the v0 hard limit of " + Limits.Hard.MaxCollectionItems.ToString(CultureInfo.InvariantCulture)); + } + + if (ConstructorId(name) is null) + { + throw JacquardException.Binding("unknown constructor " + Identities.Excerpt(name)); + } + + return new Dynamic.Variant(Identities.DisplayToStore(name), fields); + } + + /// Encodes a native value, bounded by the selected limits. + /// + /// An invalid argument naming the value's location; a codec that throws + /// anything else is reported so too, its exception kept as the inner + /// exception. + /// + public Value Encode(Codec codec, T value) + { + ArgumentNullException.ThrowIfNull(codec); + try + { + return Encoder.Root(this, "value").Encode(codec, value); + } + catch (Exception error) when (Caller.Wraps(error)) + { + throw CodecFailed("value", error); + } + } + + /// A caller's codec that threw while encoding at : an invalid argument quoting an excerpt, the exception kept. + internal static JacquardException CodecFailed(string path, Exception error) => + JacquardException.InvalidAt(path, "its codec failed: " + Identities.Describe(error), error); + + /// + /// Decodes a boundary value into a native value. The value is first + /// measured, within the selected max_value_nodes and nesting + /// limits, so a shared or deep value cannot make decoding unbounded; + /// every value Core sends is within them. + /// + /// + /// A binding failure when it does not decode; a codec that throws anything + /// else is reported so too, its exception kept as the inner exception. + /// + public T Decode(Codec codec, Value value) + { + ArgumentNullException.ThrowIfNull(codec); + ArgumentNullException.ThrowIfNull(value); + Measure(value); + try + { + return codec.Decode(new Decoder(this), value); + } + catch (Exception error) when (Caller.Wraps(error)) + { + throw JacquardException.Binding("value: its codec failed: " + Identities.Describe(error), error); + } + } + + /// + /// Refuses a value with more nodes than max_value_nodes, or nested + /// past the JSON depth, walking it with an explicit stack that never + /// holds more than the node limit. + /// + private void Measure(Value value) + { + int nodes = 0; + var stack = new Stack<(Value Value, int Level)>(); + stack.Push((value, 0)); + while (stack.Count > 0) + { + (Value current, int level) = stack.Pop(); + if (++nodes > Limits.MaxValueNodes) + { + throw JacquardException.Binding("value exceeds max_value_nodes"); + } + + // A value at the top of a frame's payload is at JSON depth 3, and each level adds 2. + if (3L + (2L * level) > Limits.MaxJsonDepth) + { + throw JacquardException.Binding("value exceeds the v0 JSON depth"); + } + + IReadOnlyList children = current switch + { + TupleValue tuple => tuple.Items, + ConstructorValue constructor => constructor.Arguments, + _ => Array.Empty(), + }; + if ((long)nodes + stack.Count + children.Count > Limits.MaxValueNodes) + { + throw JacquardException.Binding("value exceeds max_value_nodes"); + } + + foreach (Value child in children) + { + stack.Push((child, level + 1)); + } + } + } + + internal string? Primitive(BoundaryType expected) => + expected is NominalType nominal && primitives.TryGetValue(nominal.Identity, out string? name) ? name : null; + + internal bool IsNullary(Value value, string name) => + value is ConstructorValue constructor && constructor.Arguments.Count == 0 && string.Equals(name, Label(constructor.Identity), StringComparison.Ordinal); + + /// Walks a proper List, bounded by the collection limit. + /// A binding failure when it is not a proper List within the limit. + internal List ListItems(Value value) + { + var items = new List(); + Value cursor = value; + while (true) + { + if (cursor is ConstructorValue constructor) + { + string? label = Label(constructor.Identity); + if (constructor.Arguments.Count == 2 && label == "cons") + { + if (items.Count >= Limits.MaxCollectionItems) + { + throw JacquardException.Binding("list exceeds the selected limit"); + } + + items.Add(constructor.Arguments[0]); + cursor = constructor.Arguments[1]; + continue; + } + + if (constructor.Arguments.Count == 0 && label == "nil") + { + return items; + } + } + + throw JacquardException.Binding("expected a proper List"); + } + } + + /// + /// Checks that an encoded value has the checked boundary type, as far as + /// the prepared metadata describes it: primitives, Lists, Options, and + /// tuples exactly, and other data by its constructor's declaration group. + /// Called only on measured values, so the walk is bounded by their depth + /// and node count. + /// + [SuppressMessage("Maintainability", "CA1502", Justification = "One case per boundary type, as in the other SDKs.")] + internal void Conform(Value value, BoundaryType expected, string path) + { + if (expected is TupleType tuple) + { + if (value is TupleValue values && values.Items.Count == tuple.Items.Count) + { + for (int i = 0; i < values.Items.Count; i++) + { + Conform(values.Items[i], tuple.Items[i], path + "[" + i.ToString(CultureInfo.InvariantCulture) + "]"); + } + + return; + } + + throw Refuse(path, "a " + tuple.Items.Count.ToString(CultureInfo.InvariantCulture) + "-item tuple"); + } + + var nominal = (NominalType)expected; + BoundaryType? child = nominal.Arguments.Count == 0 ? null : nominal.Arguments[0]; + string? primitive = Primitive(expected); + if (primitive is null) + { + bool known = members.TryGetValue(nominal.Identity, out HashSet? group); + if (value is ConstructorValue constructor && (!known || group!.Contains(constructor.Identity))) + { + return; + } + + throw Refuse(path, "a " + LabelOfType(nominal.Identity) + " value"); + } + + switch (primitive) + { + case "int": + if (value is not IntValue) + { + throw Refuse(path, "an integer"); + } + + break; + case "real": + if (value is not RealValue) + { + throw Refuse(path, "a Real (double)"); + } + + break; + case "text": + if (value is not TextValue) + { + throw Refuse(path, "text"); + } + + break; + case "hash": + if (value is not HashValue) + { + throw Refuse(path, "a Hash"); + } + + break; + case "bool": + if (!IsNullary(value, "true") && !IsNullary(value, "false")) + { + throw Refuse(path, "a bool"); + } + + break; + case "list": + List items; + try + { + items = ListItems(value); + } + catch (JacquardException) + { + throw JacquardException.InvalidAt(path, "expected a List"); + } + + if (child is not null) + { + for (int i = 0; i < items.Count; i++) + { + Conform(items[i], child, path + "[" + i.ToString(CultureInfo.InvariantCulture) + "]"); + } + } + + break; + default: + if (IsNullary(value, "none")) + { + return; + } + + if (value is ConstructorValue some && some.Arguments.Count == 1 && Label(some.Identity) == "some") + { + if (child is not null) + { + Conform(some.Arguments[0], child, path); + } + + return; + } + + throw Refuse(path, "an Option"); + } + } + + private static JacquardException Refuse(string path, string what) => JacquardException.InvalidAt(path, "expected " + what); + + private string LabelOfType(Hash identity) + { + foreach (KeyValuePair entry in names.OrderBy(pair => pair.Key, StringComparer.Ordinal)) + { + if (entry.Value.Equals(identity)) + { + return entry.Key; + } + } + + return identity.Hex; + } +} + +/// Which exceptions from a caller's code the SDK wraps in a . +internal static class Caller +{ + /// + /// Every exception from caller code except the SDK's own (already a + /// ) and running out of memory, which no + /// wrapping could report. + /// + internal static bool Wraps(Exception error) => error is not JacquardException and not OutOfMemoryException; +} diff --git a/dotnet/src/JacquardHost/Sdk/Runtime.cs b/dotnet/src/JacquardHost/Sdk/Runtime.cs new file mode 100644 index 0000000..7ebe02f --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Runtime.cs @@ -0,0 +1,1098 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using JacquardLang.Host.Carrier; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// Prepares programs against one Core installation. A runtime is immutable +/// and thread-safe. The --version check and prelude discovery run once +/// per Core executable and prelude contents in this process, across runtimes +/// and threads. +/// +[SuppressMessage("Naming", "CA1724", Justification = "Runtime names the same concept in every SDK; it lives in JacquardLang.Host.Sdk.")] +public sealed class Runtime +{ + /// Prepared programs live here unless is set. + public const string DefaultCacheDir = ".scratch/jacquard-host-dotnet"; + + /// The most source bytes one preparation reads, separators included. + public const long SourceLimit = 16L << 20; + + /// The most source files one preparation takes; a longer list is refused from its count before any element is read. + public const int SourceFiles = 1024; + + internal const string ProgramSchema = "jacquard-dotnet-program-v1"; + + /// Core's raw outputs, kept as verified files of each cache entry, from which every binding is derived. + internal const string CheckOutput = "core/check.out"; + + internal const string HashOutput = "core/hash.out"; + + /// The most bytes of program.json a load reads. + private const int MetadataLimit = 32 << 20; + + private const int PollMinMs = 1; + private const int PollMaxMs = 10; + + private static readonly string[] PreludeFiles = { "01-prim.jqd", "02-data.jqd", "19-audit.jqd" }; + + /// Version checks and prelude discoveries, by key, shared by every runtime in this process. + private static readonly ConcurrentDictionary Flights = new(StringComparer.Ordinal); + + private static long staging; + + private int commands; + + /// + /// Validates options; Core does not run yet. + /// + /// A binding failure for a missing prelude or invalid options. + public Runtime(RuntimeOptions? options = null) + { + options ??= RuntimeOptions.Defaults; + string? bin = options.Binary; + if (string.IsNullOrEmpty(bin)) + { + bin = Environment.GetEnvironmentVariable("JACQUARD_BIN"); + } + + string? prelude = options.Prelude; + if (string.IsNullOrEmpty(prelude)) + { + prelude = Environment.GetEnvironmentVariable("JACQUARD_PRELUDE"); + } + + if (string.IsNullOrEmpty(prelude)) + { + throw JacquardException.Binding("set JACQUARD_PRELUDE or RuntimeOptions.Prelude; Core and prelude must match the compatibility pin"); + } + + if (prelude.Length > Core.PathLimit || prelude.Contains('\0', StringComparison.Ordinal) || Core.AbsoluteLength(prelude) > Core.PathLimit) + { + throw JacquardException.Binding("prelude " + Identities.Excerpt(prelude) + " is not a path or is longer than any path"); + } + + if (options.Deadline is { } deadline && deadline <= TimeSpan.Zero) + { + throw JacquardException.Binding("deadline must be positive"); + } + + Limits selected = options.Limits ?? Limits.Hard; + try + { + selected.Validate(); + } + catch (ProtocolException error) + { + throw JacquardException.Binding("invalid runtime limits: " + error.Message); + } + + Binary = string.IsNullOrEmpty(bin) ? "jacquard" : bin; + Prelude = Path.GetFullPath(prelude); + Deadline = options.Deadline; + Limits = selected; + } + + /// The Core executable as configured, before it is resolved. + public string Binary { get; } + + /// The absolute prelude directory. + public string Prelude { get; } + + internal TimeSpan? Deadline { get; } + + internal Limits Limits { get; } + + /// Test seam: how long each Core command may run. + internal TimeSpan CommandTimeout { get; set; } = Core.CommandTimeout; + + /// Test seam: how many Core commands this runtime has run. + internal int CoreCommands => Volatile.Read(ref commands); + + /// Test seam: runs on a preparing thread when it starts to wait for another thread's version check or prelude discovery. Null outside tests. + internal static volatile Action? FlightWaitHook; + + /// Test seam: runs after each source file is read, with its path. Null outside tests. + internal static volatile Action? SourceReadHook; + + /// Test seam: runs when a cache entry has been loaded, just before its program is returned. Null outside tests. + internal static volatile Action? LoadedHook; + + /// Test seam: runs with the vacated staging path once an entry is published. Null outside tests. + internal static volatile Action? PublishedHook; + + /// Test seam: runs after the failure cleanup removes each entry, with its path. Null outside tests. + internal static volatile Action? CleanupHook; + + /// Test seam: runs after each Core command succeeds, with its first argument. Null outside tests. + internal static volatile Action? CommandHook; + + /// + /// Test seam: runs just before the cache root's identity is checked + /// again, with the use that follows: stage, publish, + /// load, loaded, or call (a prepared program's + /// worker start). Null outside tests. + /// + internal static volatile Action? RootHook; + + private string Command(string executable, CancellationToken cancellationToken, params string[] arguments) + { + Interlocked.Increment(ref commands); + string output = Core.Run(executable, Prelude, CommandTimeout, cancellationToken, arguments); + CommandHook?.Invoke(arguments[0]); + return output; + } + + /// + /// Refuses the preparation, or a prepared program's call, unless the + /// cache root at is still the directory that was + /// checked (the same device and inode), so a root renamed away and + /// replaced after the check is not used. + /// + private static void SameRoot(string cache, Unix.Status pinned, string use) + { + RootHook?.Invoke(use); + Unix.Status now = Core.StatOrFail(cache, follow: false, "cannot read "); + if (now.Kind != pinned.Kind || now.Device != pinned.Device || now.Inode != pinned.Inode) + { + throw JacquardException.Preparation("cache root " + cache + " was replaced after it was checked; prepare again"); + } + } + + /// One discovery: empty until a run succeeds; a failed run leaves it empty for the next caller. + private sealed class Flight + { + internal IReadOnlyList? Outputs { get; set; } + + internal bool Running { get; set; } + } + + /// + /// Runs once per key. Concurrent callers wait for + /// one run, each observing its own token in sleeps of 1 to 10 ms; the run + /// itself is bounded by the command timeout. A successful result stays + /// cached, and a failed run leaves the next caller to try again. + /// + private static IReadOnlyList Once(string key, Func> work, CancellationToken cancellationToken) + { + Flight flight = Flights.GetOrAdd(key, static _ => new Flight()); + int idle = PollMinMs; + bool waiting = false; + while (true) + { + lock (flight) + { + if (flight.Outputs is { } outputs) + { + return outputs; + } + + if (!flight.Running) + { + flight.Running = true; + break; + } + } + + if (!waiting) + { + FlightWaitHook?.Invoke(); + } + + waiting = true; + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + // poll(2) as a sleep: it neither waits on a .NET handle nor consumes a Thread.Interrupt. + Posix.Poll(Span.Empty, idle); + idle = Math.Min(idle * 2, PollMaxMs); + } + + try + { + IReadOnlyList outputs = work(); + lock (flight) + { + flight.Outputs = outputs; + } + + return outputs; + } + finally + { + lock (flight) + { + flight.Running = false; + } + } + } + + /// + /// Runs --version once per executable fingerprint (its digest and + /// its file identity); accepts output whose last field is the pinned + /// version. A run during which the executable changed is refused, and + /// is not kept for later callers. + /// + private void CheckVersion(string executable, string binaryKey, IReadOnlyList pins, CancellationToken cancellationToken) => + Once("version:" + binaryKey, () => + { + string version = Command(executable, cancellationToken, "--version").Trim(); + Core.Unchanged(pins, cancellationToken); + string[] fields = version.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + if (fields.Length == 0 || !string.Equals(fields[^1], Compat.CoreVersion, StringComparison.Ordinal)) + { + throw JacquardException.Preparation("expected pinned Core version " + Compat.CoreVersion + ", found \"" + + Identities.Excerpt(version) + "\"; the exact release is also required"); + } + + return Array.Empty(); + }, cancellationToken); + + /// + /// Core's hash output for each prelude file, which must parse. + /// Outputs made while the executable or the prelude changed are refused, + /// and are not kept for later callers. + /// + private IReadOnlyList PreludeOutputs(string executable, string key, IReadOnlyList pins, CancellationToken cancellationToken) => + Once("prelude:" + key, () => + { + var outputs = new List(); + foreach (string name in PreludeFiles) + { + string output = Command(executable, cancellationToken, "hash", Path.Combine(Prelude, name)); + Identities.HashGroups(output); + outputs.Add(output); + } + + Core.Unchanged(pins, cancellationToken); + return outputs.AsReadOnly(); + }, cancellationToken); + + /// The bindings of a program: what program.json records beside its identity and files. + internal sealed record Bindings( + IReadOnlyList Names, + IReadOnlyList Constructors, + SortedDictionary Signatures, + IReadOnlyList> Groups) + { + /// Whether two bindings record the same names, constructors, signatures, and groups in the same order. + internal bool Same(Bindings other) => + Names.SequenceEqual(other.Names) + && Constructors.SequenceEqual(other.Constructors) + && Signatures.Count == other.Signatures.Count + && Signatures.SequenceEqual(other.Signatures) + && Groups.Count == other.Groups.Count + && Groups.Zip(other.Groups).All(pair => pair.First.SequenceEqual(pair.Second)); + } + + /// + /// Derives every binding from Core's raw outputs: check --print-sigs + /// for the signatures, and hash of each prelude file and of the + /// source for the names, constructors, and declaration groups. + /// + internal static Bindings Derive(string check, IReadOnlyList preludeHashes, string hash) + { + var signatures = new SortedDictionary(Json.CodePointOrder); + foreach (string line in check.Split('\n')) + { + string trimmed = line.Trim(); + int split = trimmed.IndexOf(" : ", StringComparison.Ordinal); + if (split >= 0) + { + string name = trimmed[..split]; + if (!name.Contains(' ', StringComparison.Ordinal) && !name.Contains('\t', StringComparison.Ordinal)) + { + signatures[name] = trimmed[(split + 3)..]; + } + } + } + + var groups = new List>(); + foreach (string output in preludeHashes) + { + groups.AddRange(Identities.HashGroups(output)); + } + + groups.AddRange(Identities.HashGroups(hash)); + Identities.CheckGroups(groups); + var names = new List(); + var constructorOrder = new List(); + var constructors = new Dictionary(StringComparer.Ordinal); + var seen = new HashSet(StringComparer.Ordinal); + foreach (List group in groups) + { + foreach (Identities.Entry entry in group) + { + if (seen.Add(entry.Name)) + { + names.Add(entry); + } + + if (!constructors.ContainsKey(entry.Name)) + { + constructorOrder.Add(entry.Name); + } + + constructors[entry.Name] = entry.Hash; + } + } + + var constructorList = constructorOrder.Select(name => new Identities.Entry(name, constructors[name])).ToList(); + return new Bindings(names, constructorList, signatures, groups.Select(group => (IReadOnlyList)group).ToList()); + } + + /// + /// Checks, hashes, and installs first-party declaration sources + /// (.jac files, or .jqd files in dependency order) into a + /// verified cache keyed by the source bytes, the Core executable's bytes, + /// every prelude file, and the protocol. Each Core command is bounded by + /// a 30-second timeout and bounded output. Cancelling + /// stops the running command and + /// fails the preparation. + /// + /// The source files; more than is refused from the count alone. + /// The cache directory; null selects the defaults. + /// Stops the preparation. + /// The prepared, verified program. + /// A preparation failure. + /// or one of its elements is null. + public PreparedProgram Prepare(IReadOnlyList sources, PrepareOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(sources); + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + if (!Unix.Supported) + { + throw JacquardException.Preparation("the .NET SDK prepares programs on Linux and macOS only"); + } + + PreparedProgram program; + try + { + program = PrepareChecked(sources, options ?? PrepareOptions.Defaults, cancellationToken); + } + catch (JacquardException error) + { + // Checked here, not in a filter: a filter runs before the + // preparation's cleanup, which may observe the stop. Report the + // stop, not a misleading read or cache failure it caused. + if (cancellationToken.IsCancellationRequested && !error.Message.EndsWith(Core.Aborted, StringComparison.Ordinal)) + { + throw JacquardException.Preparation(Core.Aborted); + } + + throw; + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw JacquardException.Preparation("cannot prepare: " + error.Message); + } + + // A stop during the last of the work is reported, not a program. + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + return program; + } + + private PreparedProgram PrepareChecked(IReadOnlyList sources, PrepareOptions options, CancellationToken cancellationToken) + { + JacquardException Usage() => JacquardException.Preparation("pass .jac files or .jqd files in dependency order"); + + // The caller's list runs the caller's code; what it throws is wrapped with the cause. + static T Read(Func read) + { + try + { + return read(); + } + catch (Exception error) when (Caller.Wraps(error)) + { + throw JacquardException.Preparation("reading the source list failed: " + Identities.Describe(error), error); + } + } + + // The count comes from Count alone, before any element is read. + int count = Read(() => sources.Count); + if (count == 0) + { + throw Usage(); + } + + if (count > SourceFiles) + { + throw JacquardException.Preparation("more than " + SourceFiles.ToString(CultureInfo.InvariantCulture) + " sources"); + } + + string? syntax = null; + + // Each element is read once, so the files read below are the ones checked here. + var checkedSources = new List(count); + for (int i = 0; i < count; i++) + { + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + int index = i; + string source = Core.CheckedPath(Read(() => sources[index]), "source"); + checkedSources.Add(source); + string kind = Extension(source); + syntax ??= kind; + if (!(kind is ".jac" or ".jqd") || !string.Equals(syntax, kind, StringComparison.Ordinal)) + { + throw Usage(); + } + } + + string configured = Path.GetFullPath(Core.CheckedPath(options.CacheDir ?? DefaultCacheDir, "cache directory")); + string executable = Core.Executable(Binary); + string binaryHash = Core.FileDigest(executable, cancellationToken, out Unix.Status binaryStatus); + + // What Core runs is the executable and the prelude as they are when + // each command starts; their identities, read as they are + // fingerprinted, must be unchanged once the commands have finished. + var pins = new List { new("Core executable", executable, Follow: true, binaryStatus) }; + string binaryKey = binaryHash + ":" + Core.IdentityKey(binaryStatus); + CheckVersion(executable, binaryKey, pins.ToArray(), cancellationToken); + string preludeHash; + try + { + preludeHash = Core.MapDigest(Core.Tree(Prelude, Core.Access.Any, cancellationToken, pins)); + } + catch (JacquardException error) + { + throw JacquardException.Preparation("cannot read prelude " + Prelude + ": " + error.Message); + } + + byte[] bytes = ReadSources(checkedSources, cancellationToken); + var identity = new SortedDictionary(Json.CodePointOrder) + { + ["binary"] = binaryHash, + ["prelude"] = preludeHash, + ["protocol"] = Compat.Protocol, + ["schema"] = ProgramSchema, + ["source"] = Core.Digest(bytes), + ["syntax"] = syntax!, + }; + string cache; + + // One password database listing at most for this preparation's checks. + var listing = new Core.PasswordListing(); + try + { + if (OperatingSystem.IsWindows()) + { + Directory.CreateDirectory(configured); + } + else + { + // Each missing level is made owner-only, whatever the umask, + // since no directory above the root may be writable by others. + var missing = new Stack(); + string? level = configured; + for (; level is not null && !Directory.Exists(level); level = Path.GetDirectoryName(level)) + { + missing.Push(level); + } + + // The directories that exist are checked before any level + // is made, so a refused preparation makes nothing. + if (missing.Count > 0 && level is not null && Unix.Supported) + { + Core.CheckDirectories(Unix.RealPath(level), configured, listing, cancellationToken); + } + + while (missing.Count > 0) + { + Directory.CreateDirectory(missing.Pop(), UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + } + + // Every symbolic link in the cache root's path is resolved once, + // here, and the real directory is checked and used from then on; + // nothing inside it is followed. + cache = Unix.Supported ? Unix.RealPath(configured) : new DirectoryInfo(configured).ResolveLinkTarget(returnFinalTarget: true)?.FullName ?? configured; + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException or ArgumentException) + { + throw JacquardException.Preparation("cannot create " + configured + ": " + error.Message); + } + + string directory = Path.Combine(cache, Core.MapDigest(identity)); + + // Only the cache's owner may write it; nothing inside it can resist + // someone else who may. The root is used by its path, so no other + // user may write a directory above it (one who could would rename it + // away and put another directory there), and its identity is checked + // again before each use, a prepared program's worker starts included. + Unix.Status root = Core.Check(cache, Core.Access.OwnerOnly); + Core.CheckAncestors(cache, listing, cancellationToken); + if (Unix.Stat(directory, follow: false).Kind != Unix.Kind.Missing) + { + Core.Unchanged(pins, cancellationToken); + return Load(directory, identity, executable, cache, root, cancellationToken); + } + + SameRoot(cache, root, "stage"); + string stage = StagingDirectory(cache); + bool published = false; + try + { + string snapshot = Path.Combine(stage, "source" + syntax); + Write(snapshot, bytes, "cannot write the source snapshot"); + string check = Command(executable, cancellationToken, "check", snapshot, "--print-sigs"); + IReadOnlyList preludeHashes = PreludeOutputs(executable, binaryKey + preludeHash, pins, cancellationToken); + string hash = Command(executable, cancellationToken, "hash", snapshot); + Bindings bindings = Derive(check, preludeHashes, hash); + string store = Path.Combine(stage, "store"); + Command(executable, cancellationToken, "run", snapshot, "--store", store); + + // Core's commands have finished: what they ran must be what was fingerprinted. + Core.Unchanged(pins, cancellationToken); + CreateDirectory(Path.Combine(stage, "core")); + Write(Path.Combine(stage, CheckOutput), Encoding.UTF8.GetBytes(check), "cannot write Core's output"); + Write(Path.Combine(stage, HashOutput), Encoding.UTF8.GetBytes(hash), "cannot write Core's output"); + for (int i = 0; i < PreludeFiles.Length; i++) + { + Write(Path.Combine(stage, PreludeOutput(PreludeFiles[i])), Encoding.UTF8.GetBytes(preludeHashes[i]), "cannot write Core's output"); + } + + // Core made the store with its own permissions; the entry is owner-only. + var metadata = new Metadata(identity, bindings, Core.Tree(stage, Core.Access.Restrict, cancellationToken)); + Write(Path.Combine(stage, "program.json"), Encoding.UTF8.GetBytes(metadata.ToJson()), "cannot write program.json"); + SameRoot(cache, root, "publish"); + try + { + Directory.Move(stage, directory); + + // The staging path is vacated; whatever appears there later is not this call's. + published = true; + PublishedHook?.Invoke(stage); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + // A concurrent preparation published the same artifact first. + if (Unix.Stat(directory, follow: false).Kind == Unix.Kind.Missing) + { + throw JacquardException.Preparation("cannot publish " + directory + ": " + error.Message); + } + } + } + finally + { + if (!published) + { + RemoveTree(stage, cancellationToken); + } + } + + return Load(directory, identity, executable, cache, root, cancellationToken); + } + + private static string PreludeOutput(string file) => "core/prelude-" + file + ".out"; + + private static string Extension(string source) + { + string name = Path.GetFileName(source); + int dot = name.LastIndexOf('.'); + return dot <= 0 ? string.Empty : name[dot..]; + } + + /// + /// The sources joined by newlines. Each must be a regular file (a + /// symbolic link to one is followed); the total is checked against + /// from the files' sizes before each is read, + /// and no read goes past it. Before each file the token is observed. + /// + private static byte[] ReadSources(List sources, CancellationToken cancellationToken) + { + JacquardException Over() => JacquardException.Preparation("sources exceed " + (SourceLimit >> 20).ToString(CultureInfo.InvariantCulture) + " MiB in total"); + using var content = new MemoryStream(); + long total = 0; + for (int i = 0; i < sources.Count; i++) + { + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + string source = sources[i]; + Unix.Status status = Core.StatOrFail(source, follow: true, "cannot read "); + if (status.Kind != Unix.Kind.Regular) + { + throw JacquardException.Preparation("cannot read " + source + ": not a regular file"); + } + + total += (i > 0 ? 1 : 0) + status.Size; + if (total > SourceLimit) + { + throw Over(); + } + + if (i > 0) + { + content.WriteByte((byte)'\n'); + } + + int room = (int)(SourceLimit - content.Length); + byte[] buffer = new byte[Math.Min(room + 1L, status.Size + 1L)]; + int read = 0; + try + { + using var stream = new FileStream(source, new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Share = FileShare.ReadWrite, BufferSize = 0 }); + while (true) + { + if (read == buffer.Length) + { + // The file grew past its size: read one more byte to see whether it is over. + if (buffer.Length > room) + { + break; + } + + Array.Resize(ref buffer, (int)Math.Min(room + 1L, buffer.Length * 2L)); + } + + int n = stream.Read(buffer, read, buffer.Length - read); + if (n <= 0) + { + break; + } + + read += n; + } + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw JacquardException.Preparation("cannot read " + source + ": " + error.Message); + } + + if (read > room) + { + throw Over(); + } + + content.Write(buffer, 0, read); + SourceReadHook?.Invoke(source); + } + + return content.ToArray(); + } + + private static void CreateDirectory(string path) + { + try + { + if (!Unix.MakeDirectory(path, 0x1C0)) + { + throw JacquardException.Preparation("cannot create " + path + ": it already exists"); + } + } + catch (IOException error) + { + throw JacquardException.Preparation("cannot create " + path + ": " + error.Message); + } + } + + /// Writes a new owner-only file. + private static void Write(string path, byte[] bytes, string context) + { + try + { + using var stream = new FileStream(path, Core.OwnerOnly(FileMode.CreateNew)); + stream.Write(bytes); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw JacquardException.Preparation(context + ": " + error.Message); + } + } + + /// A new owner-only staging directory, created exclusively, so a directory the call did not make is never used or removed. + private static string StagingDirectory(string cache) + { + long stamp = Stopwatch.GetTimestamp() & 0xffffffffL; + for (int attempt = 0; attempt < 100; attempt++) + { + string path = Path.Combine(cache, string.Format( + CultureInfo.InvariantCulture, + "prepare-{0}-{1:x8}-{2}", + Environment.ProcessId, + stamp, + Interlocked.Increment(ref staging))); + try + { + if (Unix.MakeDirectory(path, 0x1C0)) + { + return path; + } + } + catch (IOException error) + { + throw JacquardException.Preparation("cannot create " + path + ": " + error.Message); + } + } + + throw JacquardException.Preparation("cannot create a staging directory"); + } + + /// + /// Removes a staging directory unless it was published (and so no longer + /// exists), in a streamed post-order walk that holds one directory + /// enumerator per level on an explicit stack. It removes at most the + /// tree's entry cap, and ends at the first cancellation, checked before + /// each entry. What it leaves is an unpublished prepare-* + /// directory, which is never loaded as a cache entry: entries are named by + /// their identity digest. + /// + [SuppressMessage("Design", "CA1031", Justification = "Cleanup is best effort; what it cannot remove is left for the user and never becomes a cache entry.")] + internal static void RemoveTree(string root, CancellationToken cancellationToken) + { + int cap = Core.TreeEntryCap; + int removed = 0; + var stack = new Stack<(string Directory, IEnumerator Entries)>(); + try + { + if (Unix.Stat(root, follow: false).Kind != Unix.Kind.Directory) + { + return; + } + + stack.Push((root, Directory.EnumerateFileSystemEntries(root).GetEnumerator())); + while (stack.Count > 0) + { + if (cancellationToken.IsCancellationRequested || removed >= cap) + { + return; + } + + (string directory, IEnumerator entries) = stack.Peek(); + if (!entries.MoveNext()) + { + stack.Pop().Entries.Dispose(); + Directory.Delete(directory, recursive: false); + removed++; + CleanupHook?.Invoke(directory); + continue; + } + + string path = entries.Current; + if (Unix.Stat(path, follow: false).Kind == Unix.Kind.Directory) + { + stack.Push((path, Directory.EnumerateFileSystemEntries(path).GetEnumerator())); + continue; + } + + File.Delete(path); + removed++; + CleanupHook?.Invoke(path); + } + } + catch (Exception) + { + // Left for the user to remove; it never becomes a cache entry. + } + finally + { + while (stack.Count > 0) + { + stack.Pop().Entries.Dispose(); + } + } + } + + /// + /// Loads a cache entry, refusing an accidental or partial change: the + /// entry and everything in it must be owned by this user and writable by + /// no one else, every file but program.json must match the digests + /// it records, and every binding it records must equal what is derived + /// again from Core's raw outputs, which are among those verified files. A + /// coordinated rewrite of several files is accepted: only the cache's + /// owner can make one, and the owner is trusted. + /// + private PreparedProgram Load(string directory, SortedDictionary identity, string executable, string cache, Unix.Status root, CancellationToken cancellationToken) + { + JacquardException Fail(string detail) => JacquardException.Preparation("prepared cache " + directory + + " is incomplete or changed (" + detail + "); remove that cache entry and prepare again"); + + SameRoot(cache, root, "load"); + SortedDictionary observed; + try + { + // Regular files only, bounded, so no read below can block. + observed = Core.Tree(directory, Core.Access.OwnerOnly, cancellationToken); + } + catch (JacquardException error) + { + if (error.Message == Core.Aborted) + { + throw; + } + + throw Fail(error.Message); + } + + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + if (!observed.ContainsKey("program.json")) + { + throw Fail("no program.json"); + } + + Metadata metadata = Metadata.FromJson(ReadVerified(directory, "program.json", null, MetadataLimit, Fail)) ?? throw Fail("unreadable program.json"); + observed.Remove("program.json"); + if (!metadata.Identity.SequenceEqual(identity) || !metadata.Files.SequenceEqual(observed)) + { + throw Fail("cached artifact digests do not match"); + } + + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + Bindings derived; + try + { + var preludeHashes = new List(); + foreach (string file in PreludeFiles) + { + preludeHashes.Add(Text(ReadVerified(directory, PreludeOutput(file), observed, Core.StdoutLimit, Fail), Fail)); + } + + derived = Derive( + Text(ReadVerified(directory, CheckOutput, observed, Core.StdoutLimit, Fail), Fail), + preludeHashes, + Text(ReadVerified(directory, HashOutput, observed, Core.StdoutLimit, Fail), Fail)); + } + catch (JacquardException error) when (!error.Message.StartsWith("prepared cache ", StringComparison.Ordinal)) + { + throw Fail("Core's cached output does not parse: " + error.Message); + } + + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + if (!derived.Same(metadata.Bindings)) + { + throw Fail("program.json bindings differ from Core's cached output"); + } + + // What was read came from the root that was checked. + SameRoot(cache, root, "loaded"); + + var program = new PreparedProgram(this, directory, metadata.Bindings, executable, () => SameRoot(cache, root, "call")); + LoadedHook?.Invoke(); + return program; + } + + /// + /// The bytes of a regular file in a cache entry, not followed if it is a + /// link, at most ; when + /// is given, they must have the digest it records. + /// + private static byte[] ReadVerified(string directory, string name, SortedDictionary? digests, int limit, Func fail) + { + if (digests is not null && !digests.ContainsKey(name)) + { + throw fail("no " + name); + } + + string path = Path.Combine(directory, name); + byte[] bytes; + try + { + Unix.Status status = Unix.Stat(path, follow: false); + if (status.Kind != Unix.Kind.Regular) + { + throw fail(name + " is not a regular file"); + } + + if (status.Size > limit) + { + throw fail(name + " is too long"); + } + + using var stream = new FileStream(path, new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Share = FileShare.ReadWrite, BufferSize = 0 }); + byte[] buffer = new byte[status.Size + 1]; + int read = 0; + int n; + while (read < buffer.Length && (n = stream.Read(buffer, read, buffer.Length - read)) > 0) + { + read += n; + } + + if (read > status.Size) + { + throw fail(name + " changed while it was read"); + } + + bytes = buffer.AsSpan(0, read).ToArray(); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw fail(name + ": " + error.Message); + } + + if (digests is not null && !string.Equals(Core.Digest(bytes), digests[name], StringComparison.Ordinal)) + { + throw fail("cached artifact digests do not match"); + } + + return bytes; + } + + private static string Text(byte[] bytes, Func fail) => Core.Utf8(bytes) ?? throw fail("Core's cached output is not UTF-8"); + + /// The prepared metadata in program.json. + internal sealed record Metadata(SortedDictionary Identity, Bindings Bindings, SortedDictionary Files) + { + private static JsonObject Strings(IEnumerable> map) => + Json.Object(map.Select(pair => new KeyValuePair(pair.Key, Json.Of(pair.Value)))); + + private static JsonArray Pairs(IEnumerable entries) => + Json.Of(entries.Select(entry => (Json)Json.Of(new Json[] { Json.Of(entry.Name), Json.Of(entry.Hash.Hex) }))); + + internal string ToJson() + { + JsonArray groups = Json.Of(Bindings.Groups.Select(group => (Json)Json.Of(group.Select(entry => + (Json)Json.Object(("hash", Json.Of(entry.Hash.Hex)), ("name", Json.Of(entry.Name))))))); + JsonObject json = Json.Object( + ("constructors", Pairs(Bindings.Constructors)), + ("files", Strings(Files)), + ("groups", groups), + ("identity", Strings(Identity)), + ("names", Pairs(Bindings.Names)), + ("signatures", Strings(Bindings.Signatures))); + return JsonCodec.Stringify(json, 8); + } + + private static SortedDictionary? Map(JsonObject json, string key) + { + if (json.Get(key) is not JsonObject fields) + { + return null; + } + + var output = new SortedDictionary(Json.CodePointOrder); + foreach (KeyValuePair field in fields) + { + if (field.Value.AsString() is not { } text) + { + return null; + } + + output[field.Key] = text; + } + + return output; + } + + private static Identities.Entry? Entry(string? name, Json? hash) + { + string? hex = hash?.AsString(); + return name is null || hex is null || !Protocol.Hash.LowerHex(hex, 64) ? null : new Identities.Entry(name, new Hash(hex)); + } + + private static List? Pairs(JsonObject json, string key) + { + if (json.Get(key)?.AsArray() is not { } items) + { + return null; + } + + var output = new List(items.Count); + foreach (Json item in items) + { + if (item.AsArray() is not { Count: 2 } pair || Entry(pair[0].AsString(), pair[1]) is not { } entry) + { + return null; + } + + output.Add(entry); + } + + return output; + } + + /// The metadata, or null when it is not what writes. + internal static Metadata? FromJson(byte[] text) + { + JsonObject json; + try + { + json = JsonCodec.DecodeObject(text, 8); + } + catch (ProtocolException) + { + return null; + } + + if (json.Get("groups")?.AsArray() is not { } groupItems) + { + return null; + } + + var groups = new List>(groupItems.Count); + foreach (Json group in groupItems) + { + if (group.AsArray() is not { } entries) + { + return null; + } + + var parsed = new List(entries.Count); + foreach (Json entry in entries) + { + if (entry is not JsonObject fields || Entry(fields.Get("name")?.AsString(), fields.Get("hash")) is not { } value) + { + return null; + } + + parsed.Add(value); + } + + groups.Add(parsed); + } + + SortedDictionary? identity = Map(json, "identity"); + List? names = Pairs(json, "names"); + List? constructors = Pairs(json, "constructors"); + SortedDictionary? signatures = Map(json, "signatures"); + SortedDictionary? files = Map(json, "files"); + if (identity is null || names is null || constructors is null || signatures is null || files is null) + { + return null; + } + + return new Metadata(identity, new Bindings(names, constructors, signatures, groups), files); + } + } + + /// + public override string ToString() => "Runtime[binary=" + Identities.Excerpt(Binary) + ", prelude=" + Prelude + "]"; +} diff --git a/dotnet/src/JacquardHost/Sdk/Unix.cs b/dotnet/src/JacquardHost/Sdk/Unix.cs new file mode 100644 index 0000000..751ede0 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Unix.cs @@ -0,0 +1,546 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; + +namespace JacquardLang.Host.Sdk; + +/// +/// The C library calls the SDK makes beyond the carrier's poll(2): +/// kill(2) to stop a Core command's process group (or freeze and +/// kill the command where there is none), geteuid(2) +/// and a status call (statx(2) on Linux, stat(2) and +/// lstat(2) on macOS) for the cache's owner check, for file kinds and +/// sizes, and for the identities that detect a replaced file, +/// access(2) for the execute permission, mkdir(2) to +/// create a staging directory exclusively, realpath(3) to resolve +/// the cache root's path once, and getpwuid_r(3), +/// getgrgid_r(3), and on Linux lgetxattr(2) (for an access +/// ACL) to recognize a directory writable only by the user's private group +/// (the password database is listed by getent passwd in another +/// process, so no enumeration position is shared). +/// +/// +/// The macOS calls use the 64-bit-inode struct stat layout; they are +/// compiled and selected at run time but, like the rest of the .NET host on +/// macOS, not exercised by the tests or CI. +/// +internal static partial class Unix +{ + internal const int Sigkill = 9; + + /// SIGSTOP on Linux (macOS numbers it 17; the SDK sends it only on Linux). + internal const int Sigstop = 19; + + private const int Enoent = 2; + private const int Enotdir = 20; + private const int Eexist = 17; + private const int AtFdcwd = -100; + private const int AtSymlinkNofollow = 0x100; + private const int AtEmptyPath = 0x1000; + private const int Erange = 34; + private const int Enodata = 61; + private const int Eopnotsupp = 95; + + /// The most bytes a user or group lookup may use; a larger entry is treated as not found. + internal const int AccountBytes = 64 * 1024; + private const uint StatxBasicStats = 0x7ff; + private const int FileTypeMask = 0xF000; + private const int RegularType = 0x8000; + private const int DirectoryType = 0x4000; + private const int LinkType = 0xA000; + + /// A file's kind. + internal enum Kind + { + Missing, + Regular, + Directory, + Link, + Other, + } + + /// + /// What the status call reports: the kind, size, owner, and permission + /// bits, the file's identity: its device and inode, and its + /// modification and status-change times in nanoseconds; and its group. + /// + internal readonly record struct Status(Kind Kind, long Size, uint Owner, int Mode, ulong Device = 0, ulong Inode = 0, long Modified = 0, long Changed = 0, uint Group = 0) + { + /// Whether is the same file: kind, device, and inode. + internal bool SameInode(Status other) => Kind == other.Kind && Device == other.Device && Inode == other.Inode; + + /// + /// Whether describes the same file in the + /// same state: kind, device, inode, size, mode, owner, and both times. + /// Replacing the file changes its inode; writing it, renaming it, or + /// changing its mode or owner changes its status-change time. + /// + internal bool SameFile(Status other) => + Kind == other.Kind && Device == other.Device && Inode == other.Inode && Size == other.Size + && Mode == other.Mode && Owner == other.Owner && Modified == other.Modified && Changed == other.Changed; + } + + [LibraryImport("libc", EntryPoint = "kill", SetLastError = true)] + private static partial int NativeKill(int pid, int signal); + + [LibraryImport("libc", EntryPoint = "geteuid")] + private static partial uint NativeGeteuid(); + + [LibraryImport("libc", EntryPoint = "access", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static partial int NativeAccess(string path, int mode); + + [LibraryImport("libc", EntryPoint = "realpath", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr NativeRealpath(string path, IntPtr resolved); + + [LibraryImport("libc", EntryPoint = "free")] + private static partial void NativeFree(IntPtr pointer); + + [LibraryImport("libc", EntryPoint = "mkdir", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static partial int NativeMkdir(string path, uint mode); + + [LibraryImport("libc", EntryPoint = "statx", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static unsafe partial int NativeStatx(int dirfd, string path, int flags, uint mask, byte* buffer); + + [LibraryImport("libc", EntryPoint = "statx", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static unsafe partial int NativeFstatx(int dirfd, string path, int flags, uint mask, byte* buffer); + + [LibraryImport("libc", EntryPoint = "fstat", SetLastError = true)] + private static unsafe partial int MacFstatArm(int descriptor, byte* buffer); + + [LibraryImport("libc", EntryPoint = "fstat$INODE64", SetLastError = true)] + private static unsafe partial int MacFstatIntel(int descriptor, byte* buffer); + + [LibraryImport("libc", EntryPoint = "lgetxattr", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static partial nint NativeLgetxattr(string path, string name, IntPtr value, nuint size); + + [LibraryImport("libc", EntryPoint = "getpwuid_r")] + private static unsafe partial int NativeGetpwuid(uint uid, byte* entry, byte* buffer, nuint length, byte** result); + + [LibraryImport("libc", EntryPoint = "getgrgid_r")] + private static unsafe partial int NativeGetgrgid(uint gid, byte* entry, byte* buffer, nuint length, byte** result); + + [LibraryImport("libc", EntryPoint = "stat", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static unsafe partial int MacStatArm(string path, byte* buffer); + + [LibraryImport("libc", EntryPoint = "lstat", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static unsafe partial int MacLstatArm(string path, byte* buffer); + + [LibraryImport("libc", EntryPoint = "stat$INODE64", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static unsafe partial int MacStatIntel(string path, byte* buffer); + + [LibraryImport("libc", EntryPoint = "lstat$INODE64", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static unsafe partial int MacLstatIntel(string path, byte* buffer); + + /// Whether the SDK's file and process checks can run here: Linux or macOS. + internal static bool Supported => OperatingSystem.IsLinux() || OperatingSystem.IsMacOS(); + + /// This process's effective user id. + internal static uint EffectiveUser() => NativeGeteuid(); + + /// + /// Sends SIGKILL to every process in group . + /// A group id of one or less is never signalled (kill(-1) and + /// kill(0) mean every process and the caller's own group). + /// + internal static void KillGroup(int group) + { + if (group > 1) + { + _ = NativeKill(-group, Sigkill); + } + } + + /// Sends SIGKILL to , never to a pid of one or less. + internal static void Kill(int pid) + { + if (pid > 1) + { + _ = NativeKill(pid, Sigkill); + } + } + + /// Sends to , never to a pid of one or less. + internal static void Signal(int pid, int signal) + { + if (pid > 1) + { + _ = NativeKill(pid, signal); + } + } + + /// Whether this process may execute , as access(path, X_OK) reports. + internal static bool MayExecute(string path) => NativeAccess(path, 1) == 0; + + /// + /// The absolute path of with every symbolic link, + /// ., and .. resolved, as realpath(3) reports it. + /// + /// The path cannot be resolved. + internal static string RealPath(string path) + { + IntPtr resolved = NativeRealpath(path, IntPtr.Zero); + if (resolved == IntPtr.Zero) + { + throw new IOException("realpath failed with errno " + Marshal.GetLastPInvokeError().ToString(CultureInfo.InvariantCulture)); + } + + try + { + return Marshal.PtrToStringUTF8(resolved) ?? throw new IOException("realpath returned no path"); + } + finally + { + NativeFree(resolved); + } + } + + /// Creates with ; false when it already exists. + /// The directory cannot be created for another reason. + internal static bool MakeDirectory(string path, int mode) + { + if (NativeMkdir(path, (uint)mode) == 0) + { + return true; + } + + int error = Marshal.GetLastPInvokeError(); + if (error == Eexist) + { + return false; + } + + throw new IOException("mkdir failed with errno " + error.ToString(CultureInfo.InvariantCulture)); + } + + /// + /// The status of , following a final symbolic link + /// when is set; when + /// it does not exist. + /// + /// The status cannot be read. + internal static unsafe Status Stat(string path, bool follow) + { + byte* buffer = stackalloc byte[256]; + new Span(buffer, 256).Clear(); + int result; + if (OperatingSystem.IsMacOS()) + { + bool arm = RuntimeInformation.ProcessArchitecture == Architecture.Arm64; + result = (follow, arm) switch + { + (true, true) => MacStatArm(path, buffer), + (false, true) => MacLstatArm(path, buffer), + (true, false) => MacStatIntel(path, buffer), + _ => MacLstatIntel(path, buffer), + }; + } + else + { + result = NativeStatx(AtFdcwd, path, follow ? 0 : AtSymlinkNofollow, StatxBasicStats, buffer); + } + + if (result != 0) + { + int error = Marshal.GetLastPInvokeError(); + if (error == Enoent || error == Enotdir) + { + return new Status(Kind.Missing, 0, 0, 0); + } + + throw new IOException("cannot read its status (errno " + error.ToString(CultureInfo.InvariantCulture) + ")"); + } + + return Decode(buffer); + } + + /// The status of the open file (fstat(2)). + /// The status cannot be read. + internal static unsafe Status Stat(SafeHandle handle) + { + byte* buffer = stackalloc byte[256]; + new Span(buffer, 256).Clear(); + bool added = false; + int result; + try + { + handle.DangerousAddRef(ref added); + int descriptor = (int)handle.DangerousGetHandle(); + if (OperatingSystem.IsMacOS()) + { + result = RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? MacFstatArm(descriptor, buffer) : MacFstatIntel(descriptor, buffer); + } + else + { + result = NativeFstatx(descriptor, string.Empty, AtEmptyPath, StatxBasicStats, buffer); + } + + if (result != 0) + { + throw new IOException("cannot read its status (errno " + Marshal.GetLastPInvokeError().ToString(CultureInfo.InvariantCulture) + ")"); + } + } + finally + { + if (added) + { + handle.DangerousRelease(); + } + } + + return Decode(buffer); + } + + private static unsafe Status Decode(byte* buffer) + { + int modeOffset; + int ownerOffset; + int groupOffset; + int sizeOffset; + ulong device; + ulong inode; + long modified; + long changed; + if (OperatingSystem.IsMacOS()) + { + modeOffset = 4; + ownerOffset = 16; + groupOffset = 20; + sizeOffset = 96; + + // st_dev (int32) at 0, st_ino at 8, st_mtimespec at 48, st_ctimespec at 64. + device = *(uint*)buffer; + inode = *(ulong*)(buffer + 8); + modified = Nanoseconds(*(long*)(buffer + 48), *(long*)(buffer + 56)); + changed = Nanoseconds(*(long*)(buffer + 64), *(long*)(buffer + 72)); + } + else + { + modeOffset = 28; + ownerOffset = 20; + groupOffset = 24; + sizeOffset = 40; + + // stx_ino at 32, stx_ctime at 96, stx_mtime at 112 (each a + // 64-bit second and a 32-bit nanosecond), stx_dev_major and + // stx_dev_minor at 136 and 140. + inode = *(ulong*)(buffer + 32); + changed = Nanoseconds(*(long*)(buffer + 96), *(uint*)(buffer + 104)); + modified = Nanoseconds(*(long*)(buffer + 112), *(uint*)(buffer + 120)); + device = ((ulong)*(uint*)(buffer + 136) << 32) | *(uint*)(buffer + 140); + } + + int mode = *(ushort*)(buffer + modeOffset); + uint owner = *(uint*)(buffer + ownerOffset); + uint group = *(uint*)(buffer + groupOffset); + long size = *(long*)(buffer + sizeOffset); + Kind kind = (mode & FileTypeMask) switch + { + RegularType => Kind.Regular, + DirectoryType => Kind.Directory, + LinkType => Kind.Link, + _ => Kind.Other, + }; + return new Status(kind, size, owner, mode & 0xFFF, device, inode, modified, changed, group); + } + + /// + /// Whether , not followed if it is a link, may + /// have a POSIX access ACL beyond its mode bits: on Linux, whether it + /// has the system.posix_acl_access attribute (true when that + /// cannot be read for another reason than its absence); false elsewhere. + /// + internal static bool HasAccessAcl(string path) + { + if (!OperatingSystem.IsLinux()) + { + return false; + } + + if (NativeLgetxattr(path, "system.posix_acl_access", IntPtr.Zero, 0) >= 0) + { + return true; + } + + int error = Marshal.GetLastPInvokeError(); + return error != Enodata && error != Eopnotsupp; + } + + /// A user's name and primary group, from the password database. + internal readonly record struct Account(string Name, uint Group); + + /// A group's name and its listed members (gr_mem), from the group database. + internal sealed record GroupEntry(string Name, IReadOnlyList Members); + + /// One password database entry: the account's name, user id, and primary group. + internal readonly record struct PasswdEntry(string Name, uint Uid, uint Group); + + /// + /// The password database entry of + /// (getpwuid_r(3)); null when there is none, it cannot be read, + /// it needs more than , or libc returns a + /// pointer outside the buffer it was given. + /// + internal static unsafe Account? User(uint uid) + { + if (IntPtr.Size != 8) + { + return null; + } + + byte* entry = stackalloc byte[256]; + for (int length = 1024; length <= AccountBytes; length *= 2) + { + byte[] buffer = new byte[length]; + byte* result = null; + int error; + fixed (byte* data = buffer) + { + error = NativeGetpwuid(uid, entry, data, (nuint)length, &result); + if (error == 0) + { + return result == null ? null : ReadAccount(entry, data, length) is { } read ? new Account(read.Name, read.Group) : null; + } + } + + if (error != Erange) + { + return null; + } + } + + return null; + } + + /// + /// The group database entry of + /// (getgrgid_r(3)); null when there is none, it cannot be read, + /// it needs more than , or libc returns a + /// pointer outside the buffer it was given. + /// + internal static unsafe GroupEntry? GroupOf(uint gid) + { + if (IntPtr.Size != 8) + { + return null; + } + + byte* entry = stackalloc byte[256]; + for (int length = 1024; length <= AccountBytes; length *= 2) + { + byte[] buffer = new byte[length]; + byte* result = null; + int error; + fixed (byte* data = buffer) + { + error = NativeGetgrgid(gid, entry, data, (nuint)length, &result); + if (error == 0) + { + return result == null ? null : ReadGroup(entry, data, length); + } + } + + if (error != Erange) + { + return null; + } + } + + return null; + } + + /// + /// Reads a struct passwd at (LP64 on + /// Linux and macOS: pw_name at 0, pw_uid at 16, + /// pw_gid at 20) whose strings libc put in the + /// bytes at ; null when + /// the name's pointer or its terminating NUL is outside them, or the + /// name is not valid UTF-8. + /// + internal static unsafe PasswdEntry? ReadAccount(byte* entry, byte* data, int length) => + Text(*(IntPtr*)entry, data, length) is { } name ? new PasswdEntry(name, *(uint*)(entry + 16), *(uint*)(entry + 20)) : null; + + /// + /// Reads a struct group at (LP64 on + /// Linux and macOS: gr_name at 0, gr_gid at 16, + /// gr_mem at 24) whose strings and member array libc put in the + /// bytes at ; null when + /// the name, the member array, any slot of it up to its terminating null, + /// or any member's name lies even partly outside them, or a name is not + /// valid UTF-8. + /// + internal static unsafe GroupEntry? ReadGroup(byte* entry, byte* data, int length) + { + if (Text(*(IntPtr*)entry, data, length) is not { } name) + { + return null; + } + + var list = new List(); + byte* members = *(byte**)(entry + 24); + if (members == null) + { + return new GroupEntry(name, list); + } + + byte* end = data + length; + for (byte* slot = members; ; slot += IntPtr.Size) + { + // The slot itself is checked before it is read. + if (slot < data || slot > end - IntPtr.Size) + { + return null; + } + + IntPtr member = *(IntPtr*)slot; + if (member == IntPtr.Zero) + { + return new GroupEntry(name, list); + } + + if (Text(member, data, length) is not { } text) + { + return null; + } + + list.Add(text); + } + } + + private static readonly UTF8Encoding StrictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + + /// + /// The NUL-terminated UTF-8 string at , which + /// must lie with its NUL inside the bytes at + /// and be valid UTF-8 (distinct malformed names + /// would otherwise decode to the same replacement text); null otherwise. + /// + private static unsafe string? Text(IntPtr pointer, byte* data, int length) + { + byte* start = (byte*)pointer; + if (start < data || start >= data + length) + { + return null; + } + + var bytes = new ReadOnlySpan(start, (int)(data + length - start)); + int nul = bytes.IndexOf((byte)0); + if (nul < 0) + { + return null; + } + + try + { + return StrictUtf8.GetString(bytes[..nul]); + } + catch (DecoderFallbackException) + { + return null; + } + } + + private static long Nanoseconds(long seconds, long nanoseconds) => unchecked((seconds * 1_000_000_000L) + nanoseconds); +} diff --git a/dotnet/src/JacquardHost/Sdk/Values.cs b/dotnet/src/JacquardHost/Sdk/Values.cs new file mode 100644 index 0000000..d5c9a3b --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Values.cs @@ -0,0 +1,276 @@ +using System; +using System.Collections.Generic; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// Jacquard's unit, the empty tuple, as a .NET value. +public readonly record struct Unit +{ + /// The unit value. + public static Unit Value => default; + + /// + public override string ToString() => "()"; +} + +/// +/// A Jacquard Option: or Some of a value. +/// Unlike a nullable type, options nest: Option (Option Int) is +/// Option<Option<long>>, and Some(None) stays distinct +/// from None. +/// +/// The value's type. +[System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1000", Justification = "Option.None and Option.Some name the two cases, as Nullable and ImmutableArray do.")] +public readonly struct Option : IEquatable> +{ + private readonly T value; + + private Option(T value) + { + this.value = value; + HasValue = true; + } + + /// The empty option. + public static Option None => default; + + /// Whether this is Some. + public bool HasValue { get; } + + /// The value of Some. + /// The option is None. + public T Value => HasValue ? value : throw new InvalidOperationException("the option is None"); + + /// Some(value). + public static Option Some(T value) => new(value); + + /// Whether two options are both None, or both Some of equal values. + public static bool operator ==(Option left, Option right) => left.Equals(right); + + /// Whether two options differ. + public static bool operator !=(Option left, Option right) => !left.Equals(right); + + /// The value of Some, or for None. + public T GetValueOrDefault(T fallback) => HasValue ? value : fallback; + + /// + public bool Equals(Option other) => HasValue == other.HasValue && (!HasValue || EqualityComparer.Default.Equals(value, other.value)); + + /// + public override bool Equals(object? obj) => obj is Option other && Equals(other); + + /// + public override int GetHashCode() => HasValue ? HashCode.Combine(true, value) : 0; + + /// + public override string ToString() => HasValue ? "Some(" + value + ")" : "None"; +} + +/// Builds values with the type inferred. +public static class Option +{ + /// Some(value). + public static Option Some(T value) => Option.Some(value); +} + +/// +/// Why each .NET numeric type without a Jacquard type is refused. Without +/// these refusals C# would convert a or +/// silently through , a +/// to an Int, and a to a +/// with a different decimal text. The conversions to +/// and the overloads +/// for these types are marked obsolete as errors, so using one does not +/// compile; reached anyway (by reflection), the argument is refused when it +/// is encoded and the Of overload throws. +/// +internal static class NoConversion +{ + internal const string Unsigned64 = "ulong has no Jacquard type and is not converted: Int is a signed 63-bit integer; pass a range-checked long"; + internal const string UnsignedNative = "nuint has no Jacquard type and is not converted: Int is a signed 63-bit integer; pass a range-checked long"; + internal const string Character = "char has no Jacquard type and is not converted: pass a string for Text, or a long for its code"; + internal const string Single = "float is not converted: Real is a double; pass (double)value"; + internal const string HalfFloat = "Half is not converted: Real is a double; pass (double)value"; + internal const string Decimal = "decimal has no Jacquard type and is never converted: pass money as an integer count of minor units (long); Real is a double"; + internal const string Signed128 = "Int128 has no Jacquard type and is not converted: Int is a signed 63-bit integer; pass a range-checked long"; + internal const string Unsigned128 = "UInt128 has no Jacquard type and is not converted: Int is a signed 63-bit integer; pass a range-checked long"; + internal const string Big = "BigInteger has no Jacquard type and is not converted: Int is a signed 63-bit integer; pass a range-checked long"; +} + +/// +/// A function argument with the codec that encodes it. Plain values convert +/// implicitly: and (and the smaller +/// integer types and , which widen exactly) as Int, +/// as Real, as Text, +/// , , a protocol +/// , and . Anything else +/// is passed as codec.Arg(value), as are lists and records. +/// , , , +/// , , , +/// , , and +/// are refused at compile time +/// instead of converting silently: money crosses as an integer count of +/// minor units. +/// +public abstract class Argument +{ + private protected Argument() + { + } + + /// An Int, Core's full 63-bit range; a value outside it is refused. + public static implicit operator Argument(long value) => Codecs.Int.Arg(value); + + /// An Int. + public static implicit operator Argument(int value) => Codecs.Int.Arg(value); + + /// An Int; a widens exactly. + public static implicit operator Argument(uint value) => Codecs.Int.Arg(value); + + /// An Int. + public static implicit operator Argument(short value) => Codecs.Int.Arg(value); + + /// An Int. + public static implicit operator Argument(ushort value) => Codecs.Int.Arg(value); + + /// An Int. + public static implicit operator Argument(sbyte value) => Codecs.Int.Arg(value); + + /// An Int. + public static implicit operator Argument(byte value) => Codecs.Int.Arg(value); + + /// A Real, the exact binary64 bits of the double. + public static implicit operator Argument(double value) => Codecs.Real.Arg(value); + + /// A Text; a lone surrogate is refused. + public static implicit operator Argument(string value) => Codecs.Text.Arg(value); + + /// A Bool. + public static implicit operator Argument(bool value) => Codecs.Bool.Arg(value); + + /// A Hash. + public static implicit operator Argument(Hash value) => Codecs.Hash.Arg(value); + + /// A protocol value, passed through within the same bounds. + public static implicit operator Argument(Value value) => Codecs.Value.Arg(value); + + /// A value by public name. + public static implicit operator Argument(Dynamic value) => Codecs.Dynamic.Arg(value); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Unsigned64, error: true)] + public static implicit operator Argument(ulong value) => new Refused(NoConversion.Unsigned64); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.UnsignedNative, error: true)] + public static implicit operator Argument(nuint value) => new Refused(NoConversion.UnsignedNative); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Character, error: true)] + public static implicit operator Argument(char value) => new Refused(NoConversion.Character); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Single, error: true)] + public static implicit operator Argument(float value) => new Refused(NoConversion.Single); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.HalfFloat, error: true)] + public static implicit operator Argument(Half value) => new Refused(NoConversion.HalfFloat); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Decimal, error: true)] + public static implicit operator Argument(decimal value) => new Refused(NoConversion.Decimal); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Signed128, error: true)] + public static implicit operator Argument(Int128 value) => new Refused(NoConversion.Signed128); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Unsigned128, error: true)] + public static implicit operator Argument(UInt128 value) => new Refused(NoConversion.Unsigned128); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Big, error: true)] + public static implicit operator Argument(System.Numerics.BigInteger value) => new Refused(NoConversion.Big); + + /// encoded with ; the same as codec.Arg(value). + public static Argument Of(Codec codec, T value) + { + ArgumentNullException.ThrowIfNull(codec); + return new Typed(codec, value); + } + + /// An Int. + public static Argument FromInt64(long value) => value; + + /// An Int. + public static Argument FromInt32(int value) => value; + + /// A Real. + public static Argument FromDouble(double value) => value; + + /// A Text. + public static Argument FromString(string value) => value; + + /// A Bool. + public static Argument FromBoolean(bool value) => value; + + /// A Hash. + public static Argument FromHash(Hash value) => value; + + /// A protocol value. + public static Argument FromValue(Value value) => value; + + /// A value by public name. + public static Argument FromDynamic(Dynamic value) => value; + + /// Encodes the argument at 's location. + internal abstract Value Encode(Encoder encoder); + + private sealed class Typed : Argument + { + private readonly Codec codec; + private readonly T value; + + internal Typed(Codec codec, T value) + { + this.codec = codec; + this.value = value; + } + + internal override Value Encode(Encoder encoder) => encoder.Encode(codec, value); + } + + /// A value of a type with no Jacquard type, refused when it is encoded, before any worker starts. + private sealed class Refused : Argument + { + private readonly string reason; + + internal Refused(string reason) + { + this.reason = reason; + } + + internal override Value Encode(Encoder encoder) => throw encoder.Invalid(reason); + } +} + +/// One batch's rows as the final List argument, encoded from the caller's list without copying it. +internal sealed class Rows : Argument +{ + private readonly Codec codec; + private readonly IReadOnlyList rows; + private readonly int start; + private readonly int count; + + internal Rows(Codec codec, IReadOnlyList rows, int start, int count) + { + this.codec = codec; + this.rows = rows; + this.start = start; + this.count = count; + } + + internal override Value Encode(Encoder encoder) => encoder.List(codec, rows, start, count); +} diff --git a/dotnet/test/JacquardHost.Tests/DocumentTests.cs b/dotnet/test/JacquardHost.Tests/DocumentTests.cs index d860cf0..eaaa3f5 100644 --- a/dotnet/test/JacquardHost.Tests/DocumentTests.cs +++ b/dotnet/test/JacquardHost.Tests/DocumentTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; @@ -15,18 +16,26 @@ private static string DotnetRoot() => typeof(DocumentTests).Assembly.GetCustomAttributes().Single(attribute => attribute.Key == "DotnetRoot").Value!; [Test] - public void TheReadmeExampleIsTheCompiledOne() + public void TheReadmeExamplesAreTheCompiledOnes() { - // The README's example is Readme/Echo.cs, which a live test runs - // against Core, minus its namespace line. + // The README's examples are the files below under Readme/, in this + // order, minus their namespace lines; the live tests run them. + string[] sources = { "Shipping.cs", "Incidents.cs", "Documents.cs", "Echo.cs" }; string readme = File.ReadAllText(Path.Combine(DotnetRoot(), "README.md")); - int start = readme.IndexOf("```csharp\n", StringComparison.Ordinal); - Assert.That(start, Is.GreaterThanOrEqualTo(0), "README.md has no csharp example"); - int end = readme.IndexOf("```", start + 10, StringComparison.Ordinal); - string example = readme[(start + 10)..end]; - string source = File.ReadAllText(Path.Combine(DotnetRoot(), "test", "JacquardHost.Tests", "Readme", "Echo.cs")); - string body = source[(source.IndexOf('\n', source.IndexOf("namespace ", StringComparison.Ordinal)) + 1)..].TrimStart(); - Assert.That(example, Is.EqualTo(body), "README.md's example differs from test/JacquardHost.Tests/Readme/Echo.cs"); + var examples = new List(); + for (int start = readme.IndexOf("```csharp\n", StringComparison.Ordinal); start >= 0; start = readme.IndexOf("```csharp\n", start + 10, StringComparison.Ordinal)) + { + int end = readme.IndexOf("```", start + 10, StringComparison.Ordinal); + examples.Add(readme[(start + 10)..end]); + } + + Assert.That(examples, Has.Count.EqualTo(sources.Length), "README.md's csharp examples"); + for (int i = 0; i < sources.Length; i++) + { + string source = File.ReadAllText(Path.Combine(DotnetRoot(), "test", "JacquardHost.Tests", "Readme", sources[i])); + string body = source[(source.IndexOf('\n', source.IndexOf("namespace ", StringComparison.Ordinal)) + 1)..].TrimStart(); + Assert.That(examples[i], Is.EqualTo(body), "README.md's example " + (i + 1) + " differs from test/JacquardHost.Tests/Readme/" + sources[i]); + } } [Test] diff --git a/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj b/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj index 15e13ae..c645ca4 100644 --- a/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj +++ b/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj @@ -8,9 +8,13 @@ JacquardLang.Host.Tests false true + + true 8.0 - - $(NoWarn);CA1707 + + $(NoWarn);CA1707;CA1416 @@ -29,6 +33,7 @@ + diff --git a/dotnet/test/JacquardHost.Tests/Readme/Documents.cs b/dotnet/test/JacquardHost.Tests/Readme/Documents.cs new file mode 100644 index 0000000..21e4e05 --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Readme/Documents.cs @@ -0,0 +1,26 @@ +namespace JacquardLang.Host.Tests.Readme; + +using System.Collections.Generic; +using JacquardLang.Host; +using JacquardLang.Host.Sdk; + +public static class Documents +{ + // summarize : (Text) ->{Documents} Text, where Documents has one operation, + // read-document : (Text) -> Text. + public static Function Summarize(PreparedProgram program) + { + Operations documents = new Operations().On("read-document", Codecs.Text, Codecs.Text, (context, key) => + { + // The context reports a cancelled or ended invocation; its + // CancellationToken is cancelled then too. + if (context.IsCancelled) + { + throw HandlerException.CompletionUnknown("cancelled"); + } + + return key == "example" ? "A synthetic document supplied by .NET." : throw HandlerException.Refused("unknown document"); + }); + return program.Function("summarize", new Dictionary { ["Documents"] = documents }); + } +} diff --git a/dotnet/test/JacquardHost.Tests/Readme/Incidents.cs b/dotnet/test/JacquardHost.Tests/Readme/Incidents.cs new file mode 100644 index 0000000..28a2ef1 --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Readme/Incidents.cs @@ -0,0 +1,54 @@ +namespace JacquardLang.Host.Tests.Readme; + +using System.Collections.Generic; +using JacquardLang.Host.Sdk; + +// type Incident = | Incident(id: Text, service: Text, severity: Int, age: Int, visible: Bool, owner: Option Text) +// type Urgency = | PageNow | WorkToday | WatchOnly +// type Triage = | Routed Text Text Urgency (List Text) | InvalidIncident Text +public static class Incidents +{ + public sealed record Incident(string Id, string Service, long Severity, long Age, bool Visible, string? Owner); + + public enum Urgency + { + PageNow, + WorkToday, + WatchOnly, + } + + public abstract record Triage; + + public sealed record Routed(string Id, string Team, Urgency Urgency, IReadOnlyList Reasons) : Triage; + + public sealed record InvalidIncident(string Id) : Triage; + + // A record is one constructor: its fields in order, each with a codec and + // a reader, and a function that builds it from the decoded fields. + public static readonly RecordCodec IncidentCodec = Codecs.Record( + "Incident", + Codecs.Field("id", Codecs.Text, (Incident incident) => incident.Id), + Codecs.Field("service", Codecs.Text, (Incident incident) => incident.Service), + Codecs.Field("severity", Codecs.Int, (Incident incident) => incident.Severity), + Codecs.Field("age", Codecs.Int, (Incident incident) => incident.Age), + Codecs.Field("visible", Codecs.Bool, (Incident incident) => incident.Visible), + Codecs.Field("owner", Codecs.Nullable(Codecs.Text), (Incident incident) => incident.Owner), + (id, service, severity, age, visible, owner) => new Incident(id, service, severity, age, visible, owner)); + + // An enumeration's members name nullary constructors: PageNow is page-now. + // A union's cases are records, enumerations, or unions of its subtypes. + public static readonly UnionCodec TriageCodec = Codecs.Union( + Codecs.Record( + "Routed", + Codecs.Field("id", Codecs.Text, (Routed routed) => routed.Id), + Codecs.Field("team", Codecs.Text, (Routed routed) => routed.Team), + Codecs.Field("urgency", Codecs.Enum(), (Routed routed) => routed.Urgency), + Codecs.Field("reasons", Codecs.List(Codecs.Text), (Routed routed) => routed.Reasons), + (id, team, urgency, reasons) => new Routed(id, team, urgency, reasons)), + Codecs.Record("InvalidIncident", Codecs.Field("id", Codecs.Text, (InvalidIncident invalid) => invalid.Id), id => new InvalidIncident(id))); + + // triage.batch : (TriagePolicy, List Incident) ->{} List Triage, run in + // explicit batches of eight; each batch is checked before any runs. + public static IReadOnlyList Route(Function batch, Dynamic policy, IReadOnlyList incidents) => + batch.MapBatches(TriageCodec, IncidentCodec, incidents, 8, new Argument[] { policy }); +} diff --git a/dotnet/test/JacquardHost.Tests/Readme/Shipping.cs b/dotnet/test/JacquardHost.Tests/Readme/Shipping.cs new file mode 100644 index 0000000..8cf8f3a --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Readme/Shipping.cs @@ -0,0 +1,22 @@ +namespace JacquardLang.Host.Tests.Readme; + +using JacquardLang.Host.Sdk; + +public static class Shipping +{ + public static long Cost(string rules, string cacheDir, string inspection) + { + // Core is $JACQUARD_BIN and $JACQUARD_PRELUDE unless RuntimeOptions say otherwise. + var runtime = new Runtime(); + PreparedProgram program = runtime.Prepare(new[] { rules }, new PrepareOptions { CacheDir = cacheDir }); + Function shipping = program.Function("shipping-cost"); + + // A long converts to an Int argument; Call decodes the result. + long free = shipping.Call(Codecs.Int, 75L); + + // Invoke keeps the raw frames and evidence, saved only on request. + Invocation invocation = shipping.Invoke(35L); + invocation.Save(inspection); + return invocation.Decode(Codecs.Int) + free; + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs new file mode 100644 index 0000000..0b02ba6 --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs @@ -0,0 +1,431 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; +using JacquardLang.Host.Carrier; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// +/// Function preflight with synthetic identities and no Core: arguments are +/// counted before they are encoded, a caller's codec that throws is an +/// invalid argument, and in MapBatches a cancellation ends validation +/// between batches and after the last, there is no validation deadline, and +/// no batch's encoding is kept once it is checked. Rows go through a codec +/// that counts its encodings, so each bound is an exact count. The worker +/// command does not exist, so a call that gets past validation fails at +/// batch 0. +/// +[TestFixture] +public sealed class BatchesTests +{ + private string root = string.Empty; + + [SetUp] + public void Prepare() => root = SdkSupport.Scratch("batches"); + + [TearDown] + public void CleanUp() + { + Function.ConformHook = null; + Invocation.BeforeRenameHook = null; + Invocation.HandleStatHook = null; + SdkSupport.NoAccounts(); + SdkSupport.RemoveScratch(root); + } + + private Host MissingWorker() => + new(Path.Combine(root, "store"), new HostOptions { Carrier = new WorkerOptions { Command = new[] { Path.Combine(root, "no-such-jacquard") } } }); + + /// ints : (List Int) ->{} List Int over the data fixture. + private Function Ints() + { + ProgramData data = DataTests.Fixture(Limits.Hard); + NominalType list = DataTests.Nominal(data.Name("list")!, DataTests.Nominal(data.Name("int")!)); + var signature = new CallInterface(new BoundaryType[] { list }, list, Array.Empty()); + return new Function("ints", signature, DataTests.Numbered(99), new OperationRegistry(), MissingWorker(), data); + } + + /// Encodes Ints, counting each encoding and running the action returns for the count. + private sealed class Counting : Codec + { + private readonly Func each; + + internal Counting(Func each) + { + this.each = each; + } + + internal int Encodes { get; private set; } + + public override Value Encode(Encoder encoder, long value) + { + each(++Encodes)?.Invoke(); + return Codecs.Int.Encode(encoder, value); + } + + public override long Decode(Decoder decoder, Value value) => Codecs.Int.Decode(decoder, value); + } + + private static DataTests.RepeatList Rows(int count) => new(1L, count); + + /// + /// The argument count is checked from the count alone: a million + /// arguments to a one-parameter function, or a million leading arguments + /// to a batch, are refused before one is copied or encoded. + /// + [Test] + public void TheWrongArgumentCountIsRefusedBeforeAnyArgumentIsEncoded() + { + var codec = new Counting(_ => null); + Argument[] many = Enumerable.Repeat(codec.Arg(1L), 1_000_000).ToArray(); + JacquardException call = Support.Throws(() => Ints().Invoke(many)); + Assert.That(call.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(call.Message, Is.EqualTo("ints: expected 1 arguments, received 1000000")); + JacquardException batches = Support.Throws(() => Ints().MapBatches(Codecs.Int, codec, Rows(10), 5, many)); + Assert.That(batches.Message, Is.EqualTo("ints: expected 1 arguments, received 1000001")); + Assert.That(codec.Encodes, Is.EqualTo(0), "arguments were encoded before the count was checked"); + } + + /// A caller's codec that throws is a local refusal, as any other: no worker starts, and its exception is kept. + [Test] + public void ACodecThatThrowsIsAnInvalidArgument() + { + var broken = new Counting(_ => () => throw new InvalidOperationException("broken codec")); + JacquardException error = Support.Throws(() => Ints().Invoke(Codecs.List(broken).Arg(new[] { 1L }))); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(error.Message, Is.EqualTo("ints.argument[0]: its codec failed: System.InvalidOperationException: broken codec")); + Assert.That(error.InnerException, Is.InstanceOf()); + JacquardException nothing = Support.Throws(() => Ints().Invoke(new Argument[] { null! })); + Assert.That(nothing.Message, Does.StartWith("ints.argument[0]: null has no Jacquard value")); + } + + [Test] + public void AnAlreadyCancelledCallEncodesNoBatch() + { + var codec = new Counting(_ => null); + using var cancel = new CancellationTokenSource(); + cancel.Cancel(); + JacquardException error = Support.Throws(() => Ints().MapBatches(Codecs.Int, codec, Rows(10_000), 10, cancellationToken: cancel.Token)); + Assert.That(codec.Encodes, Is.EqualTo(0), "rows were encoded after the stop"); + Assert.That(error.BatchIndex, Is.EqualTo(-1)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Invocation), error.Message); + Assert.That(Support.Reason(error.Result!), Is.EqualTo("aborted")); + Assert.That(error.Failure, Is.EqualTo(Failure.WorkerLost)); + Assert.That(error.Completed, Is.Empty); + } + + [Test] + public void ACancelDuringValidationStopsAtTheNextBatch() + { + using var cancel = new CancellationTokenSource(); + + // Cancelled while batch 2 (rows 21 to 30) is being encoded. + var codec = new Counting(n => n == 25 ? cancel.Cancel : null); + JacquardException error = Support.Throws(() => Ints().MapBatches(Codecs.Int, codec, Rows(10_000), 10, cancellationToken: cancel.Token)); + Assert.That(codec.Encodes, Is.EqualTo(30)); + Assert.That(error.BatchIndex, Is.EqualTo(-1)); + Assert.That(Support.Reason(error.Result!), Is.EqualTo("aborted")); + } + + /// + /// A cancellation while the last batch is validated is observed before + /// any worker starts: the refusal is the host's aborted result at batch + /// index -1, and no batch is encoded again to run. + /// + [Test] + public void AStopDuringTheLastBatchsValidationStartsNoWorker() + { + using var cancel = new CancellationTokenSource(); + var codec = new Counting(n => n == 30 ? cancel.Cancel : null); + JacquardException error = Support.Throws(() => Ints().MapBatches(Codecs.Int, codec, Rows(30), 10, cancellationToken: cancel.Token)); + Assert.That(codec.Encodes, Is.EqualTo(30), "a batch was encoded again after the stop"); + Assert.That(error.BatchIndex, Is.EqualTo(-1)); + Assert.That(Support.Reason(error.Result!), Is.EqualTo("aborted")); + } + + /// + /// Validation has no deadline of its own (as in the other SDKs), so it + /// does not depend on the machine's speed: every batch is validated, and + /// the call goes on to the worker (here a missing executable, reported + /// at batch 0), encoding batch 0 again to run it. + /// + [Test] + public void EveryBatchIsValidatedAndTheFirstEncodedAgainToRun() + { + var codec = new Counting(_ => null); + JacquardException error = Support.Throws(() => Ints().MapBatches(Codecs.Int, codec, Rows(30), 10)); + Assert.That(error.BatchIndex, Is.EqualTo(0), error.Message); + Assert.That(Support.Reason(error.Result!), Is.EqualTo("worker_spawn_failed")); + Assert.That(codec.Encodes, Is.EqualTo(40)); + } + + /// + /// Retention is structurally impossible: the validation step returns + /// nothing, so no caller can keep a batch's encoding, and no field of a + /// function can hold one. + /// + [Test] + public void ValidationKeepsNoEncoding() + { + MethodInfo validate = typeof(Function).GetMethod("Validate", BindingFlags.NonPublic | BindingFlags.Instance)!; + Assert.That(validate.ReturnType, Is.EqualTo(typeof(void))); + foreach (FieldInfo field in typeof(Function).GetFields(BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public)) + { + Assert.That( + field.FieldType == typeof(string) || (!typeof(IEnumerable).IsAssignableFrom(field.FieldType) && !typeof(Value).IsAssignableFrom(field.FieldType)), + Is.True, + "a field could hold encodings: " + field); + } + } + + /// Invokes a one-parameter function of with a constant-time codec returning , failing the call if the type check is reached. + private JacquardException RefusedBeforeTheTypeCheck(ProgramData data, BoundaryType type, Value value) + { + var function = new Function("deep", new CallInterface(new[] { type }, type, Array.Empty()), DataTests.Numbered(98), new OperationRegistry(), MissingWorker(), data); + bool conformed = false; + Function.ConformHook = () => + { + conformed = true; + throw new InvalidOperationException("the type check ran"); + }; + Codec constant = Codecs.Create((encoder, ignored) => value, (decoder, ignored) => 0); + JacquardException error = Support.Throws(() => function.Invoke(constant.Arg(0))); + Assert.That(conformed, Is.False, "the type check ran before the value was measured"); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument), error.Message); + return error; + } + + /// + /// A constant-time custom codec returns a small shared graph standing for + /// 2^28 leaves, and it is measured before its type is checked, under the + /// shared budgets, so the type check never runs. As 28 levels of a + /// two-item tuple it is refused by the node budget; as 28 levels of a + /// two-item list it is refused by the JSON depth bound first. + /// + [Test] + public void ASharedGraphFromACustomCodecIsMeasuredBeforeItsTypeIsChecked() + { + ProgramData data = DataTests.Fixture(Limits.Hard); + BoundaryType tupleType = DataTests.Nominal(data.Name("int")!); + Value tuple = new IntValue(1); + for (int i = 0; i < 28; i++) + { + tupleType = new TupleType(tupleType, tupleType); + tuple = new TupleValue(tuple, tuple); + } + + Assert.That(RefusedBeforeTheTypeCheck(data, tupleType, tuple).Message, Does.EndWith("value exceeds max_value_nodes; use smaller independent batches")); + Hash list = data.Name("list")!; + Hash cons = data.ConstructorId("cons")!; + var nil = new ConstructorValue(data.ConstructorId("nil")!, Array.Empty()); + BoundaryType listType = DataTests.Nominal(data.Name("int")!); + Value lists = new IntValue(1); + for (int i = 0; i < 28; i++) + { + listType = DataTests.Nominal(list, listType); + lists = new ConstructorValue(cons, new[] { lists, new ConstructorValue(cons, new[] { lists, nil }) }); + } + + Assert.That(RefusedBeforeTheTypeCheck(data, listType, lists).Message, Does.EndWith("value exceeds the v0 JSON depth; use smaller independent batches")); + } + + [Test] + public void CompletedResultsAreKeptReadOnly() + { + JacquardException error = JacquardException.Batch(1, new object?[] { Unit.Value, null }, JacquardException.Invalid("later")); + Assert.That(error.BatchIndex, Is.EqualTo(1)); + Assert.That(error.Completed, Is.EqualTo(new object?[] { Unit.Value, null })); + Assert.That(error.Message, Is.EqualTo("batch 1: later")); + Assert.That(((IList)error.Completed).IsReadOnly, Is.True); + Support.Throws(() => ((IList)error.Completed).Add(1L)); + } + + /// MapBatches needs a pure function with a final List parameter and a List result; a batch size must be positive. + [Test] + public void MapBatchesNeedsAListFunction() + { + ProgramData data = DataTests.Fixture(Limits.Hard); + NominalType integer = DataTests.Nominal(data.Name("int")!); + var scalar = new Function("one", new CallInterface(new BoundaryType[] { integer }, integer, Array.Empty()), DataTests.Numbered(97), new OperationRegistry(), MissingWorker(), data); + Assert.That(Support.Throws(() => scalar.MapBatches(Codecs.Int, Codecs.Int, Rows(3), 1)).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(Support.Throws(() => Ints().MapBatches(Codecs.Int, Codecs.Int, Rows(3), 0)).Message, Is.EqualTo("batch size must be positive")); + } + + /// Rows whose count drops from 10 to 3 once is called. + private sealed class Shrinking : IReadOnlyList + { + private bool shrunk; + + public int Count => shrunk ? 3 : 10; + + public long this[int index] => 1L; + + internal void Shrink() => shrunk = true; + + public IEnumerator GetEnumerator() => Enumerable.Repeat(1L, Count).GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// + /// The number of rows is read once, so batches come from that count; a + /// list that shrinks while the batches are validated is refused before a + /// worker starts, instead of making a batch of negative size. + /// + [Test] + public void RowsThatShrinkAreRefused() + { + var rows = new Shrinking(); + var codec = new Counting(n => n == 1 ? (Action)rows.Shrink : null); + JacquardException error = Support.Throws(() => Ints().MapBatches(Codecs.Int, codec, rows, 4)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(error.Message, Is.EqualTo("rows changed from 10 to 3 during MapBatches")); + Assert.That(error.BatchIndex, Is.EqualTo(-1)); + Assert.That(codec.Encodes, Is.EqualTo(4)); + } + + /// + /// Save writes a new owner-only file and renames it over the target: an + /// existing world-writable file becomes owner-only (0600) with the + /// record, a symbolic link at the target is replaced by the file while + /// the link's target is left unchanged, and no temporary file is left. + /// + [Test] + [Platform("Linux")] + public void SaveReplacesTheTargetWithANewOwnerOnlyFile() + { + Invocation invocation = Ints().Invoke(Argument.Of(Codecs.List(Codecs.Int), new long[] { 1 })); + Assert.That(Support.Reason(invocation.Raw), Is.EqualTo("worker_spawn_failed")); + const UnixFileMode Open = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.OtherRead | UnixFileMode.OtherWrite; + string existing = Path.Combine(root, "existing.json"); + File.WriteAllText(existing, "old"); + File.SetUnixFileMode(existing, Open); + invocation.Save(existing); + Assert.That(File.GetUnixFileMode(existing), Is.EqualTo(UnixFileMode.UserRead | UnixFileMode.UserWrite)); + Assert.That(File.ReadAllText(existing), Does.StartWith("{").And.EndWith("}\n")); + + string victim = Path.Combine(root, "victim.txt"); + File.WriteAllText(victim, "keep"); + File.SetUnixFileMode(victim, Open); + string link = Path.Combine(root, "link.json"); + File.CreateSymbolicLink(link, victim); + invocation.Save(link); + Assert.That(new FileInfo(link).LinkTarget, Is.Null, "the link was followed"); + Assert.That(File.GetUnixFileMode(link), Is.EqualTo(UnixFileMode.UserRead | UnixFileMode.UserWrite)); + Assert.That(File.ReadAllText(link), Is.EqualTo(File.ReadAllText(existing))); + Assert.That(File.ReadAllText(victim), Is.EqualTo("keep")); + Assert.That(File.GetUnixFileMode(victim), Is.EqualTo(Open)); + Assert.That(Directory.EnumerateFiles(root).Select(Path.GetFileName), Is.EquivalentTo(new[] { "existing.json", "victim.txt", "link.json" })); + } + + /// + /// Save refuses a target directory another user could write (0777, not + /// sticky), as for the cache's directories, before it writes anything + /// there; a sticky one (0777 with the sticky bit, as /tmp is) is used. + /// + [Test] + [Platform("Linux")] + public void SaveRefusesATargetDirectoryOthersMayWrite() + { + Invocation invocation = Ints().Invoke(Argument.Of(Codecs.List(Codecs.Int), new long[] { 1 })); + string shared = Directory.CreateDirectory(Path.Combine(root, "shared")).FullName; + File.SetUnixFileMode(shared, (UnixFileMode)0x1FF); + string target = Path.Combine(shared, "record.json"); + IOException error = Support.Throws(() => invocation.Save(target))!; + Assert.That(error.Message, Is.EqualTo( + "save target " + target + " could be replaced by another user through the directories above it." + + " Make each of these writable only by you (for example chmod go-w DIR), or use a location under directories only you or root may write: " + + shared + " (writable by others)")); + Assert.That(Directory.EnumerateFileSystemEntries(shared), Is.Empty); + + File.SetUnixFileMode(shared, (UnixFileMode)0x3FF); + invocation.Save(target); + Assert.That(Directory.EnumerateFileSystemEntries(shared).Select(Path.GetFileName), Is.EqualTo(new[] { "record.json" })); + } + + /// + /// The temporary file is written through the one handle that created it, + /// and the file at its path is checked to be that file (the same device + /// and inode) just before the rename. A seam replaces the temporary path + /// with another file before the rename, as someone who may write the + /// directory could: the replacement is detected, it is neither renamed + /// over the target nor removed, and nothing is saved. + /// + [Test] + [Platform("Linux")] + public void SaveDetectsATemporaryFileReplacedBeforeItsRename() + { + Invocation invocation = Ints().Invoke(Argument.Of(Codecs.List(Codecs.Int), new long[] { 1 })); + string target = Path.Combine(root, "record.json"); + string? temporary = null; + Invocation.BeforeRenameHook = path => + { + temporary = path; + File.Delete(path); + File.WriteAllText(path, "planted"); + }; + IOException error = Support.Throws(() => invocation.Save(target))!; + Assert.That(error.Message, Is.EqualTo("the temporary file for " + target + " was replaced before it was renamed; nothing was saved")); + Assert.That(File.Exists(target), Is.False, "the replacement was installed"); + Assert.That(File.ReadAllText(temporary!), Is.EqualTo("planted"), "the replacement was removed"); + + // Without the replacement, the record is saved and nothing else is left. + File.Delete(temporary!); + Invocation.BeforeRenameHook = null; + invocation.Save(target); + Assert.That(File.ReadAllText(target), Does.StartWith("{").And.EndWith("}\n")); + Assert.That(Directory.EnumerateFiles(root).Select(Path.GetFileName), Is.EquivalentTo(new[] { "record.json" })); + } + + /// + /// Save applies the private-group rule with its password database scan: + /// a 0770 target directory whose group is the user's private group, + /// listing no members, is refused when another account has that group as + /// its primary group (nothing is written), and used when only the user + /// has it. The account answers come from a seam. + /// + [Test] + [Platform("Linux")] + public void SaveRefusesAPrivateGroupDirectoryAnotherAccountHasAsItsPrimaryGroup() + { + Invocation invocation = Ints().Invoke(Argument.Of(Codecs.List(Codecs.Int), new long[] { 1 })); + string shared = Directory.CreateDirectory(Path.Combine(root, "shared")).FullName; + File.SetUnixFileMode(shared, (UnixFileMode)0x1F8); + uint group = Unix.Stat(shared, follow: false).Group; + string target = Path.Combine(shared, "record.json"); + SdkSupport.Accounts(shared, "me", Array.Empty(), new Unix.PasswdEntry("attacker", Unix.EffectiveUser() + 1, group)); + IOException error = Support.Throws(() => invocation.Save(target))!; + Assert.That(error.Message, Is.EqualTo( + "save target " + target + " could be replaced by another user through the directories above it." + + " Make each of these writable only by you (for example chmod go-w DIR), or use a location under directories only you or root may write: " + + shared + " (writable by its group, which is not your private group)")); + Assert.That(Directory.EnumerateFileSystemEntries(shared), Is.Empty); + + SdkSupport.Accounts(shared, "me", Array.Empty()); + invocation.Save(target); + Assert.That(Directory.EnumerateFileSystemEntries(shared).Select(Path.GetFileName), Is.EqualTo(new[] { "record.json" })); + } + + /// + /// When the status of the new temporary file's handle cannot be read, + /// the error is raised and the file, still the empty one just created + /// exclusively, is removed by its path: nothing is left behind. + /// + [Test] + [Platform("Linux")] + public void SaveRemovesItsTemporaryFileWhenTheHandleCannotBeRead() + { + Invocation invocation = Ints().Invoke(Argument.Of(Codecs.List(Codecs.Int), new long[] { 1 })); + string target = Path.Combine(root, "record.json"); + Invocation.HandleStatHook = handle => throw new IOException("fstat failed"); + Assert.That(Support.Throws(() => invocation.Save(target))!.Message, Is.EqualTo("fstat failed")); + Assert.That(Directory.EnumerateFiles(root), Is.Empty); + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs new file mode 100644 index 0000000..5afc236 --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs @@ -0,0 +1,624 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using JacquardLang.Host.Carrier; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// +/// Core commands are bounded: their exit status, bounded output, and added +/// environment are reported; a timeout or a cancellation kills the command +/// and its process group; a background child is killed when the command +/// exits, and an unrelated process holding the output is not; nothing a +/// descendant holds can hold the caller; and no descriptor is left behind. +/// +[TestFixture] +[Platform("Linux")] +public sealed class CoreCommandTests +{ + private static readonly TimeSpan Minute = TimeSpan.FromMinutes(1); + private string root = string.Empty; + + [SetUp] + public void Prepare() => root = SdkSupport.Scratch("command"); + + [TearDown] + public void CleanUp() + { + CoreCommand.WithoutGroups = false; + CoreCommand.RecordDescendants = true; + CoreCommand.RecordedHook = null; + CoreCommand.StartedHook = null; + CoreCommand.ExitedHook = null; + CoreCommand.KillGroupAtExit = true; + CoreCommand.KillGroupOnStop = true; + CoreCommand.ProcRoot = "/proc"; + CoreCommand.ProcEntryCap = CoreCommand.ProcEntries; + CoreCommand.ProcByteCap = CoreCommand.ProcBytes; + CoreCommand.StatReadHook = null; + CoreCommand.ClockHook = null; + CoreCommand.SignalHook = null; + CoreCommand.ExitWaitHook = null; + foreach (string file in Directory.EnumerateFiles(root, "*.pid")) + { + Support.Kill(int.Parse(File.ReadAllText(file).Trim(), CultureInfo.InvariantCulture)); + } + + SdkSupport.RemoveScratch(root); + } + + private static bool HasSetsid => CoreCommand.Setsid is not null; + + private string PidPath(string name) => Path.Combine(root, name + ".pid"); + + private static CoreCommand.Output Run(string script, TimeSpan? timeout = null, CancellationToken cancellationToken = default, int stdoutLimit = 1 << 20, int stderrLimit = 4096) => + CoreCommand.Run(SdkSupport.Sh(script), null, timeout ?? Minute, stdoutLimit, stderrLimit, cancellationToken); + + [Test] + public void TheExitStatusOutputAndAddedEnvironmentAreReported() + { + CoreCommand.Output output = CoreCommand.Run( + SdkSupport.Sh("echo out; echo err >&2; echo \"$JQ_ADDED\"; exit 3"), + new Dictionary { ["JQ_ADDED"] = "added" }, + Minute, + 1 << 20, + 4096, + default); + Assert.That(output.ExitStatus, Is.EqualTo(3)); + Assert.That(SdkSupport.Text(output.Stdout), Is.EqualTo("out\nadded\n")); + Assert.That(SdkSupport.Text(output.Stderr), Is.EqualTo("err\n")); + Assert.That(output.StdoutOverflow, Is.False); + + // The command's stdin reads end of file at once. + Assert.That(SdkSupport.Text(Run("cat; echo done").Stdout), Is.EqualTo("done\n")); + } + + [Test] + public void OutputBeyondTheLimitsIsReadAndCountedButNotKept() + { + CoreCommand.Output output = Run("head -c 3000000 /dev/zero; head -c 100000 /dev/zero >&2", stdoutLimit: 1000, stderrLimit: 10); + Assert.That(output.ExitStatus, Is.EqualTo(0)); + Assert.That(output.Stdout, Has.Length.EqualTo(1000)); + Assert.That(output.StdoutOverflow, Is.True); + Assert.That(output.Stderr, Has.Length.EqualTo(10)); + } + + [Test] + public void ACommandThatDoesNotExitInTimeIsKilledWithItsChild() + { + CarrierException error = Support.Throws(() => + Run("sleep 300 & echo $! > '" + PidPath("child") + "'; wait", TimeSpan.FromMilliseconds(500)))!; + Assert.That(error.IsTimeout, Is.True, error.Message); + SdkSupport.Dies("the timed-out command's child", SdkSupport.PidFile(PidPath("child"))); + } + + [Test] + public void CancellationKillsTheCommandAndItsChild() + { + using var cancel = new CancellationTokenSource(); + Thread canceller = Support.After(TimeSpan.Zero, () => + { + SdkSupport.PidFile(PidPath("child")); + cancel.Cancel(); + }); + CarrierException error = Support.Throws(() => + Run("sleep 300 & echo $! > '" + PidPath("child") + "'; wait", cancellationToken: cancel.Token))!; + Assert.That(error.IsAborted, Is.True, error.Message); + SdkSupport.Dies("the cancelled command's child", SdkSupport.PidFile(PidPath("child"))); + canceller.Join(); + } + + /// Under setsid the command keeps its pid (no fork) and leads its own process group. + [Test] + public void TheCommandKeepsItsPidAndLeadsItsOwnGroup() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + int started = 0; + CoreCommand.StartedHook = pid => started = pid; + string[] fields = SdkSupport.Text(Run("echo $$; cut -d')' -f2 /proc/$$/stat | cut -d' ' -f4,5").Stdout).Split('\n'); + Assert.That(int.Parse(fields[0], CultureInfo.InvariantCulture), Is.EqualTo(started), "setsid forked"); + string[] ids = fields[1].Split(' '); + Assert.That(ids, Is.EqualTo(new[] { started.ToString(CultureInfo.InvariantCulture), started.ToString(CultureInfo.InvariantCulture) }), "pgid and sid"); + } + + /// + /// sh -c 'sleep 300 & echo $!' exits at once, leaving a child + /// that holds its output; the group kill at the exit ends it, and the + /// command still succeeds, as in Rust. With that kill disabled (a test + /// seam) the child survives: the kill is what ends it. + /// + [Test] + public void ABackgroundChildIsKilledWhenTheCommandExits() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + CoreCommand.Output output = Run("sleep 300 & echo $!"); + Assert.That(output.ExitStatus, Is.EqualTo(0)); + SdkSupport.Dies("the background child", int.Parse(SdkSupport.Text(output.Stdout).Trim(), CultureInfo.InvariantCulture)); + + CoreCommand.KillGroupAtExit = false; + output = Run("sleep 300 & echo $!"); + int survivor = int.Parse(SdkSupport.Text(output.Stdout).Trim(), CultureInfo.InvariantCulture); + File.WriteAllText(PidPath("survivor"), survivor + "\n"); + Assert.That(Support.Alive(survivor), Is.True, "without the exit kill the child should survive"); + } + + /// + /// Once the leader has exited and been reaped, a live process holding its + /// pid means the id was reused: the old group is empty, and the group + /// kill sends nothing. Here an unrelated group leader stands for the + /// process that took the pid after a wraparound; without the check the + /// kill would end its group. + /// + [Test] + public void AGroupIdReusedAfterTheLeaderIsReapedIsNotKilled() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + using Process leader = Process.Start(new ProcessStartInfo("/bin/true") { UseShellExecute = false })!; + leader.WaitForExit(); + Assert.That(leader.HasExited, Is.True); + string pidFile = PidPath("reused"); + using Process other = Process.Start(new ProcessStartInfo("/bin/sh", new[] { "-c", "exec " + CoreCommand.Setsid + " sh -c 'echo $$ > \"" + pidFile + "\"; exec sleep 300'" }) + { + UseShellExecute = false, + })!; + int unrelated = SdkSupport.PidFile(pidFile); + CoreCommand.KillGroup(unrelated, leader); + Thread.Sleep(200); + Assert.That(Support.Alive(unrelated), Is.True, "a group whose id was reused was killed"); + } + + /// A child that closed its output and keeps running is in the group, so it is killed at the exit too. + [Test] + public void ABackgroundChildThatClosedItsOutputIsKilled() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + CoreCommand.Output output = Run("sleep 300 /dev/null 2>&1 & echo $! > '" + PidPath("quiet") + "'"); + Assert.That(output.ExitStatus, Is.EqualTo(0)); + SdkSupport.Dies("the quiet child", SdkSupport.PidFile(PidPath("quiet"))); + } + + /// + /// A process outside the command's group that holds the command's + /// output is neither killed nor waited for: the command returns, and the + /// holder lives on. + /// + [Test] + public void AnUnrelatedProcessHoldingTheOutputIsNotKilled() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + Process? holder = null; + CoreCommand.StartedHook = pid => + { + // Started by the test, not the command: it opens the command's stdout pipe through /proc. + holder = Process.Start(new ProcessStartInfo("/bin/sh", new[] { "-c", "exec sleep 300 >>/proc/" + pid.ToString(CultureInfo.InvariantCulture) + "/fd/1" }) + { + UseShellExecute = false, + }); + }; + try + { + CoreCommand.Output output = Support.Within(Minute, () => Run("sleep 0.5; echo done")); + Assert.That(SdkSupport.Text(output.Stdout), Is.EqualTo("done\n")); + Assert.That(holder, Is.Not.Null); + Assert.That(Support.Alive(holder!.Id), Is.True, "an unrelated holder was killed"); + } + finally + { + if (holder is not null) + { + Support.Kill(holder.Id); + holder.WaitForExit(10_000); + holder.Dispose(); + } + } + } + + /// A stop during a long command kills its whole group: a child and a grandchild. + [Test] + public void AStopKillsTheWholeGroup() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + using var cancel = new CancellationTokenSource(); + string grandchild = PidPath("grandchild"); + string child = PidPath("child"); + Thread canceller = Support.After(TimeSpan.Zero, () => + { + SdkSupport.PidFile(grandchild); + cancel.Cancel(); + }); + Support.Throws(() => Run( + "(sleep 300 & echo $! > '" + grandchild + "'; wait) & echo $! > '" + child + "'; wait", + cancellationToken: cancel.Token)); + canceller.Join(); + SdkSupport.Dies("the child", SdkSupport.PidFile(child)); + SdkSupport.Dies("the grandchild", SdkSupport.PidFile(grandchild)); + } + + /// + /// Without a group (a test seam), a background child recorded while the + /// command ran is killed when it exits; with recording off it survives, + /// so the recorded kill is what ends it. + /// + [Test] + public void WithoutGroupsARecordedDescendantIsKilledAtTheExit() + { + CoreCommand.WithoutGroups = true; + var recorded = new List(); + CoreCommand.RecordedHook = pids => + { + lock (recorded) + { + recorded.Clear(); + recorded.AddRange(pids); + } + }; + string script = "sleep 300 /dev/null 2>&1 & echo $! > '" + PidPath("recorded") + "'; sleep 0.3"; + Assert.That(Run(script).ExitStatus, Is.EqualTo(0)); + int pid = SdkSupport.PidFile(PidPath("recorded")); + lock (recorded) + { + Assert.That(recorded, Does.Contain(pid), "the child was never recorded"); + } + + SdkSupport.Dies("the recorded child", pid); + + CoreCommand.RecordDescendants = false; + File.Delete(PidPath("recorded")); + Run(script); + Assert.That(Support.Alive(SdkSupport.PidFile(PidPath("recorded"))), Is.True, "without recording the child should survive"); + } + + [Test] + public void WithoutGroupsAStopKillsTheCommandAndItsChild() + { + CoreCommand.WithoutGroups = true; + CarrierException error = Support.Throws(() => + Run("sleep 300 & echo $! > '" + PidPath("child") + "'; wait", TimeSpan.FromMilliseconds(500)))!; + Assert.That(error.IsTimeout, Is.True); + SdkSupport.Dies("the child", SdkSupport.PidFile(PidPath("child"))); + } + + /// A cancellation that lands during the exit cleanup is reported, not the output. + [Test] + public void AStopDuringTheExitCleanupIsReported() + { + using var cancel = new CancellationTokenSource(); + CoreCommand.ExitedHook = cancel.Cancel; + CarrierException error = Support.Throws(() => Run("echo hi", cancellationToken: cancel.Token))!; + Assert.That(error.IsAborted, Is.True); + } + + /// + /// A fake /proc of processes whose parent + /// is , each stat file padded to 64 KiB. + /// Their pids start at 5,000,001, above Linux's largest pid (2^22), so a + /// kill sent to one reaches no process. + /// + private string FakeProc(string name, int count, int parent) + { + string proc = Directory.CreateDirectory(Path.Combine(root, name)).FullName; + string padding = string.Concat(Enumerable.Repeat(" 0", 32 * 1024)); + for (int i = 0; i < count; i++) + { + string pid = (5_000_001 + i).ToString(CultureInfo.InvariantCulture); + string directory = Directory.CreateDirectory(Path.Combine(proc, pid)).FullName; + File.WriteAllText( + Path.Combine(directory, "stat"), + pid + " (fake) S " + parent.ToString(CultureInfo.InvariantCulture) + " " + pid + " " + pid + " 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 " + + (1000 + i).ToString(CultureInfo.InvariantCulture) + padding + "\n"); + } + + return proc; + } + + /// A command that runs until exists, for at most 30 seconds. + private static string Waiting(string release) => + "i=0; while [ ! -e '" + release + "' ] && [ $i -lt 3000 ]; do sleep 0.01; i=$((i+1)); done"; + + /// + /// Without groups, one record reads at most its entry cap of + /// /proc entries and its byte cap of stat bytes, and at + /// most 1,024 bytes of each stat file, however many processes and + /// bytes there are (a fake /proc through a test seam, with lowered + /// caps). + /// + [Test] + public void AProcRecordReadsAtMostItsCaps() + { + CoreCommand.WithoutGroups = true; + CoreCommand.ProcRoot = FakeProc("proc", 50, 1); + foreach ((int entryCap, int byteCap, int expected) in new[] { (20, CoreCommand.ProcBytes, 20), (1000, 5 * 1024, 5) }) + { + CoreCommand.ProcEntryCap = entryCap; + CoreCommand.ProcByteCap = byteCap; + var reads = new List(); + int records = 0; + using var cancel = new CancellationTokenSource(); + CoreCommand.StatReadHook = (path, bytes, stoppable) => + { + if (stoppable && records == 0) + { + reads.Add(bytes); + } + }; + CoreCommand.RecordedHook = pids => + { + if (++records == 1) + { + cancel.Cancel(); + } + }; + CarrierException error = Support.Throws(() => Run(Waiting(Path.Combine(root, "never")), cancellationToken: cancel.Token))!; + Assert.That(error.IsAborted, Is.True); + Assert.That(reads, Has.Count.EqualTo(expected), "caps " + entryCap + "/" + byteCap); + Assert.That(reads, Is.All.EqualTo(CoreCommand.StatBytes)); + } + } + + /// + /// The byte cap holds when it is not a multiple of the 1,024 bytes read + /// of each stat file: with a 1,025-byte cap, a record reads 1,024 + /// bytes and then the 1 byte left, never a second full file. + /// + [Test] + public void AProcRecordNeverReadsPastANonAlignedByteCap() + { + CoreCommand.WithoutGroups = true; + CoreCommand.ProcRoot = FakeProc("proc", 10, 1); + CoreCommand.ProcByteCap = 1025; + var reads = new List(); + int records = 0; + using var cancel = new CancellationTokenSource(); + CoreCommand.StatReadHook = (path, bytes, stoppable) => + { + if (stoppable && records == 0) + { + reads.Add(bytes); + } + }; + CoreCommand.RecordedHook = pids => + { + if (++records == 1) + { + cancel.Cancel(); + } + }; + CarrierException error = Support.Throws(() => Run(Waiting(Path.Combine(root, "never")), cancellationToken: cancel.Token))!; + Assert.That(error.IsAborted, Is.True); + Assert.That(reads, Is.EqualTo(new[] { 1024, 1 })); + } + + /// The state letter of a live process, from /proc/<pid>/stat. + private static char State(int pid) + { + string stat = File.ReadAllText("/proc/" + pid.ToString(CultureInfo.InvariantCulture) + "/stat"); + return stat[(stat.LastIndexOf(')') + 2)..][0]; + } + + /// + /// On a stop without groups, the command is frozen with SIGSTOP before + /// the last record, the last record ends at its 250 ms budget, and only + /// then is the command killed. A cancellation during a record ends that + /// record at the next entry (the third of 50). The budget is measured on + /// a test clock that each read of the last record advances by 100 ms + /// (and each reading by 1 ms), so exactly three entries fit in it: the + /// fourth would start 304 ms after the stop. Without the budget, the + /// last record would read all 40 entries of its cap. + /// + [Test] + public void AStopFreezesTheCommandBeforeItsLastRecordAndBoundsIt() + { + CoreCommand.WithoutGroups = true; + CoreCommand.ProcRoot = FakeProc("proc", 50, 1); + CoreCommand.ProcEntryCap = 40; + long clock = 1_000_000; + long hundredMs = Stopwatch.Frequency / 10; + long oneMs = Stopwatch.Frequency / 1000; + CoreCommand.ClockHook = () => Interlocked.Add(ref clock, oneMs); + int command = 0; + CoreCommand.StartedHook = pid => command = pid; + var events = new List(); + int stoppable = 0; + char frozen = '?'; + using var cancel = new CancellationTokenSource(); + CoreCommand.SignalHook = (pid, signal) => events.Add((pid == command ? "command " : "other ") + signal.ToString(CultureInfo.InvariantCulture)); + CoreCommand.StatReadHook = (path, bytes, canStop) => + { + if (canStop) + { + if (++stoppable == 3) + { + cancel.Cancel(); + } + + return; + } + + if (!events.Contains("read")) + { + // SIGSTOP is delivered asynchronously; the command must reach the stopped state. + Support.Eventually(() => State(command) == 'T', TimeSpan.FromSeconds(10)); + frozen = State(command); + } + + events.Add("read"); + Interlocked.Add(ref clock, hundredMs); + }; + CarrierException error = Support.Throws(() => Run(Waiting(Path.Combine(root, "never")), cancellationToken: cancel.Token))!; + Assert.That(error.IsAborted, Is.True); + Assert.That(stoppable, Is.EqualTo(3)); + Assert.That(events, Is.EqualTo(new[] { "command 19", "read", "read", "read", "command 9" })); + Assert.That(frozen, Is.EqualTo('T')); + SdkSupport.Dies("the command", command); + } + + /// + /// The cleanup after a stop ends at its budgets on the test clock, even + /// when the last record and the kills of 300 recorded descendants would + /// take 600 ms each: the clock stands still except that each + /// stat read after the stop advances it by 2 ms. The last record + /// reads at 0, 2, ..., 248 ms after the stop (125 reads) and ends at + /// 250 ms, when the command is killed; the kills read at 250, ..., 498 ms + /// (125 reads) and end at 500 ms; the wait for the command to exit ends + /// at 750 ms. Without the budgets, all 300 entries would be read in each. + /// + [Test] + public void TheCleanupAfterAStopEndsAtItsBudgets() + { + CoreCommand.WithoutGroups = true; + string proc = Path.Combine(root, "proc"); + CoreCommand.ProcRoot = proc; + CoreCommand.StartedHook = pid => FakeProc("proc", 300, pid); + long start = 1_000_000_000; + long clock = start; + long ms = Stopwatch.Frequency / 1000; + CoreCommand.ClockHook = () => Interlocked.Read(ref clock); + int recorded = 0; + using var cancel = new CancellationTokenSource(); + CoreCommand.RecordedHook = pids => + { + if (recorded == 0) + { + recorded = pids.Count; + cancel.Cancel(); + } + }; + var reads = new List(); + long killed = -1; + CoreCommand.SignalHook = (pid, signal) => + { + if (signal == Unix.Sigkill) + { + killed = Interlocked.Read(ref clock) - start; + reads.Add(-1); + } + }; + CoreCommand.StatReadHook = (path, bytes, canStop) => + { + if (!canStop) + { + reads.Add(Interlocked.Read(ref clock) - start); + Interlocked.Add(ref clock, 2 * ms); + } + }; + var waits = new List(); + CoreCommand.ExitWaitHook = end => waits.Add(end - start); + Support.Throws(() => Run(Waiting(Path.Combine(root, "never")), cancellationToken: cancel.Token)); + Assert.That(recorded, Is.EqualTo(300)); + int split = reads.IndexOf(-1); + Assert.That(split, Is.GreaterThanOrEqualTo(0), "the command was not killed"); + List record = reads.Take(split).ToList(); + List kills = reads.Skip(split + 1).ToList(); + Assert.That(record, Is.EqualTo(Enumerable.Range(0, 125).Select(i => 2L * i * ms)), "the last record ends at 250 ms"); + Assert.That(killed, Is.EqualTo(250 * ms), "the command is killed when the record ends"); + Assert.That(kills, Is.EqualTo(Enumerable.Range(125, 125).Select(i => 2L * i * ms)), "the kills end at 500 ms"); + Assert.That(reads.Count - 1, Is.LessThan(600), "the record and the kills were not cut at their budgets"); + Assert.That(waits, Is.EqualTo(new[] { (long)CoreCommand.StopBudgetMs * ms }), "the exit wait ends at 750 ms"); + } + + /// + /// A stop that lands during the kills of recorded descendants when the + /// command exits does not leave the rest running: ten fake descendants + /// are recorded, the command exits, a cancellation lands during the + /// second kill check, the other eight are still checked (within the kill + /// budget), and the stop is reported. + /// + [Test] + public void AStopDuringTheExitKillsFinishesThem() + { + CoreCommand.WithoutGroups = true; + string release = Path.Combine(root, "release"); + string proc = Path.Combine(root, "proc"); + CoreCommand.ProcRoot = proc; + CoreCommand.StartedHook = pid => FakeProc("proc", 10, pid); + int recorded = 0; + CoreCommand.RecordedHook = pids => + { + recorded = pids.Count; + CoreCommand.RecordDescendants = false; + File.WriteAllText(release, string.Empty); + }; + int checks = 0; + int afterStop = 0; + using var cancel = new CancellationTokenSource(); + CoreCommand.StatReadHook = (path, bytes, stoppable) => + { + if (!stoppable) + { + afterStop++; + } + else if (!CoreCommand.RecordDescendants && ++checks == 2) + { + cancel.Cancel(); + } + }; + CarrierException error = Support.Throws(() => Run(Waiting(release), cancellationToken: cancel.Token))!; + Assert.That(error.IsAborted, Is.True); + Assert.That(recorded, Is.EqualTo(10)); + Assert.That(checks, Is.EqualTo(2)); + Assert.That(afterStop, Is.EqualTo(8)); + } + + [Test] + public void ACommandThatCannotStartIsALoss() + { + CarrierException error = Support.Throws(() => + CoreCommand.Run(new[] { Path.Combine(root, "missing") }, null, Minute, 1024, 1024, default))!; + Assert.That(error.IsLoss && !error.IsTimeout && !error.IsAborted, Is.True); + Assert.That(error.Message, Does.StartWith("could not start ")); + CoreCommand.WithoutGroups = true; + Assert.That(Support.Throws(() => + CoreCommand.Run(new[] { Path.Combine(root, "missing") }, null, Minute, 1024, 1024, default))!.Message, Does.StartWith("could not start ")); + } + + /// An executable setsid cannot exec (its interpreter is missing) is a start failure, not the command's exit status. + [Test] + public void ACommandSetsidCannotExecIsALoss() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + string broken = Path.Combine(root, "broken"); + File.WriteAllText(broken, "#!/no/such/interpreter\n"); + File.SetUnixFileMode(broken, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + CarrierException error = Support.Throws(() => CoreCommand.Run(new[] { broken }, null, Minute, 1024, 1024, default))!; + Assert.That(error.IsLoss, Is.True); + Assert.That(error.Message, Does.StartWith("could not start " + broken + " (setsid: "), error.Message); + } + + /// + /// A descendant that escapes the group and floods the output holds + /// neither the caller nor the heap: the command returns once it exits, + /// with at most the kept limit of output. + /// + [Test] + public void AnEscapedFlooderHoldsNeitherTheCallerNorTheHeap() + { + string flooder = PidPath("flooder"); + CoreCommand.Output output = Support.Within(Minute, () => + Run("setsid sh -c 'echo $$ > \"" + flooder + "\"; exec yes' & sleep 0.2", stdoutLimit: 4096)); + Assert.That(output.Stdout.Length, Is.LessThanOrEqualTo(4096)); + // It is not the host's to kill (it left the group); its next write + // after the host closed the pipe fails, and the test ends it. + Support.Kill(SdkSupport.PidFile(flooder)); + } + + /// Commands release every descriptor they opened by the time they return (counted at once, with no wait a finalizer could satisfy). + [Test] + public void CommandsLeaveNoDescriptorsBehind() + { + Run("true"); + int before = Support.Descriptors(); + for (int i = 0; i < 20; i++) + { + Run("echo $i; echo e >&2"); + Support.Throws(() => Run("sleep 300", TimeSpan.FromMilliseconds(50))); + } + + Assert.That(Support.Descriptors(), Is.EqualTo(before)); + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs new file mode 100644 index 0000000..43a4385 --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs @@ -0,0 +1,859 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Numerics; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// Signatures, identity groups, and native values through checked types, with synthetic identities. +[TestFixture] +public sealed class DataTests +{ + internal static Hash Id(char letter) => new(new string(letter, 64)); + + internal static Hash Numbered(int n) => new(n.ToString("x64", CultureInfo.InvariantCulture)); + + internal static NominalType Nominal(Hash identity, params BoundaryType[] arguments) => new(identity, arguments); + + /// Data with synthetic identities for the builtins, an Incident record, an Urgency enumeration, and a Triage union. + internal static ProgramData Fixture(Limits limits) + { + var names = new Dictionary + { + ["int"] = Id('1'), + ["real"] = Id('2'), + ["text"] = Id('3'), + ["bool"] = Id('4'), + ["hash"] = Id('5'), + ["list"] = Id('6'), + ["option"] = Id('7'), + ["incident"] = Id('8'), + ["urgency"] = Id('9'), + ["triage"] = Numbered(10), + }; + var constructors = new Dictionary + { + ["true"] = Id('a'), + ["false"] = Id('b'), + ["some"] = Id('c'), + ["none"] = Id('d'), + ["cons"] = Id('e'), + ["nil"] = Id('f'), + ["incident"] = Id('0'), + ["page-now"] = Numbered(1), + ["watch-only"] = Numbered(2), + ["routed"] = Numbered(3), + ["invalid-incident"] = Numbered(4), + }; + var groups = new List>(); + foreach (string[] group in new[] { new[] { "incident", "incident" }, new[] { "urgency", "page-now", "watch-only" }, new[] { "triage", "routed", "invalid-incident" } }) + { + var entries = new List { new(group[0], names[group[0]]) }; + entries.AddRange(group.Skip(1).Select(member => new Identities.Entry(member, constructors[member]))); + groups.Add(entries); + } + + return new ProgramData(names, constructors, groups, limits); + } + + internal sealed record Incident(string Id, long Severity, string? Owner); + + internal enum Urgency + { + PageNow, + WatchOnly, + } + + internal abstract record Triage; + + internal sealed record Routed(string Id, Urgency Urgency) : Triage; + + internal sealed record InvalidIncident(string Id) : Triage; + + internal static readonly RecordCodec IncidentCodec = Codecs.Record( + "Incident", + Codecs.Field("id", Codecs.Text, (Incident incident) => incident.Id), + Codecs.Field("severity", Codecs.Int, (Incident incident) => incident.Severity), + Codecs.Field("owner", Codecs.Nullable(Codecs.Text), (Incident incident) => incident.Owner), + (id, severity, owner) => new Incident(id, severity, owner)); + + internal static readonly UnionCodec TriageCodec = Codecs.Union( + Codecs.Record("Routed", Codecs.Field("id", Codecs.Text, (Routed routed) => routed.Id), Codecs.Field("urgency", Codecs.Enum(), (Routed routed) => routed.Urgency), (id, urgency) => new Routed(id, urgency)), + Codecs.Record("InvalidIncident", Codecs.Field("id", Codecs.Text, (InvalidIncident invalid) => invalid.Id), id => new InvalidIncident(id))); + + private static string Refusal(Action body) => Support.Throws(body).Message; + + [Test] + public void SignaturesAndGroups() + { + var names = new Dictionary + { + ["list"] = Id('6'), ["line"] = Id('8'), ["int"] = Id('1'), ["option"] = Id('7'), ["text"] = Id('3'), ["inventory"] = Id('9'), ["dispatch"] = Id('a'), + }; + CallInterface spec = Identities.ParseSignature("(List Line, Int) ->{Inventory, Dispatch} Option Text", names); + Assert.That(spec.Parameters, Is.EqualTo(new BoundaryType[] { Nominal(Id('6'), Nominal(Id('8'))), Nominal(Id('1')) })); + Assert.That(spec.Effects, Is.EqualTo(new[] { Id('9'), Id('a') })); + Assert.That(spec.Result, Is.EqualTo(Nominal(Id('7'), Nominal(Id('3'))))); + Assert.That(Identities.ParseSignature("() ->{} ()", names).Result, Is.EqualTo(new TupleType())); + Assert.That(Identities.ParseSignature("((Int, Text)) ->{} (Int)", names).Result, Is.EqualTo(Nominal(Id('1')))); + Assert.That(Identities.ParseSignature("((Int, Text)) ->{} Int", names).Parameters, Is.EqualTo(new BoundaryType[] { new TupleType(Nominal(Id('1')), Nominal(Id('3'))) })); + foreach (string text in new[] { "forall a | e. (List a) ->{| e} Bool", "(Missing) ->{} Int", "(Int) ->{} Int extra)", "(Int) ->{} Int $" }) + { + Support.Throws(() => Identities.ParseSignature(text, names), text); + } + + string output = "0 " + Id('1').Hex + "\n0:line " + Id('1').Hex + "\n0:line " + Id('2').Hex + "\nnoise\n1:other " + Id('3').Hex + "\n"; + List> groups = Identities.HashGroups(output); + Assert.That(groups, Has.Count.EqualTo(2)); + Assert.That(groups[0][1].Hash, Is.EqualTo(Id('2'))); + Assert.That(Support.Throws(() => Identities.HashGroups("nothing here")).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + var x1 = new Identities.Entry("x", Id('1')); + Support.Throws(() => Identities.CheckGroups(new List> { new() { x1 }, new() { new("x", Id('2')) } })); + Identities.CheckGroups(new List> { new() { x1 }, new() { x1 } }); + foreach ((string display, string store) in new[] { ("WorkToday", "work-today"), ("PageNow", "page-now"), ("text", "text"), ("Inventory", "inventory") }) + { + Assert.That(Identities.DisplayToStore(display), Is.EqualTo(store)); + } + + Assert.That(Identities.Normal("lookup_stock", "operation"), Is.EqualTo("lookup-stock")); + Assert.That(Identities.DisplayToStore("\ud800X"), Is.EqualTo("\ud800X"), "a lone surrogate is not upper case"); + } + + [Test] + public void NativeValuesRoundtripAndConformToCheckedTypes() + { + ProgramData data = Fixture(Limits.Hard); + var rows = new List { new("a", 5, null), new("b", 2, "ops") }; + BoundaryType list = Nominal(Id('6'), Nominal(Id('8'))); + Value value = data.Encode(Codecs.List(IncidentCodec), rows); + data.Conform(value, list, "rows"); + Assert.That(data.Decode(Codecs.List(IncidentCodec), value), Is.EqualTo(rows)); + BoundaryType real = Nominal(Id('2')); + foreach (double number in new[] { 1.0, -0.0, double.PositiveInfinity, double.NaN, BitConverter.Int64BitsToDouble(0x7ff8000000000123L) }) + { + Value encoded = data.Encode(Codecs.Real, number); + data.Conform(encoded, real, "real"); + Assert.That(BitConverter.DoubleToInt64Bits(data.Decode(Codecs.Real, encoded)), Is.EqualTo(BitConverter.DoubleToInt64Bits(number))); + } + + BoundaryType integer = Nominal(Id('1')); + Assert.That(Refusal(() => data.Encode(Codecs.Int, 1L << 62)), Is.EqualTo("value: integer is outside Core's 63-bit Int range")); + Assert.That(Refusal(() => data.Encode(Codecs.Int, long.MinValue)), Is.EqualTo("value: integer is outside Core's 63-bit Int range")); + Assert.That(data.Encode(Codecs.Int, Json.IntMax), Is.EqualTo(new IntValue(Json.IntMax))); + Assert.That(data.Encode(Codecs.Int, Json.IntMin), Is.EqualTo(new IntValue(Json.IntMin))); + Support.Throws(() => data.Decode(Codecs.Int32, new IntValue(1L << 40))); + Assert.That(data.Decode(Codecs.Int32, new IntValue(7)), Is.EqualTo(7)); + Assert.That(Refusal(() => data.Conform(data.Encode(Codecs.Text, "x"), integer, "argument[0]")), Is.EqualTo("argument[0]: expected an integer")); + Assert.That(Refusal(() => data.Conform(data.Encode(Codecs.Int, 3L), real, "argument[0]")), Is.EqualTo("argument[0]: expected a Real (double)")); + Assert.That(Refusal(() => data.Conform(data.Encode(Codecs.Enum(), Urgency.PageNow), Nominal(Id('8')), "argument[0]")), Does.Contain("expected a incident value")); + Value texts = data.Encode(Codecs.List(Codecs.Text), new[] { "ok", "ok" }); + Assert.That(Refusal(() => data.Conform(texts, Nominal(Id('6'), integer), "argument[0]")), Is.EqualTo("argument[0][0]: expected an integer")); + var unit = new TupleType(); + data.Conform(data.Encode(Codecs.Unit, Unit.Value), unit, "unit"); + Assert.That(data.Decode(Codecs.Unit, Value.Unit), Is.EqualTo(Unit.Value)); + Support.Throws(() => data.Conform(data.Encode(Codecs.Tuple(Codecs.Int, Codecs.Int), (1L, 2L)), unit, "unit")); + + // A record with an unknown constructor is refused at its location. + Assert.That(Refusal(() => data.Encode(Codecs.Record("Unknown", Codecs.Field("field", Codecs.Int, (long x) => x), x => x), 1L)), Is.EqualTo("value: unknown constructor Unknown")); + + // A record over a builtin constructor is refused, as Dynamic is. + Assert.That(Refusal(() => data.Encode(Codecs.Record("Some", Codecs.Field("inner", Codecs.Int, (long x) => x), x => x), 1L)), Does.Contain("native form of builtin constructor Some")); + + // null is no value; Option and Unit are. + Assert.That(Refusal(() => data.Encode(Codecs.Text, null!)), Does.StartWith("value: null has no Jacquard value")); + Assert.That(Refusal(() => data.Encode(Codecs.Text, "\ud800")), Does.Contain("lone surrogate")); + } + + [Test] + public void OptionsEnumsUnionsAndDynamicValues() + { + ProgramData data = Fixture(Limits.Hard); + Triage routed = new Routed("r", Urgency.PageNow); + Value value = data.Encode(TriageCodec, routed); + Assert.That(value, Is.EqualTo(new ConstructorValue(Numbered(3), new Value[] { new TextValue("r"), new ConstructorValue(Numbered(1), Array.Empty()) }))); + Assert.That(data.Decode(TriageCodec, value), Is.EqualTo(routed)); + Value invalid = data.Encode(TriageCodec, new InvalidIncident("x")); + Assert.That(data.Decode(TriageCodec, invalid), Is.EqualTo(new InvalidIncident("x"))); + Assert.That(Refusal(() => data.Decode(Codecs.Enum(), invalid)), Does.Contain("unknown Urgency case")); + var shortRouted = new ConstructorValue(Numbered(3), new Value[] { new TextValue("r") }); + Assert.That(Refusal(() => data.Decode(TriageCodec, shortRouted)), Does.Contain("wrong constructor or field count")); + BoundaryType option = Nominal(Id('7'), Nominal(Id('7'), Nominal(Id('1')))); + Codec>> nested = Codecs.Option(Codecs.Option(Codecs.Int)); + var cases = new[] { Option>.None, Option.Some(Option.None), Option.Some(Option.Some(3L)) }; + var dynamics = new[] { Dynamic.None(), Dynamic.Some(Dynamic.None()), Dynamic.Some(Dynamic.Some(Dynamic.Of(3L))) }; + for (int i = 0; i < cases.Length; i++) + { + Value encoded = data.Encode(nested, cases[i]); + data.Conform(encoded, option, "option"); + Assert.That(data.Decode(nested, encoded), Is.EqualTo(cases[i])); + Assert.That(data.Decode(Codecs.Dynamic, encoded), Is.EqualTo(dynamics[i])); + Assert.That(data.Encode(Codecs.Dynamic, dynamics[i]), Is.EqualTo(encoded)); + } + + // A nullable value type maps null to None. + Assert.That(data.Decode(Codecs.NullableValue(Codecs.Int), data.Encode(Codecs.NullableValue(Codecs.Int), null)), Is.Null); + Assert.That(data.Decode(Codecs.NullableValue(Codecs.Int), data.Encode(Codecs.NullableValue(Codecs.Int), 4L)), Is.EqualTo(4L)); + Dynamic watch = data.Make("WatchOnly"); + Assert.That(watch, Is.EqualTo(Dynamic.VariantOf("watch-only"))); + Assert.That(data.Decode(Codecs.Enum(), data.Encode(Codecs.Dynamic, watch)), Is.EqualTo(Urgency.WatchOnly)); + Assert.That(Support.Throws(() => data.Make("Missing")).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Support.Throws(() => data.Encode(Codecs.Dynamic, Dynamic.VariantOf("Some", Dynamic.Of(1L))), "a builtin constructor by name"); + Dynamic dynamic = Dynamic.ListOf(Dynamic.TupleOf(Dynamic.Of(true), Dynamic.Of("x"), Dynamic.Of(2.5)), new Dynamic.HashValue(Id('5'))); + Assert.That(data.Decode(Codecs.Dynamic, data.Encode(Codecs.Dynamic, dynamic)), Is.EqualTo(dynamic)); + Assert.That(Refusal(() => data.Decode(Codecs.Dynamic, new ConstructorValue(Id('9'), Array.Empty()))), Does.Contain("unmapped constructor")); + Unit[] units = Enumerable.Repeat(Unit.Value, 40).ToArray(); + Assert.That(Refusal(() => data.Encode(Codecs.List(Codecs.Unit), units)), Does.Contain("JSON depth"), "a list deeper than the JSON depth"); + Assert.That(data.Identity("Urgency"), Is.EqualTo(Id('9'))); + Support.Throws(() => data.Identity("missing")); + } + + [Test] + public void ExplicitlyNamedEnumsTuplesAndUnionsAgree() + { + ProgramData data = Fixture(Limits.Hard); + EnumCodec named = Codecs.Enum(new Dictionary { [Urgency.PageNow] = "PageNow", [Urgency.WatchOnly] = "watch-only" }); + Assert.That(data.Encode(named, Urgency.WatchOnly), Is.EqualTo(data.Encode(Codecs.Enum(), Urgency.WatchOnly))); + Support.Throws(() => Codecs.Enum(new Dictionary { [Urgency.PageNow] = "PageNow" })); + Assert.That(Refusal(() => data.Encode(Codecs.Enum(), (Urgency)7)), Does.Contain("Urgency has no constructor for 7")); + + // A union's cases must derive from it. + Support.Throws(() => Codecs.Union(IncidentCodec)); + Value tuple = data.Encode(Codecs.Tuple(Codecs.Int, Codecs.Text), (1L, "x")); + Assert.That(tuple, Is.EqualTo(new TupleValue(new IntValue(1), new TextValue("x")))); + Assert.That(data.Decode(Codecs.Tuple(Codecs.Int, Codecs.Text), tuple), Is.EqualTo((1L, "x"))); + Support.Throws(() => data.Decode(Codecs.Tuple(Codecs.Int, Codecs.Text, Codecs.Text), tuple)); + + // A custom codec built from two functions. + Codec upper = Codecs.Create((encoder, text) => Codecs.Text.Encode(encoder, text.ToUpperInvariant()), (decoder, v) => Codecs.Text.Decode(decoder, v)); + Assert.That(data.Decode(upper, data.Encode(upper, "abc")), Is.EqualTo("ABC")); + } + + /// + /// Numeric types with no Jacquard type are refused instead of converting + /// silently (C# would take a through + /// , so 2^53 + 1 arrives as 2^53, and a + /// to an Int): their conversions to + /// and their + /// overloads are obsolete as errors, so a use does not compile; reached by + /// reflection, the argument is refused when it is encoded and the + /// overload throws. There is no decimal codec: money crosses as integer + /// minor units. The integer types that widen exactly still convert to + /// Int. + /// + [Test] + public void NumericTypesWithoutAJacquardTypeAreRefused() + { + ProgramData data = Fixture(Limits.Hard); + const ulong Past = (1UL << 53) + 1; + var refused = new (object Value, string Reason)[] + { + (Past, NoConversion.Unsigned64), + (unchecked((nuint)Past), NoConversion.UnsignedNative), + ('x', NoConversion.Character), + (0.1f, NoConversion.Single), + ((Half)1, NoConversion.HalfFloat), + (1.10m, NoConversion.Decimal), + ((Int128)Past, NoConversion.Signed128), + ((UInt128)Past, NoConversion.Unsigned128), + (new BigInteger(Past), NoConversion.Big), + }; + foreach ((object value, string reason) in refused) + { + Type type = value.GetType(); + MethodInfo conversion = typeof(Argument).GetMethod("op_Implicit", new[] { type }) + ?? throw new AssertionException(type.Name + " has no refusing conversion"); + Assert.That(conversion.GetCustomAttribute() is { IsError: true } obsolete && obsolete.Message == reason, Is.True, type.Name + " conversion compiles"); + var argument = (Argument)conversion.Invoke(null, new[] { value })!; + JacquardException error = Support.Throws(() => argument.Encode(Encoder.Root(data, "value"))); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(error.Message, Is.EqualTo("value: " + reason)); + MethodInfo of = typeof(Dynamic).GetMethod("Of", new[] { type }) ?? throw new AssertionException(type.Name + " has no refusing Dynamic.Of"); + Assert.That(of.GetCustomAttribute() is { IsError: true }, Is.True, type.Name + " Dynamic.Of compiles"); + TargetInvocationException thrown = Support.Throws(() => of.Invoke(null, new[] { value })); + Assert.That(thrown.InnerException, Is.InstanceOf().With.Message.StartWith(reason)); + } + + Assert.That(typeof(Codecs).GetProperties().Select(property => property.PropertyType), Does.Not.Contain(typeof(Codec))); + foreach (Argument exact in new Argument[] { (byte)7, (sbyte)7, (short)7, (ushort)7, 7U, 7, 7L, (nint)7 }) + { + Assert.That(exact.Encode(Encoder.Root(data, "value")), Is.EqualTo(new IntValue(7))); + } + + foreach (Dynamic exact in new[] { Dynamic.Of((byte)7), Dynamic.Of((sbyte)7), Dynamic.Of((short)7), Dynamic.Of((ushort)7), Dynamic.Of(7U), Dynamic.Of(7), Dynamic.Of((nint)7) }) + { + Assert.That(exact, Is.EqualTo(Dynamic.Of(7L))); + } + } + + /// A recursive type maps through a lazy codec, which is made once even when many threads ask at once. + internal sealed record Tree(string Label, IReadOnlyList Children) + { + public bool Equals(Tree? other) => other is not null && Label == other.Label && Children.SequenceEqual(other.Children); + + public override int GetHashCode() => Label.GetHashCode(StringComparison.Ordinal); + } + + [Test] + public void RecursiveTypesMapThroughALazyCodec() + { + var names = new Dictionary { ["list"] = Id('6'), ["text"] = Id('3') }; + var constructors = new Dictionary { ["cons"] = Id('e'), ["nil"] = Id('f'), ["tree"] = Id('0') }; + var data = new ProgramData(names, constructors, Array.Empty>(), Limits.Hard); + int made = 0; + RecordCodec? codec = null; + LazyCodec self = Codecs.Lazy(() => + { + Interlocked.Increment(ref made); + return codec!; + }); + codec = Codecs.Record( + "Tree", + Codecs.Field("label", Codecs.Text, (Tree tree) => tree.Label), + Codecs.Field("children", Codecs.List(self), (Tree tree) => tree.Children), + (label, children) => new Tree(label, children)); + var tree = new Tree("root", new[] { new Tree("leaf", Array.Empty()), new Tree("branch", new[] { new Tree("leaf", Array.Empty()) }) }); + Parallel.For(0, 8, _ => Assert.That(data.Decode(self, data.Encode(self, tree)), Is.EqualTo(tree))); + Assert.That(made, Is.EqualTo(1), "the lazy codec was made more than once"); + + // A chain of a million lazy codecs resolves in a loop, and one standing for itself is refused. + LazyCodec chain = Codecs.Lazy(() => Codecs.Int); + for (int i = 0; i < 1_000_000; i++) + { + LazyCodec inner = chain; + chain = Codecs.Lazy(() => inner); + } + + Assert.That(data.Decode(chain, data.Encode(chain, 5L)), Is.EqualTo(5L)); + LazyCodec? loop = null; + loop = Codecs.Lazy(() => loop!); + JacquardException refused = Support.Throws(() => data.Encode(loop, 1L)); + Assert.That(refused.Message, Does.Contain("a lazy codec stands for itself")); + Assert.That(refused.InnerException, Is.InstanceOf()); + } + + [Test] + public void ArgumentsConvertFromPlainValuesOrCarryTheirCodec() + { + ProgramData data = Fixture(Limits.Hard); + Value Encode(Argument argument) => argument.Encode(Encoder.Root(data, "f.argument[0]")); + Assert.That(Encode(35), Is.EqualTo(data.Encode(Codecs.Int, 35L))); + Assert.That(Encode(35L), Is.EqualTo(new IntValue(35))); + Assert.That(Encode(2.5), Is.EqualTo(RealValue.Of(2.5))); + Assert.That(Encode("x"), Is.EqualTo(new TextValue("x"))); + Assert.That(Encode(true), Is.EqualTo(data.Encode(Codecs.Bool, true))); + Assert.That(Encode(Id('5')), Is.EqualTo(new HashValue(Id('5')))); + Assert.That(Encode(Dynamic.Of(4L)), Is.EqualTo(new IntValue(4))); + Assert.That(Encode(Codecs.List(IncidentCodec).Arg(Array.Empty())), Is.EqualTo(data.Encode(Codecs.List(IncidentCodec), Array.Empty()))); + Assert.That(Refusal(() => Encode(long.MaxValue)), Is.EqualTo("f.argument[0]: integer is outside Core's 63-bit Int range")); + Assert.That(Refusal(() => Encode((string)null!)), Does.StartWith("f.argument[0]: null has no Jacquard value")); + } + + [Test] + public void SelectedLimitsBoundMappedValues() + { + Limits limits = Limits.Hard.With("max_collection_items", 2).With("max_text_bytes", 3); + ProgramData data = Fixture(limits); + Assert.That(Refusal(() => data.Encode(Codecs.List(Codecs.Int), new[] { 1L, 2L, 3L })), Does.Contain("max_collection_items")); + Assert.That(Refusal(() => data.Encode(Codecs.Text, "four")), Does.Contain("max_text_bytes")); + Value longList = Fixture(Limits.Hard).Encode(Codecs.List(Codecs.Int), new[] { 1L, 2L, 3L }); + Assert.That(Refusal(() => data.Decode(Codecs.List(Codecs.Int), longList)), Does.Contain("selected limit")); + + // A record's fields count against max_collection_items too, so this one needs three. + ProgramData fields = Fixture(Limits.Hard.With("max_collection_items", 3).With("max_text_bytes", 3)); + Assert.That(Refusal(() => fields.Encode(IncidentCodec, new Incident("abcd", 1, null))), Is.EqualTo("value.id: text exceeds max_text_bytes")); + Assert.That(Refusal(() => data.Encode(IncidentCodec, new Incident("abc", 1, null))), Is.EqualTo("value: constructor exceeds max_collection_items")); + } + + /// + /// A small .NET value can denote a huge one: three levels of 64 + /// references to one tuple are 64^3 integers. Encoding stops at the + /// 4097th node, so it costs no more than one frame; a raw value is + /// refused before its walk stacks more than the node limit, and a deep + /// one is refused by its depth without recursion. + /// + [Test] + public void SharedValuesAreBoundedByTheNodeBudget() + { + ProgramData data = Fixture(Limits.Hard); + var leaf = new Dynamic.Tuple(Enumerable.Repeat(Dynamic.Of(1L), 64)); + var middle = new Dynamic.Tuple(Enumerable.Repeat(leaf, 64)); + var top = new Dynamic.Tuple(Enumerable.Repeat(middle, 64)); + + // Sixty-three tuples of 65 nodes each, then the first integer of the next. + Assert.That(Refusal(() => data.Encode(Codecs.Dynamic, top)), Is.EqualTo("value[0][63][1]: value exceeds max_value_nodes; use smaller independent batches")); + var rawLeaf = new TupleValue(Enumerable.Repeat(new IntValue(1), 64)); + var rawTop = new TupleValue(Enumerable.Repeat(new TupleValue(Enumerable.Repeat(rawLeaf, 64)), 64)); + Assert.That(Refusal(() => data.Encode(Codecs.Value, rawTop)), Is.EqualTo("value: value exceeds max_value_nodes; use smaller independent batches")); + + // Decoding a supplied value is measured first, within the same bounds. + Assert.That(Refusal(() => data.Decode(Codecs.Dynamic, rawTop)), Is.EqualTo("value exceeds max_value_nodes")); + Value deep = new IntValue(1); + for (int i = 0; i < 100_000; i++) + { + deep = new TupleValue(deep); + } + + Assert.That(Refusal(() => data.Decode(Codecs.Dynamic, deep)), Is.EqualTo("value exceeds the v0 JSON depth")); + Assert.That(Refusal(() => data.Encode(Codecs.Value, deep)), Is.EqualTo("value: value exceeds the v0 JSON depth; use smaller independent batches")); + } + + /// + /// Texts are charged against the frame as they are encoded: four + /// references to one maximal text fill it, and the fifth is refused + /// before it is scanned. + /// + [Test] + public void RepeatedTextsAreChargedAgainstTheFrame() + { + ProgramData data = Fixture(Limits.Hard); + Dynamic text = Dynamic.Of(new string('x', Limits.Hard.MaxTextBytes)); + var many = new Dynamic.Tuple(Enumerable.Repeat(text, 1000)); + Assert.That(Refusal(() => data.Encode(Codecs.Dynamic, many)), Is.EqualTo("value[4]: value exceeds max_frame_bytes; use smaller independent batches")); + var rawText = new TextValue(new string('x', Limits.Hard.MaxTextBytes)); + var raw = new TupleValue(Enumerable.Repeat(rawText, 1000)); + Assert.That(Refusal(() => data.Encode(Codecs.Value, raw)), Is.EqualTo("value: value exceeds max_frame_bytes; use smaller independent batches")); + } + + /// A text too long in chars is refused before it is scanned: the lone surrogate at its end is never reached. + [Test] + public void AnOverlongTextIsRefusedBeforeItIsScanned() + { + ProgramData data = Fixture(Limits.Hard); + string text = new string('x', Limits.Hard.MaxTextBytes) + "\ud800"; + Assert.That(Refusal(() => data.Encode(Codecs.Text, text)), Is.EqualTo("value: text exceeds max_text_bytes")); + } + + /// A raw text is charged by its UTF-8 bytes: three two-byte chars exceed a four-byte limit. + [Test] + public void ARawTextIsChargedByItsUtf8Bytes() + { + ProgramData data = Fixture(Limits.Hard.With("max_text_bytes", 4)); + Assert.That(data.Encode(Codecs.Value, new TextValue("éé")), Is.EqualTo(new TextValue("éé"))); + Assert.That(Refusal(() => data.Encode(Codecs.Value, new TextValue("ééé"))), Is.EqualTo("value: text exceeds max_text_bytes")); + Assert.That(Refusal(() => data.Encode(Codecs.Value, new TextValue("\ud800"))), Is.EqualTo("value: text contains a lone surrogate, which has no Jacquard encoding")); + } + + /// A list whose count is over the limit, and whose items throw when read. + private sealed class HugeList : IReadOnlyList + { + public int Count => 1 << 30; + + public long this[int index] => throw new InvalidOperationException("element " + index.ToString(CultureInfo.InvariantCulture) + " was read"); + + public IEnumerator GetEnumerator() => throw new InvalidOperationException("the list was enumerated"); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// A list over the collection limit is refused from its count, before any element is read. + [Test] + public void AnOversizedListIsRefusedBeforeItsElementsAreRead() + { + ProgramData data = Fixture(Limits.Hard); + Assert.That(Refusal(() => data.Encode(Codecs.List(Codecs.Int), new HugeList())), Is.EqualTo("value: list exceeds max_collection_items")); + } + + private sealed class Counting : Codec + { + internal int Encodes { get; private set; } + + public override Value Encode(Encoder encoder, long value) + { + Encodes++; + return Codecs.Int.Encode(encoder, value); + } + + public override long Decode(Decoder decoder, Value value) => Codecs.Int.Decode(decoder, value); + } + + /// Tuples and records over the collection limit are refused before any item is encoded or field read. + [Test] + public void OversizedTuplesAndRecordsAreRefusedBeforeTheirItemsAreRead() + { + ProgramData data = Fixture(Limits.Hard.With("max_collection_items", 2)); + var counted = new Counting(); + Assert.That(Refusal(() => data.Encode(Codecs.Tuple(counted, counted, counted), (1L, 2L, 3L))), Is.EqualTo("value: tuple exceeds max_collection_items")); + Assert.That(counted.Encodes, Is.EqualTo(0)); + int reads = 0; + RecordCodec<(long, long, long)> record = Codecs.Record( + "Incident", + Codecs.Field("a", Codecs.Int, ((long A, long B, long C) r) => ++reads + r.A), + Codecs.Field("b", Codecs.Int, ((long A, long B, long C) r) => ++reads + r.B), + Codecs.Field("c", Codecs.Int, ((long A, long B, long C) r) => ++reads + r.C), + (a, b, c) => (a, b, c)); + Assert.That(Refusal(() => data.Encode(record, (1L, 2L, 3L))), Is.EqualTo("value: constructor exceeds max_collection_items")); + Assert.That(reads, Is.EqualTo(0)); + Fixture(Limits.Hard.With("max_collection_items", 3)).Encode(record, (1L, 2L, 3L)); + Assert.That(reads, Is.EqualTo(3)); + } + + /// A codec building a constructor by name has its fields counted before they are copied. + [Test] + public void AConstructorWithTooManyFieldsIsRefusedBeforeItIsBuilt() + { + ProgramData data = Fixture(Limits.Hard); + Value one = data.Encode(Codecs.Int, 1L); + Codec wide = Codecs.Create((encoder, count) => encoder.Constructor("routed", new RepeatList(one, (int)count)), (decoder, value) => 0); + Assert.That(Refusal(() => data.Encode(wide, 20_000_000L)), Is.EqualTo("value: constructor exceeds max_collection_items")); + Assert.That(Refusal(() => data.Encode(wide, Limits.Hard.MaxCollectionItems + 1L)), Is.EqualTo("value: constructor exceeds max_collection_items")); + } + + /// A list of one value repeated, that holds no items. + internal sealed class RepeatList : IReadOnlyList + { + private readonly T item; + + internal RepeatList(T item, int count) + { + this.item = item; + Count = count; + } + + public int Count { get; } + + public T this[int index] => item; + + public IEnumerator GetEnumerator() + { + for (int i = 0; i < Count; i++) + { + yield return item; + } + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// A Dynamic value over the hard limit is refused at construction, before its items are copied. + [Test] + public void AnOversizedDynamicIsRefusedBeforeItIsCopied() + { + int limit = Limits.Hard.MaxCollectionItems; + var many = new RepeatList(Dynamic.Of(1L), limit + 1); + Support.Throws(() => _ = new Dynamic.List(many)); + Support.Throws(() => _ = new Dynamic.Tuple(many)); + Support.Throws(() => _ = new Dynamic.Variant("routed", many)); + + // An enumeration that does not say its count is read at most one past the limit. + int read = 0; + IEnumerable endless = Enumerable.Range(0, int.MaxValue).Select(_ => + { + read++; + return Dynamic.Of(1L); + }); + Support.Throws(() => _ = new Dynamic.List(endless)); + Assert.That(read, Is.EqualTo(limit + 1)); + Assert.That(Support.Throws(() => Fixture(Limits.Hard).Make("routed", many.ToArray())).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(new Dynamic.List(new RepeatList(Dynamic.Of(1L), limit)).Items, Has.Count.EqualTo(limit)); + } + + /// A Dynamic value's equality and text walk it without recursion, so a deep caller-built value cannot exhaust the stack. + [Test] + public void DeepDynamicValuesCompareAndPrintWithoutRecursion() + { + Dynamic Deep() + { + Dynamic value = Dynamic.Of(1L); + for (int i = 0; i < 200_000; i++) + { + value = Dynamic.Some(value); + } + + return value; + } + + Assert.That(Deep().Equals(Deep()), Is.True); + Assert.That(Deep().ToString(), Has.Length.LessThanOrEqualTo(4100)); + Assert.That(Dynamic.ListOf(Dynamic.Of(1L), Dynamic.Of(2.5)).ToString(), Is.EqualTo("List(Int(1), Real(2.5))")); + Assert.That(Dynamic.VariantOf("Job", Dynamic.Of("a")).ToString(), Is.EqualTo("Variant(Job, Text(a))")); + Assert.That(Dynamic.Of(double.NaN), Is.EqualTo(Dynamic.Of(double.NaN))); + Assert.That(Dynamic.Of(0.0), Is.Not.EqualTo(Dynamic.Of(-0.0))); + + // Encoding a deep value is refused by the depth limit before it recurses past it. + Assert.That(Refusal(() => Fixture(Limits.Hard).Encode(Codecs.Dynamic, Deep())), Does.Contain("JSON depth")); + } + + /// + /// A caller's name over 4,096 chars is refused before it is looked up, + /// hashed, or converted, quoting its first 64 chars and its length: a + /// public name, a constructor by name (dynamic, made, or a record's), + /// and an operation. A name of exactly 4,096 chars is looked up, and + /// names nothing. + /// + [Test] + public void AnOverlongNameIsRefusedBeforeItIsLookedUp() + { + string huge = new('X', 1_000_000); + string excerpt = new string('X', 64) + "... (1000000 chars)"; + ProgramData data = Fixture(Limits.Hard); + Assert.That(Refusal(() => Identities.DisplayToStore(huge)), Is.EqualTo("a name " + excerpt + " exceeds 4096 chars")); + Assert.That(Refusal(() => data.Identity(huge)), Is.EqualTo("public name " + excerpt + " exceeds 4096 chars")); + Assert.That(Refusal(() => data.Make(huge)), Is.EqualTo("constructor name " + excerpt + " exceeds 4096 chars")); + JacquardException dynamic = Support.Throws(() => data.Encode(Codecs.Dynamic, Dynamic.VariantOf(huge))); + Assert.That(dynamic.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(dynamic.Message, Is.EqualTo("value: constructor name " + excerpt + " exceeds 4096 chars")); + Assert.That(Refusal(() => data.Encode(Codecs.Record(huge, () => 1L), 1L)), Is.EqualTo("value: constructor name " + excerpt + " exceeds 4096 chars")); + Assert.That(Refusal(() => data.Decode(Codecs.Record(huge, () => 1L), data.Encode(Codecs.Unit, Unit.Value))), Does.Contain("constructor name " + excerpt + " exceeds 4096 chars")); + Assert.That(Refusal(() => new Operations().On(huge, Codecs.Int, () => 1L)), Is.EqualTo("operation name " + excerpt + " exceeds 4096 chars")); + string longest = new('X', 4096); + Assert.That(Refusal(() => data.Encode(Codecs.Dynamic, Dynamic.VariantOf(longest))), Does.StartWith("value: unknown constructor ")); + Assert.That(Identities.DisplayToStore("JobTicket"), Is.EqualTo("job-ticket")); + } + + /// + /// A lazy codec whose factory throws keeps nothing: the next use runs + /// the factory again, and once it succeeds its codec is kept and the + /// factory never runs again, across threads. + /// + [Test] + public void ALazyCodecWhoseFactoryFailedIsMadeOnTheNextUse() + { + ProgramData data = Fixture(Limits.Hard); + int calls = 0; + LazyCodec codec = Codecs.Lazy(() => Interlocked.Increment(ref calls) <= 2 ? throw new InvalidOperationException("not ready") : Codecs.Int); + for (int attempt = 1; attempt <= 2; attempt++) + { + JacquardException error = Support.Throws(() => data.Encode(codec, 1L)); + Assert.That(error.InnerException, Is.InstanceOf().With.Message.EqualTo("not ready")); + Assert.That(calls, Is.EqualTo(attempt)); + } + + Parallel.For(0, 16, i => Assert.That(data.Decode(codec, data.Encode(codec, i)), Is.EqualTo((long)i))); + Assert.That(calls, Is.EqualTo(3), "the codec was made more than once after it succeeded"); + } + + /// + /// Equality compares each pair of shared sub-values once: two separately + /// built values of 64 levels, each level a tuple holding the level below + /// twice (2^64 paths), compare in time proportional to their nodes. + /// + [Test] + public void EqualityComparesSharedSubValuesOnce() + { + static Dynamic Layered(int depth, long leaf) + { + Dynamic node = Dynamic.Of(leaf); + for (int i = 0; i < depth; i++) + { + node = Dynamic.TupleOf(node, node); + } + + return node; + } + + Task<(bool Same, bool Different)> comparison = Task.Run(() => (Layered(64, 1).Equals(Layered(64, 1)), Layered(64, 1).Equals(Layered(64, 2)))); + Assert.That(comparison.Wait(TimeSpan.FromMinutes(1)), Is.True, "the comparison walks every path"); + Assert.That(comparison.Result, Is.EqualTo((true, false))); + } + + /// + /// Equality keeps a set of compared pairs only once it meets a shared + /// sub-value: two separately built lists of 1,000 tuples compare without + /// one (a test seam counts the sets made), and a value holding one leaf + /// twice makes one. + /// + [Test] + public void EqualityOfUnsharedValuesMakesNoPairSet() + { + static Dynamic Tree(long leaf) => new Dynamic.List(Enumerable.Range(0, 1000).Select(i => Dynamic.TupleOf(Dynamic.Of(i), new Dynamic.Option(Dynamic.Of(leaf))))); + Dynamic.PairSetsMade = 0; + Assert.That(Tree(1).Equals(Tree(1)), Is.True); + Assert.That(Tree(1).Equals(Tree(2)), Is.False); + Assert.That(Dynamic.PairSetsMade, Is.EqualTo(0)); + Dynamic leaf = Dynamic.Of(5); + Assert.That(Dynamic.TupleOf(leaf, leaf).Equals(Dynamic.TupleOf(Dynamic.Of(5), Dynamic.Of(5))), Is.True); + Assert.That(Dynamic.PairSetsMade, Is.EqualTo(1)); + } + + /// + /// Messages quote caller text as an excerpt, and no message is kept + /// past 8,192 chars: a result codec naming a huge constructor, codecs, + /// record readers, and constructors that throw huge messages each give a + /// short refusal that keeps the caller's exception. + /// + [Test] + public void MessagesQuoteCallerTextAsAnExcerpt() + { + string huge = new('X', 1_000_000); + string excerpt = new string('X', 64) + "... (1000000 chars)"; + ProgramData data = Fixture(Limits.Hard); + Value unit = data.Encode(Codecs.Unit, Unit.Value); + JacquardException decode = Support.Throws(() => data.Decode(Codecs.Record(huge, () => 1L), unit)); + Assert.That(decode.Message, Has.Length.LessThan(200)); + Assert.That(decode.Message, Does.Contain("... (1000000 chars)")); + Codec throwing = Codecs.Create((encoder, value) => throw new InvalidOperationException(huge), (decoder, value) => throw new InvalidOperationException(huge)); + JacquardException encode = Support.Throws(() => data.Encode(throwing, 1L)); + Assert.That(encode.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(encode.Message, Is.EqualTo("value: its codec failed: System.InvalidOperationException: " + excerpt)); + Assert.That(encode.InnerException, Is.InstanceOf(), "the codec's exception was not kept"); + JacquardException decoded = Support.Throws(() => data.Decode(throwing, data.Encode(Codecs.Int, 1L))); + Assert.That(decoded.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(decoded.Message, Is.EqualTo("value: its codec failed: System.InvalidOperationException: " + excerpt)); + Assert.That(decoded.InnerException, Is.InstanceOf()); + + // A record constructor that refuses its fields, and a reader that throws. + RecordCodec refusing = Codecs.Record( + "Incident", + Codecs.Field("id", Codecs.Text, (Incident incident) => incident.Id), + Codecs.Field("severity", Codecs.Int, (Incident incident) => throw new ArgumentException(huge)), + Codecs.Field("owner", Codecs.Nullable(Codecs.Text), (Incident incident) => incident.Owner), + (Func)((id, severity, owner) => throw new ArgumentException(huge))); + Value incident = data.Encode(IncidentCodec, new Incident("ok", 1, null)); + JacquardException refused = Support.Throws(() => data.Decode(refusing, incident)); + Assert.That(refused.Message, Is.EqualTo("Incident refused the decoded fields: System.ArgumentException: " + excerpt)); + Assert.That(refused.InnerException, Is.InstanceOf(), "the constructor's refusal was not kept"); + JacquardException reader = Support.Throws(() => data.Encode(refusing, new Incident("ok", 1, null))); + Assert.That(reader.Message, Is.EqualTo("value: cannot read Incident.severity: System.ArgumentException: " + excerpt)); + Assert.That(reader.InnerException, Is.InstanceOf()); + + // A caller's field name in a path. + Codec named = Codecs.Create((encoder, value) => encoder.Field(huge, Codecs.Text, null!), (decoder, value) => 0); + Assert.That(Refusal(() => data.Encode(named, 1L)), Does.StartWith("value." + excerpt + ": null has no Jacquard value")); + string bounded = JacquardException.Bounded(huge); + Assert.That(bounded, Has.Length.EqualTo(JacquardException.MessageLimit), "the cap is exact, its suffix included"); + Assert.That(bounded, Does.EndWith("X... (1000000 chars)")); + + // A caller's message through the public helpers is bounded before it is joined to a path. + Codec invalidating = Codecs.Create((encoder, value) => throw encoder.Invalid(huge), (decoder, value) => throw decoder.Mapping(huge)); + string invalid = Refusal(() => data.Encode(invalidating, 1L)); + Assert.That(invalid, Has.Length.LessThanOrEqualTo(JacquardException.MessageLimit)); + Assert.That(invalid, Does.StartWith("value: XXXX")); + Assert.That(invalid, Does.EndWith("X... (1000007 chars)"), "joined to its path once, the whole length noted"); + string mapping = Refusal(() => data.Decode(invalidating, data.Encode(Codecs.Int, 1L))); + Assert.That(mapping, Has.Length.EqualTo(JacquardException.MessageLimit)); + Assert.That(mapping, Does.EndWith("X... (1000000 chars)")); + + // A note larger than any string is not the SDK's, so it is not trusted. + string forged = new string('Y', JacquardException.MessageLimit - 31) + "... (9223372036854775807 chars)"; + Assert.That(forged, Has.Length.EqualTo(JacquardException.MessageLimit)); + Assert.That(JacquardException.Joined("p: ", forged), Does.EndWith("... (" + (forged.Length + 3).ToString(CultureInfo.InvariantCulture) + " chars)")); + Assert.That(JacquardException.Joined("p: ", null), Is.EqualTo("p: null")); + + // A cut that dropped a high surrogate at the boundary is still recognised as cut. + string emoji = new string('X', JacquardException.MessageLimit - 20) + "😀" + huge; + string cut = JacquardException.Bounded(emoji); + Assert.That(cut, Has.Length.EqualTo(JacquardException.MessageLimit - 1), "the cut did not land on the surrogate"); + Assert.That(JacquardException.Joined("batch 1: ", cut), Does.EndWith("... (" + (emoji.Length + 9).ToString(CultureInfo.InvariantCulture) + " chars)")); + string again = JacquardException.Joined("batch 1: ", mapping); + Assert.That(again, Has.Length.EqualTo(JacquardException.MessageLimit)); + Assert.That(again, Does.StartWith("batch 1: XXX").And.EndWith("X... (1000009 chars)")); + + // A caller's exception whose Message throws is described by its type alone. + Assert.That(Identities.Describe(new ThrowingMessage()), Is.EqualTo(typeof(ThrowingMessage).FullName)); + } + + private sealed class ThrowingMessage : Exception + { + public override string Message => throw new InvalidOperationException("no message"); + } + + internal abstract record Shape; + + internal sealed record Circle : Shape; + + internal sealed record Square : Shape; + + internal sealed record One; + + /// An explicit union whose first case is a narrower union: encoding asks the flattened cases in decoding's order, so both agree. + [Test] + public void EncodingAndDecodingAgreeOnNestedUnions() + { + ProgramData data = Fixture(Limits.Hard); + UnionCodec circle = Codecs.Union(Codecs.Record("routed", () => new Circle())); + UnionCodec outer = Codecs.Union(circle, Codecs.Record("invalid-incident", () => new Square())); + Assert.That(data.Decode(outer, data.Encode(outer, new Square())), Is.EqualTo(new Square())); + Assert.That(data.Decode(outer, data.Encode(outer, new Circle())), Is.EqualTo(new Circle())); + } + + /// + /// Nested unions are flattened on first use by one iterative walk, so + /// decoding asks each distinct case once, and 100,000 layers of a + /// two-case union over one record neither overflow the stack nor take + /// long: one distinct case, and a round trip. + /// + [Test] + public void NestedUnionsFlattenWithoutRecursion() + { + ProgramData data = Fixture(Limits.Hard); + UnionCodec codec = Codecs.Union(Codecs.Record("routed", () => new One())); + for (int i = 0; i < 100_000; i++) + { + codec = Codecs.Union(codec, codec); + } + + Assert.That(codec.LeafCount, Is.EqualTo(1)); + Value notRouted = data.Encode(Codecs.Int, 1L); + Assert.That(Support.Throws(() => data.Decode(codec, notRouted)).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(data.Decode(codec, data.Encode(codec, new One())), Is.EqualTo(new One())); + } + + /// A recursive union, reached through a lazy case, flattens to its distinct cases and round-trips, also from many threads at once. + internal abstract record Expr; + + internal sealed record Leaf(long Value) : Expr; + + internal sealed record Pair(Expr Left, Expr Right) : Expr; + + [Test] + public void ARecursiveUnionThroughALazyCaseRoundTrips() + { + var constructors = new Dictionary { ["leaf"] = Id('1'), ["pair"] = Id('2') }; + var data = new ProgramData(new Dictionary(), constructors, Array.Empty>(), Limits.Hard); + UnionCodec? expr = null; + LazyCodec self = Codecs.Lazy(() => expr!); + expr = Codecs.Union( + Codecs.Record("Leaf", Codecs.Field("value", Codecs.Int, (Leaf leaf) => leaf.Value), value => new Leaf(value)), + Codecs.Record("Pair", Codecs.Field("left", self, (Pair pair) => pair.Left), Codecs.Field("right", self, (Pair pair) => pair.Right), (left, right) => new Pair(left, right)), + self); + Expr tree = new Pair(new Leaf(1), new Pair(new Leaf(2), new Leaf(3))); + Parallel.For(0, 8, _ => Assert.That(data.Decode(expr, data.Encode(expr, tree)), Is.EqualTo(tree))); + Assert.That(expr.LeafCount, Is.EqualTo(2)); + } + + private static string Nested(int depth) => + "(" + string.Concat(Enumerable.Repeat("Option (", depth - 2)) + "Option Int" + new string(')', depth - 2) + ") ->{} Int"; + + /// + /// A signature's types may nest 64 levels, the protocol's value depth, + /// and no deeper. Nesting is refused before the parser recurses past it, + /// so 5,000 parentheses, and 100,000, are refused with a binding failure + /// rather than a stack overflow. + /// + [Test] + public void SignatureNestingIsBounded() + { + var names = new Dictionary { ["int"] = Id('1'), ["option"] = Id('7') }; + Assert.That(Identities.SignatureDepth, Is.EqualTo(64)); + BoundaryType type = Identities.ParseSignature(Nested(64), names).Parameters[0]; + int depth = 1; + while (type is NominalType { Arguments.Count: > 0 } nominal) + { + type = nominal.Arguments[0]; + depth++; + } + + Assert.That(depth, Is.EqualTo(64)); + const string deep = "signature nests types deeper than 64 levels, the protocol's value-depth limit (max_json_depth)"; + foreach (string text in new[] { Nested(65), Nested(2500), "(" + new string('(', 5000) + "Int" + new string(')', 5000) + ") ->{} Int" }) + { + JacquardException error = Support.Throws(() => Identities.ParseSignature(text, names)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(error.Message, Is.EqualTo(deep)); + } + + JacquardException huge = Support.Throws(() => + Identities.ParseSignature("(" + string.Concat(Enumerable.Repeat("Option (", 100_000)) + "Int" + new string(')', 100_000) + ") ->{} Int", names)); + Assert.That(huge.Message, Is.EqualTo("signature is longer than " + Identities.SignatureTokens.ToString(CultureInfo.InvariantCulture) + " tokens")); + string wide = "(" + string.Concat(Enumerable.Repeat("Int, ", Identities.SignatureNodes)) + "Int) ->{} Int"; + Assert.That(Support.Throws(() => Identities.ParseSignature(wide, names)).Message, Is.EqualTo("signature names more than 4096 types")); + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/ExamplesTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/ExamplesTests.cs new file mode 100644 index 0000000..cd1040c --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/ExamplesTests.cs @@ -0,0 +1,206 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using JacquardLang.Host.Examples; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// +/// The example projects run as commands through the .NET SDK against the +/// real Core named by JACQUARD_BIN and JACQUARD_PRELUDE: the +/// live tests are ignored without them, and fail instead when +/// JACQUARD_REQUIRE_LIVE is set. The usage test needs no Core. +/// +[TestFixture] +[Platform("Linux,MacOsX")] +public sealed class ExamplesTests +{ + private string root = string.Empty; + + private static string ExamplesDirectory => Path.GetFullPath(Path.Combine(SdkLiveTests.DotnetRoot(), "..", "examples")); + + [SetUp] + public void Prepare() => root = SdkSupport.Scratch("examples"); + + [TearDown] + public void CleanUp() => SdkSupport.RemoveScratch(root); + + /// One command's output and its failure, if any. + private sealed record Outcome(string Output, Exception? Failure) + { + internal bool Ok => Failure is null; + } + + private static Outcome Command(params string[] arguments) + { + var output = new StringWriter(); + try + { + Examples.Examples.Run(arguments, output, ExamplesDirectory); + return new Outcome(output.ToString(), null); + } + catch (Exception error) when (error is JacquardException or UsageException) + { + return new Outcome(output.ToString(), error); + } + } + + private static JsonObject Summary(string output) => + JsonCodec.DecodeObject(System.Text.Encoding.UTF8.GetBytes(output.Trim().Split('\n')[^1]), 4); + + [Test] + public void AllFourProjectsRunAsCommands() + { + SdkLiveTests.RequireLive(); + foreach (string project in Examples.Examples.Projects) + { + string workdir = Path.Combine(root, project); + Outcome outcome = Command("--workdir", workdir, project); + Assert.That(outcome.Ok, Is.True, project + ": " + outcome.Failure + "\n" + outcome.Output); + switch (project) + { + case "quickstart": + Assert.That(outcome.Output, Is.EqualTo("Order total 35: shipping 5\nOrder total 75: shipping 0\n")); + break; + case "planner": + Assert.That(outcome.Output, Does.Contain("\"score\": 16").And.Contain("planner.plan: ok")); + break; + case "triage": + IReadOnlyList rows = JsonCodec.Decode(System.Text.Encoding.UTF8.GetBytes(outcome.Output.Trim()), 16).AsArray()!; + Assert.That(rows.Select(row => row.AsObject()!["urgency"].AsString()), Is.EqualTo(new[] { "PAGE", "TODAY", "WATCH" }), outcome.Output); + break; + default: + Assert.That(outcome.Output, Does.Contain("\"$tag\": \"Fulfilled\"")); + Outcome repeated = Command("--workdir", workdir, project); + Assert.That(repeated.Ok && repeated.Output.Contains("DuplicateOrder", StringComparison.Ordinal) && repeated.Output.Contains("\"deliveries\":1", StringComparison.Ordinal), Is.True, repeated.Output); + break; + } + } + + string recovery = Path.Combine(root, "recovery"); + Outcome failed = Command("--workdir", recovery, "--fault", "publish-failed", "fulfillment"); + Assert.That(failed.Failure, Is.InstanceOf(), "publish failure: " + failed.Output); + var error = (JacquardException)failed.Failure!; + Assert.That(error.Failure, Is.EqualTo(Failure.OperationFailed)); + Assert.That(error.Result!.Code, Is.EqualTo("E1613")); + Assert.That(failed.Output, Does.Contain("\"pendingDeliveries\":1")); + + // A failed explicit delivery prints no count. + Outcome undelivered = Command("--workdir", recovery, "--fault", "publish-failed", "--deliver-pending", "fulfillment"); + Assert.That(undelivered.Failure is UsageException && !undelivered.Output.Contains("delivered", StringComparison.Ordinal), Is.True, undelivered.Output); + foreach (string expected in new[] { "{\"delivered\":1}", "{\"delivered\":0}" }) + { + Outcome delivered = Command("--workdir", recovery, "--deliver-pending", "fulfillment"); + Assert.That(delivered.Ok && delivered.Output.Contains(expected, StringComparison.Ordinal), Is.True, "deliver pending: " + delivered.Failure + " " + delivered.Output); + } + + string inspect = Path.Combine(root, "triage-inspect"); + Outcome inspected = Command("--workdir", Path.Combine(root, "triage"), "--inspect", inspect, "triage"); + Assert.That(inspected.Ok, Is.True, inspected.Failure?.ToString()); + Assert.That(File.Exists(inspect + ".0.json"), Is.True, "inspection not saved"); + Assert.That(File.GetUnixFileMode(inspect + ".0.json"), Is.EqualTo(UnixFileMode.UserRead | UnixFileMode.UserWrite)); + Assert.That(Command("unknown").Failure, Is.InstanceOf()); + Assert.That(Command("--fault", "nope", "fulfillment").Failure, Is.InstanceOf()); + } + + [Test] + public void PartialCompletionFaultsRecoverExplicitlyAcrossRestart() + { + SdkLiveTests.RequireLive(); + foreach ((string fault, string code, long delivered) in new[] { ("after-commit", "E1612", 1L), ("publish-failed", "E1613", 1L), ("after-delivery", "E1612", 0L) }) + { + string workdir = Path.Combine(root, fault); + Outcome faulted = Command("--workdir", workdir, "--fault", fault, "fulfillment"); + Assert.That(faulted.Failure, Is.InstanceOf(), fault); + Assert.That(((JacquardException)faulted.Failure!).Result!.Code, Is.EqualTo(code), fault); + Assert.That(faulted.Output, Does.Contain("\"bookings\":1"), fault); + + // A restart sees the committed booking; recovery is explicit. + Outcome recovered = Command("--workdir", workdir, "--deliver-pending", "fulfillment"); + Assert.That(recovered.Ok && recovered.Output.Contains("{\"delivered\":" + delivered + "}", StringComparison.Ordinal), Is.True, fault + ": " + recovered.Output); + Outcome repeated = Command("--workdir", workdir, "fulfillment"); + Assert.That(repeated.Ok && repeated.Output.Contains("DuplicateOrder", StringComparison.Ordinal) && repeated.Output.Contains("\"pendingDeliveries\":0", StringComparison.Ordinal), Is.True, fault + ": " + repeated.Output); + Assert.That(Summary(repeated.Output)["deliveries"], Is.EqualTo(Json.Of(1)), fault); + } + } + + /// + /// The store's temporary file is created exclusively, so a file already + /// at its name (not the store's) makes the commit fail and is left as it + /// was. + /// + [Test] + public void TheStoreRemovesOnlyTheTemporaryFileItCreated() + { + string inventory = Path.Combine(root, "inventory.json"); + Examples.Examples.Warehouse warehouse = Examples.Examples.Warehouse.Open(inventory, string.Empty); + + // The name the next commit will use: the store's counter plus one. + long upcoming = (long)typeof(Examples.Examples.Warehouse).GetField("next", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!.GetValue(null)! + 1; + string occupied = inventory + "." + Environment.ProcessId + "-" + upcoming + ".tmp"; + File.WriteAllText(occupied, "caller-owned"); + var order = new Examples.Examples.Order("o", new[] { new Examples.Examples.Line("sensor", 1) }); + Support.Throws(() => warehouse.ReserveOrder(order)); + Assert.That(File.ReadAllText(occupied), Is.EqualTo("caller-owned"), "a file the store did not create was changed or removed"); + + // The next commit takes the next name and succeeds. + Assert.That(warehouse.ReserveOrder(order), Is.InstanceOf()); + Assert.That(File.Exists(occupied), Is.True); + } + + /// Runs the command in a fresh process; returns its exit status, stdout, and stderr. + private (int Status, string Stdout, string Stderr) Process(params string[] arguments) + { + var info = new ProcessStartInfo(Support.Muxer()) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + WorkingDirectory = root, + }; + foreach (string argument in new[] { "exec", "--runtimeconfig", Path.Combine(AppContext.BaseDirectory, "JacquardLang.Host.Tests.runtimeconfig.json"), Path.Combine(AppContext.BaseDirectory, "JacquardExamples.dll") }.Concat(arguments)) + { + info.ArgumentList.Add(argument); + } + + info.Environment["JACQUARD_EXAMPLES"] = ExamplesDirectory; + info.Environment["DOTNET_GCHeapHardLimit"] = "0x10000000"; + using System.Diagnostics.Process process = System.Diagnostics.Process.Start(info)!; + var stdout = System.Threading.Tasks.Task.Run(() => process.StandardOutput.ReadToEnd()); + var stderr = System.Threading.Tasks.Task.Run(() => process.StandardError.ReadToEnd()); + if (!process.WaitForExit(120_000)) + { + process.Kill(entireProcessTree: true); + Assert.Fail("the command did not finish"); + } + + process.WaitForExit(); + return (process.ExitCode, stdout.Result, stderr.Result); + } + + [Test] + public void TheCommandExits2ForUsageErrors() + { + foreach (string[] arguments in new[] { new[] { "unknown" }, new[] { "--fault", "nope", "fulfillment" }, new[] { "--workdir" }, Array.Empty() }) + { + (int status, _, string stderr) = Process(arguments); + Assert.That(status, Is.EqualTo(2), string.Join(" ", arguments) + ": " + stderr); + Assert.That(stderr.Trim(), Is.Not.Empty, string.Join(" ", arguments) + ": no message"); + } + } + + [Test] + public void TheCommandExits1ForAJacquardFailure() + { + SdkLiveTests.RequireLive(); + (int status, string stdout, string stderr) = Process("--workdir", Path.Combine(root, "work"), "--fault", "publish-failed", "fulfillment"); + Assert.That(status, Is.EqualTo(1), stderr); + Assert.That(stderr, Does.Contain("E1613")); + Assert.That(stdout, Does.Contain("\"pendingDeliveries\":1")); + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs b/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs new file mode 100644 index 0000000..571fbbd --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs @@ -0,0 +1,133 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// +/// A fake Core: a shell script that logs its first argument, answers +/// --version, and runs a case for each other command. Every process a +/// case leaves behind records its pid in a *.pid or pids file; +/// kills them. +/// +internal sealed class FakeCore : IDisposable +{ + internal static readonly string First = new('a', 64); + internal static readonly string Second = new('b', 64); + + private FakeCore(string root, Runtime runtime) + { + Root = root; + Runtime = runtime; + } + + internal string Root { get; } + + internal Runtime Runtime { get; } + + internal string Binary => Path.Combine(Root, "jacquard"); + + internal PrepareOptions Options => new() { CacheDir = Path.Combine(Root, "cache") }; + + /// + /// A fake whose --version prints (unless + /// replaces it), and whose other commands + /// run (shell case arms). $ROOT + /// names the fake's directory. + /// + internal static FakeCore Create(string root, string version, string? versionCase, string cases) + { + // Owner-only whatever the umask: the cache under it must have no parent others may write. + Directory.CreateDirectory(root); + File.SetUnixFileMode(root, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + Directory.CreateDirectory(Path.Combine(root, "prelude")); + File.WriteAllText(Path.Combine(root, "prelude", "01-prim.jqd"), "prelude\n"); + File.WriteAllText(Path.Combine(root, "rules.jac"), "rules\n"); + string script = "#!/bin/sh\nROOT='" + root + "'\necho \"$1\" >> \"$ROOT/log\"\ncase \"$1\" in\n--version) " + + (versionCase ?? "echo 'jacquard " + version + "'") + " ;;\n" + cases + "\nesac\n"; + string binary = Path.Combine(root, "jacquard"); + File.WriteAllText(binary, script); + File.SetUnixFileMode(binary, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + var runtime = new Runtime(new RuntimeOptions { Binary = binary, Prelude = Path.Combine(root, "prelude") }); + return new FakeCore(root, runtime); + } + + /// A fake whose check fails with boom. + internal static FakeCore Failing(string root, string version) => Create(root, version, null, "check) echo 'boom' >&2; exit 3 ;;"); + + /// A fake whose check and hash succeed and whose run fails. + internal static FakeCore Hashing(string root) => + Create(root, Compat.CoreVersion, null, "check) ;;\nhash) echo '0:x " + new string('0', 64) + "' ;;\nrun) exit 4 ;;"); + + /// + /// A fake that prepares every source as two functions, first and + /// second, with identities and + /// , over a one-type prelude. + /// + internal static FakeCore Programs(string root) => Create(root, Compat.CoreVersion, null, + "check) echo 'first : () ->{} Int'; echo 'second : () ->{} Int' ;;\n" + + "hash) case \"$2\" in */prelude/*) echo '0:int " + new string('1', 64) + "' ;; *) echo '0:first " + First + "'; echo '1:second " + + Second + "' ;; esac ;;\n" + + "run) mkdir -p \"$4\" && echo objects > \"$4/objects\" ;;"); + + /// + /// A fake whose one function, first, uses the effect + /// Inventory, whose one operation is lookup. + /// + internal static FakeCore Effects(string root) => Create(root, Compat.CoreVersion, null, + "check) echo 'first : () ->{Inventory} Int' ;;\n" + + "hash) case \"$2\" in */prelude/*) echo '0:int " + new string('1', 64) + "' ;; *) echo '0:first " + First + "'; echo '1:inventory " + + new string('c', 64) + "'; echo '1:lookup " + new string('d', 64) + "' ;; esac ;;\n" + + "run) mkdir -p \"$4\" && echo objects > \"$4/objects\" ;;"); + + internal string Source(string name) + { + string source = Path.Combine(Root, name); + if (!File.Exists(source)) + { + File.WriteAllText(source, name + "\n"); + } + + return source; + } + + /// Prepares , which must fail. + internal JacquardException PrepareFails(string name = "rules.jac", CancellationToken cancellationToken = default) => + Support.Throws(() => Runtime.Prepare(new[] { Source(name) }, Options, cancellationToken))!; + + internal long Calls(string command) + { + string log = Path.Combine(Root, "log"); + return File.Exists(log) ? File.ReadAllLines(log).Count(line => line == command) : 0; + } + + internal bool StagingLeft() + { + string cache = Path.Combine(Root, "cache"); + return Directory.Exists(cache) && Directory.EnumerateDirectories(cache).Any(entry => Path.GetFileName(entry).StartsWith("prepare-", StringComparison.Ordinal)); + } + + /// The cache's entries, each a staging directory or a published entry, sorted. + internal List CacheEntries() => + Directory.EnumerateFileSystemEntries(Path.Combine(Root, "cache")).Select(Path.GetFileName).OrderBy(name => name, StringComparer.Ordinal).ToList()!; + + /// Waits for to hold a complete pid line and returns it. + internal int Pid(string name) => SdkSupport.PidFile(Path.Combine(Root, name)); + + public void Dispose() + { + foreach (string file in Directory.EnumerateFiles(Root).Where(path => path.EndsWith("pids", StringComparison.Ordinal) || path.EndsWith(".pid", StringComparison.Ordinal))) + { + foreach (string pid in File.ReadAllText(file).Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)) + { + Support.Kill(int.Parse(pid, CultureInfo.InvariantCulture)); + } + } + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/OperationsTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/OperationsTests.cs new file mode 100644 index 0000000..9898705 --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/OperationsTests.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// .NET handlers adapted to operations: decoding, results, failures, the context and its token, and the stop check before a handler starts. +[TestFixture] +public sealed class OperationsTests +{ + private readonly ProgramData data = DataTests.Fixture(Limits.Hard); + + [TearDown] + public void ClearHook() => Operations.AfterDecodeHook = null; + + private static HandlerContext Context(Func? stopping = null, CancellationToken token = default) => new(stopping ?? (() => false), () => false, token); + + private static Value Text(string text) => new TextValue(text); + + private Value Call(Operations service, string name, HandlerContext context, params Value[] arguments) => + service.Snapshot(100)![name](data, context, arguments); + + private HandlerException Failure(Operations service, string name, params Value[] arguments) => + Support.Throws(() => Call(service, name, Context(), arguments)); + + [Test] + public void HandlersAdaptToOperations() + { + int calls = 0; + Operations service = new Operations() + .On("read-document", Codecs.Text, Codecs.Text, (context, key) => + { + calls++; + if (context.IsCancelled) + { + throw HandlerException.CompletionUnknown("cancelled"); + } + + return key == "example" ? "text" : throw HandlerException.Refused("unknown document"); + }) + .On("forget", Codecs.Text, Codecs.Unit, (string key) => throw new IOException("private detail")) + .On("lookup_stock", Codecs.Text, Codecs.Int, sku => (long)sku.Length) + .On("pair", Codecs.Tuple(Codecs.Int, Codecs.Int), () => (3L, 4L)) + .On("route", DataTests.IncidentCodec, DataTests.TriageCodec, incident => new DataTests.Routed(incident.Id, DataTests.Urgency.WatchOnly)) + .On("huge", Codecs.Int, () => long.MaxValue) + .On("nothing", Codecs.Text, () => null!) + .On("six", Codecs.Int, Codecs.Int, Codecs.Int, Codecs.Int, Codecs.Int, Codecs.Int, Codecs.Int, (context, a, b, c, d, e, f) => a + b + c + d + e + f); + Assert.That(Call(service, "read-document", Context(), Text("example")), Is.EqualTo(Text("text"))); + Assert.That(calls, Is.EqualTo(1)); + Assert.That(Failure(service, "read-document", Text("other")).Category, Is.EqualTo(FailureCategory.RefusedAuthority)); + HandlerException arity = Failure(service, "read-document"); + Assert.That(arity.Category, Is.EqualTo(FailureCategory.OutsideFailure)); + Assert.That(arity.Message, Is.EqualTo("handler parameter count differs from the request")); + Assert.That(Failure(service, "read-document", new IntValue(1)).Message, Is.EqualTo("handler argument cannot be mapped")); + Assert.That(calls, Is.EqualTo(2), "a handler ran with unmappable arguments"); + + // Any other exception reaches the host, which reports completion unknown without its message. + Support.Throws(() => Call(service, "forget", Context(), Text("x"))); + Assert.That(Call(service, "lookup-stock", Context(), Text("abc")), Is.EqualTo(new IntValue(3))); + Assert.That(Call(service, "pair", Context()), Is.EqualTo(new TupleValue(new IntValue(3), new IntValue(4)))); + Value incident = data.Encode(DataTests.IncidentCodec, new DataTests.Incident("i", 1, null)); + Assert.That(data.Decode(DataTests.TriageCodec, Call(service, "route", Context(), incident)), Is.EqualTo(new DataTests.Routed("i", DataTests.Urgency.WatchOnly))); + foreach (string unrepresentable in new[] { "huge", "nothing" }) + { + HandlerException result = Failure(service, unrepresentable); + Assert.That(result.Category, Is.EqualTo(FailureCategory.OutsideFailure)); + Assert.That(result.Message, Is.EqualTo("handler result cannot be represented")); + } + + Value[] six = { new IntValue(1), new IntValue(2), new IntValue(3), new IntValue(4), new IntValue(5), new IntValue(6) }; + Assert.That(Call(service, "six", Context(), six), Is.EqualTo(new IntValue(21))); + Operations replaced = new Operations().On("op", Codecs.Int, () => 1L).On("op", Codecs.Int, () => 2L); + Assert.That(Call(replaced, "op", Context()), Is.EqualTo(new IntValue(2))); + Assert.That(replaced.Count, Is.EqualTo(1)); + + // A service over a limit is refused from its count, before anything is copied. + Assert.That(service.Snapshot(3), Is.Null); + } + + /// The handler sees the invocation's context and its token. + [Test] + public void HandlersSeeTheContextAndItsToken() + { + using var stop = new CancellationTokenSource(); + CancellationToken seen = default; + Operations service = new Operations().On("watch", Codecs.Bool, context => + { + seen = context.CancellationToken; + return context.CancellationToken.IsCancellationRequested; + }); + Assert.That(Call(service, "watch", Context(token: stop.Token)), Is.EqualTo(data.Encode(Codecs.Bool, false))); + Assert.That(seen, Is.EqualTo(stop.Token)); + } + + /// + /// Decoding a request takes time. A stop that lands after the arguments + /// are decoded, here through the test seam, must keep the handler from + /// starting; the handler is never called. + /// + [Test] + public void NoHandlerStartsOnceTheCallStoppedDuringDecoding() + { + bool stopped = false; + int calls = 0; + Operations service = new Operations().On("act", Codecs.Text, Codecs.Unit, key => + { + calls++; + return Unit.Value; + }); + Operations.AfterDecodeHook = () => stopped = true; + HandlerException refused = Support.Throws(() => Call(service, "act", Context(() => stopped), Text("x"))); + Assert.That(refused.Message, Is.EqualTo("handler was not started")); + Assert.That(calls, Is.EqualTo(0), "the handler started after the call stopped"); + Operations.AfterDecodeHook = null; + Assert.That(Call(service, "act", Context(), Text("x")), Is.InstanceOf()); + Assert.That(calls, Is.EqualTo(1)); + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs new file mode 100644 index 0000000..ba8362c --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs @@ -0,0 +1,1946 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using JacquardLang.Host.Carrier; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// +/// Runtime options, and preparation against a fake Core: one version check +/// and prelude discovery per key across threads and runtimes, a child +/// holding a command's output killed, every Core command bounded by its +/// timeout, its output limit, and the token, bounded regular-file reads and +/// walks, a verified owner-only cache, and a bounded failure cleanup that +/// removes only what the preparation made. +/// +[TestFixture] +[Platform("Linux")] +public sealed class RuntimeTests +{ + private const string SlowCheck = "check) sleep 300 & echo $! > \"$ROOT/child.pid\"; wait ;;"; + + private string scratch = string.Empty; + + [SetUp] + public void Prepare() => scratch = SdkSupport.Scratch("runtime"); + + [TearDown] + public void CleanUp() + { + Runtime.FlightWaitHook = null; + Runtime.SourceReadHook = null; + Runtime.LoadedHook = null; + Runtime.PublishedHook = null; + Runtime.CleanupHook = null; + Runtime.CommandHook = null; + Runtime.RootHook = null; + Core.DigestChunkHook = null; + Core.TreeFileHook = null; + Core.TreeEntryCap = Core.TreeEntries; + SdkSupport.NoAccounts(); + SdkSupport.RemoveScratch(scratch); + } + + private string At(string name) => Path.Combine(scratch, name); + + private static string Excerpt(char c, int length) => new string(c, 64) + "... (" + length.ToString(CultureInfo.InvariantCulture) + " chars)"; + + [Test] + public void RuntimeOptionsAreValidatedBeforeCoreRuns() + { + var baseOptions = new RuntimeOptions { Prelude = scratch }; + Assert.That(Support.Throws(() => _ = new Runtime(baseOptions with { Deadline = TimeSpan.Zero })).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That( + Support.Throws(() => _ = new Runtime(baseOptions with { Limits = Limits.Hard.With("max_frame_bytes", 0) })).Message, + Does.StartWith("invalid runtime limits")); + var runtime = new Runtime(baseOptions with { Binary = "/no/such/jacquard" }); + foreach (string[] sources in new[] { new[] { "rules.txt" }, Array.Empty(), new[] { "a.jac", "b.jqd" }, new[] { ".jac" } }) + { + JacquardException error = Support.Throws(() => runtime.Prepare(sources)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Preparation), string.Join(",", sources)); + Assert.That(error.Message, Is.EqualTo("pass .jac files or .jqd files in dependency order")); + } + + Assert.That(Support.Throws(() => runtime.Prepare(new[] { "rules.jac" })).Message, Does.Contain("cannot find Core executable /no/such/jacquard")); + Assert.That(runtime.CoreCommands, Is.EqualTo(0)); + + // A name longer than any path is refused with an excerpt, never used or copied whole. + var huge = new Runtime(baseOptions with { Binary = "/" + new string('x', 1_000_000) }); + Assert.That(Support.Throws(() => huge.Prepare(new[] { "rules.jac" })).Message, Does.StartWith("cannot find Core executable /" + new string('x', 63) + "... (1000001 chars)")); + string longer = Support.Throws(() => new Runtime(baseOptions with { Binary = "/no/such/" + new string('y', 1000) }).Prepare(new[] { "rules.jac" })).Message; + Assert.That(longer, Has.Length.LessThan(200)); + Assert.That(longer, Does.Contain("... (1009 chars)")); + + // Paths longer than any path are refused with an excerpt before the file system sees them. + string source = Support.Throws(() => runtime.Prepare(new[] { "/" + new string('s', 1_000_000) + ".jac" })).Message; + Assert.That(source, Does.StartWith("source /sss").And.EndWith("(1000005 chars) is longer than any path")); + string cache = Support.Throws(() => runtime.Prepare(new[] { "rules.jac" }, new PrepareOptions { CacheDir = "/" + new string('c', 1_000_000) })).Message; + Assert.That(cache, Does.StartWith("cache directory /ccc")); + Assert.That(cache, Has.Length.LessThan(200)); + + // A relative path is measured as the file system will see it: 4,080 chars grow past the limit. + string relative = Support.Throws(() => runtime.Prepare(new[] { "rules.jac" }, new PrepareOptions { CacheDir = new string('c', 4080) })).Message; + Assert.That(relative, Does.StartWith("cache directory ccc").And.EndWith("is longer than any path")); + Assert.That(Directory.Exists(new string('c', 200)), Is.False, "a directory was made"); + JacquardException prelude = Support.Throws(() => _ = new Runtime(baseOptions with { Prelude = "/" + new string('p', 1_000_000) })); + Assert.That(prelude.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(prelude.Message, Has.Length.LessThan(200)); + foreach (string nul in new[] { "jac\0quard", "/tmp/jac\0quard" }) + { + JacquardException error = Support.Throws(() => new Runtime(baseOptions with { Binary = nul }).Prepare(new[] { "rules.jac" })); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Preparation)); + Assert.That(error.Message, Does.StartWith("cannot find Core executable")); + } + + Assert.That(Support.Throws(() => runtime.Prepare(new[] { "ru\0les.jac" })).Message, Does.StartWith("source ru").And.EndWith("is not a path")); + Support.Throws(() => runtime.Prepare(new string[] { null! })); + } + + [Test] + public void EmptyRuntimeOptionsMeanUnset() + { + var runtime = new Runtime(new RuntimeOptions { Binary = string.Empty, Prelude = scratch }); + string? bin = Environment.GetEnvironmentVariable("JACQUARD_BIN"); + Assert.That(runtime.Binary, Is.EqualTo(string.IsNullOrEmpty(bin) ? "jacquard" : bin)); + Assert.That(runtime.Prelude, Is.EqualTo(Path.GetFullPath(scratch))); + } + + [Test] + public void VersionIsCheckedOnceAcrossThreadsAndRuntimes() + { + using FakeCore fake = FakeCore.Failing(At("once"), Compat.CoreVersion); + JacquardException[] errors = Enumerable.Range(0, 4).AsParallel().WithDegreeOfParallelism(4).Select(_ => fake.PrepareFails()).ToArray(); + foreach (JacquardException error in errors) + { + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Preparation)); + Assert.That(error.Message, Is.EqualTo("Core check failed: boom")); + } + + Assert.That(fake.Calls("--version"), Is.EqualTo(1)); + Assert.That(fake.Calls("check"), Is.EqualTo(4)); + Assert.That(fake.StagingLeft(), Is.False, "a staging directory was left behind"); + + // Another runtime for the same Core shares the check. + var other = new Runtime(new RuntimeOptions { Binary = fake.Binary, Prelude = Path.Combine(fake.Root, "prelude") }); + Support.Throws(() => other.Prepare(new[] { fake.Source("rules.jac") }, fake.Options)); + Assert.That(fake.Calls("--version"), Is.EqualTo(1)); + Assert.That(other.CoreCommands, Is.EqualTo(1)); + } + + /// + /// A Core command whose child keeps its output open does not hold the + /// preparation: the command's group is killed when it exits, so the child + /// dies and preparation goes on (and here fails at run). + /// + [Test] + public void ACoreCommandsChildHoldingItsOutputIsKilledAtTheExit() + { + Assume.That(CoreCommand.Setsid, Is.Not.Null, "no setsid at /usr/bin or /bin"); + using FakeCore fake = FakeCore.Create(At("holder"), Compat.CoreVersion, null, + "check) sleep 300 & echo $! > \"$ROOT/holder.pid\" ;;\nhash) echo '0:x " + new string('0', 64) + "' ;;\nrun) exit 4 ;;"); + Assert.That(fake.PrepareFails().Message, Is.EqualTo("Core run failed: exit status 4")); + SdkSupport.Dies("the command's child", fake.Pid("holder.pid")); + Assert.That(fake.StagingLeft(), Is.False); + } + + [Test] + public void AVersionMismatchIsRefusedAndCheckedAgain() + { + using FakeCore fake = FakeCore.Failing(At("mismatch"), "9.9.9"); + for (int i = 0; i < 2; i++) + { + string message = fake.PrepareFails().Message; + Assert.That(message, Does.Contain("expected pinned Core version").And.Contain("jacquard 9.9.9")); + } + + Assert.That(fake.Calls("--version"), Is.EqualTo(2)); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + [Test] + public void PreludeDiscoveryRunsOncePerCoreAndPrelude() + { + using FakeCore fake = FakeCore.Hashing(At("prelude")); + foreach (string source in new[] { "rules.jac", "other.jac" }) + { + Assert.That(fake.PrepareFails(source).Message, Is.EqualTo("Core run failed: exit status 4")); + } + + // Three prelude files once, then one local hash per source. + Assert.That(fake.Calls("hash"), Is.EqualTo(5)); + Assert.That(fake.Calls("--version"), Is.EqualTo(1)); + Assert.That(fake.Calls("run"), Is.EqualTo(2)); + Assert.That(fake.StagingLeft(), Is.False); + } + + [Test] + public void CoreOutputBeyondItsLimitIsRefused() + { + using FakeCore fake = FakeCore.Create(At("overflow"), Compat.CoreVersion, null, "check) head -c 9000000 /dev/zero ;;"); + Assert.That(fake.PrepareFails().Message, Is.EqualTo("Core check output exceeds 8 MiB")); + } + + [Test] + public void ACoreCommandThatOutlivesItsTimeoutIsKilledWithItsChild() + { + using FakeCore fake = FakeCore.Create(At("timeout"), Compat.CoreVersion, null, SlowCheck); + fake.Runtime.CommandTimeout = TimeSpan.FromMilliseconds(500); + Assert.That(fake.PrepareFails().Message, Is.EqualTo("Core check failed: no exit within 500 ms")); + SdkSupport.Dies("the timed-out command's child", fake.Pid("child.pid")); + Assert.That(fake.StagingLeft(), Is.False); + } + + [Test] + public void CancellationStopsTheRunningCoreCommand() + { + using FakeCore fake = FakeCore.Create(At("cancel"), Compat.CoreVersion, null, SlowCheck); + using var cancel = new CancellationTokenSource(); + Thread canceller = Support.After(TimeSpan.Zero, () => + { + fake.Pid("child.pid"); + cancel.Cancel(); + }); + Assert.That(fake.PrepareFails("rules.jac", cancel.Token).Message, Is.EqualTo("Core check failed: preparation was cancelled")); + SdkSupport.Dies("the cancelled command's child", fake.Pid("child.pid")); + canceller.Join(); + + // The cleanup observes the stop too, so the unpublished staging directory stays. + Assert.That(fake.StagingLeft(), Is.True); + } + + /// + /// One preparation runs a slow --version; another waits for that + /// check. The waiter observes its own cancellation while the first still + /// runs, and never runs the command. + /// + [Test] + public void APreparationWaitingForAnotherObservesItsCancellation() + { + string slowVersion = "sleep 300 & echo $! > \"$ROOT/version.pid\"; wait; echo 'jacquard " + Compat.CoreVersion + "'"; + using FakeCore fake = FakeCore.Create(At("waiter"), Compat.CoreVersion, slowVersion, "check) exit 3 ;;"); + using var first = new CancellationTokenSource(); + Task owner = Task.Factory.StartNew(() => fake.PrepareFails("rules.jac", first.Token), CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); + int sleeper = fake.Pid("version.pid"); + + // The seam cancels each waiter only once it waits, so the wait itself must notice. + using var waiter = new CancellationTokenSource(); + Runtime.FlightWaitHook = waiter.Cancel; + Assert.That(fake.PrepareFails("rules.jac", waiter.Token).Message, Is.EqualTo("preparation was cancelled")); + Runtime.FlightWaitHook = null; + Assert.That(Support.Alive(sleeper), Is.True, "the first preparation's command stopped"); + Assert.That(fake.Calls("--version"), Is.EqualTo(1)); + first.Cancel(); + Assert.That(owner.Wait(TimeSpan.FromMinutes(1)), Is.True); + Assert.That(owner.Result.Message, Is.EqualTo("Core --version failed: preparation was cancelled")); + SdkSupport.Dies("the first preparation's command", sleeper); + } + + /// + /// Prepares on another thread, which must finish within a minute. If it + /// is blocked opening , a writer is opened to + /// release it and the test fails. + /// + private static JacquardException PreparesWithoutBlocking(FakeCore fake, string[] sources, string fifo) + { + Task preparing = Task.Factory.StartNew( + () => + { + try + { + fake.Runtime.Prepare(sources, fake.Options); + return null; + } + catch (JacquardException error) + { + return error; + } + }, + CancellationToken.None, + TaskCreationOptions.LongRunning, + TaskScheduler.Default); + if (!preparing.Wait(TimeSpan.FromMinutes(1))) + { + using (var writer = new FileStream(fifo, FileMode.Open, FileAccess.Write)) + { + writer.WriteByte((byte)'\n'); + } + + preparing.Wait(TimeSpan.FromMinutes(1)); + Assert.Fail("the preparation blocked on " + fifo); + } + + Assert.That(preparing.Result, Is.Not.Null, "the preparation succeeded"); + return preparing.Result!; + } + + [Test] + public void ASourceThatIsNotARegularFileIsRefusedWithoutBlocking() + { + using FakeCore fake = FakeCore.Failing(At("kinds"), Compat.CoreVersion); + string fifo = Path.Combine(fake.Root, "fifo.jac"); + SdkSupport.MakeFifo(fifo); + Assert.That(PreparesWithoutBlocking(fake, new[] { fifo }, fifo).Message, Is.EqualTo("cannot read " + fifo + ": not a regular file")); + string directory = Directory.CreateDirectory(Path.Combine(fake.Root, "directory.jac")).FullName; + Assert.That(Support.Throws(() => fake.Runtime.Prepare(new[] { directory }, fake.Options)).Message, Is.EqualTo("cannot read " + directory + ": not a regular file")); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + + // A symbolic link to a regular file is read. + string link = Path.Combine(fake.Root, "link.jac"); + File.CreateSymbolicLink(link, fake.Source("rules.jac")); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(new[] { link }, fake.Options)).Message, Is.EqualTo("Core check failed: boom")); + } + + [Test] + public void SourcesOverTheCeilingAreRefusedBeforeTheyAreRead() + { + using FakeCore fake = FakeCore.Failing(At("ceiling"), Compat.CoreVersion); + var read = new List(); + Runtime.SourceReadHook = path => + { + lock (read) + { + read.Add(path); + } + }; + + // Sparse and unreadable, so a refusal that is not "cannot read" was made from the size alone. + string one = SdkSupport.Sparse(Path.Combine(fake.Root, "one.jac"), Runtime.SourceLimit + 1); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(new[] { one }, fake.Options)).Message, Is.EqualTo("sources exceed 16 MiB in total")); + string half = SdkSupport.Sparse(Path.Combine(fake.Root, "half.jac"), Runtime.SourceLimit / 2, UnixFileMode.UserRead); + string other = SdkSupport.Sparse(Path.Combine(fake.Root, "other.jac"), Runtime.SourceLimit / 2); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(new[] { half, other }, fake.Options)).Message, Is.EqualTo("sources exceed 16 MiB in total")); + Assert.That(read, Is.EqualTo(new[] { half }), "a source past the ceiling was read"); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + [Test] + public void ACancellationIsObservedBetweenSourceFiles() + { + using FakeCore fake = FakeCore.Failing(At("between"), Compat.CoreVersion); + string[] sources = { fake.Source("a.jac"), fake.Source("b.jac") }; + var read = new List(); + using var cancel = new CancellationTokenSource(); + Runtime.SourceReadHook = path => + { + read.Add(path); + cancel.Cancel(); + }; + Assert.That(Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options, cancel.Token)).Message, Is.EqualTo(Core.Aborted)); + Assert.That(read, Is.EqualTo(new[] { sources[0] })); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + [Test] + public void ThePreludeWalkReadsOnlyBoundedRegularFiles() + { + using FakeCore fake = FakeCore.Failing(At("walk"), Compat.CoreVersion); + string big = SdkSupport.Sparse(Path.Combine(fake.Root, "prelude", "zz-big.jqd"), Core.TreeLimit + 1); + Assert.That(fake.PrepareFails().Message, Does.StartWith("cannot read prelude").And.EndWith("exceed 256 MiB")); + File.Delete(big); + string fifo = Path.Combine(fake.Root, "prelude", "zz-fifo.jqd"); + SdkSupport.MakeFifo(fifo); + Assert.That(PreparesWithoutBlocking(fake, new[] { fake.Source("rules.jac") }, fifo).Message, Does.EndWith("unsupported filesystem entry in prepared artifacts: zz-fifo.jqd")); + File.Delete(fifo); + File.WriteAllText(Path.Combine(fake.Root, "prelude", "02-more.jqd"), "more\n"); + var digested = new List(); + using var cancel = new CancellationTokenSource(); + Core.TreeFileHook = path => + { + digested.Add(path); + cancel.Cancel(); + }; + Assert.That(fake.PrepareFails("rules.jac", cancel.Token).Message, Does.EndWith(Core.Aborted)); + + // The walk streams each directory, so either file may come first. + Assert.That(digested, Has.Count.EqualTo(1)); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + /// A walk thousands of directories deep keeps an explicit stack, so it is refused by the entry cap or read, never by the call stack. + [Test] + public void ADeepPreludeTreeIsWalkedWithoutRecursion() + { + using FakeCore fake = FakeCore.Failing(At("deep"), Compat.CoreVersion); + string directory = Path.Combine(fake.Root, "prelude"); + for (int i = 0; i < 1500; i++) + { + directory = Path.Combine(directory, "d"); + } + + Directory.CreateDirectory(directory); + File.WriteAllText(Path.Combine(directory, "leaf.jqd"), "leaf\n"); + Assert.That(fake.PrepareFails().Message, Is.EqualTo("Core check failed: boom")); + } + + [Test] + public void AChangedCacheEntryIsRefusedIncludingItsMetadata() + { + using FakeCore fake = FakeCore.Programs(At("verified")); + string[] sources = { fake.Source("rules.jac") }; + PreparedProgram program = fake.Runtime.Prepare(sources, fake.Options); + Assert.That(program.Data.Identity("first").Hex, Is.EqualTo(FakeCore.First)); + Assert.That(File.Exists(Path.Combine(program.Directory, Runtime.HashOutput)), Is.True); + int commands = fake.Runtime.CoreCommands; + Assert.That(fake.Runtime.Prepare(sources, fake.Options).Directory, Is.EqualTo(program.Directory)); + Assert.That(fake.Runtime.CoreCommands, Is.EqualTo(commands), "a warm preparation ran Core"); + string metadata = Path.Combine(program.Directory, "program.json"); + string original = File.ReadAllText(metadata); + var changes = new[] + { + ("[\"first\",\"" + FakeCore.First + "\"]", "[\"first\",\"" + FakeCore.Second + "\"]"), + ("{\"hash\":\"" + FakeCore.First + "\",\"name\":\"first\"}", "{\"hash\":\"" + FakeCore.Second + "\",\"name\":\"first\"}"), + ("\"first\":\"() ->{} Int\"", "\"first\":\"() ->{} Text\""), + }; + foreach ((string from, string to) in changes) + { + Replace(metadata, from, to); + JacquardException error = Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options), "a changed program.json was accepted: " + to); + Assert.That(error.Message, Does.Contain("program.json bindings differ from Core's cached output")); + File.WriteAllText(metadata, original); + } + + Assert.That(fake.Runtime.Prepare(sources, fake.Options).Directory, Is.EqualTo(program.Directory)); + string hash = Path.Combine(program.Directory, Runtime.HashOutput); + string output = File.ReadAllText(hash); + Replace(hash, "0:first " + FakeCore.First, "0:first " + FakeCore.Second); + string changed = File.ReadAllText(hash); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options)).Message, Does.Contain("cached artifact digests do not match")); + Replace(metadata, Core.Digest(Encoding.UTF8.GetBytes(output)), Core.Digest(Encoding.UTF8.GetBytes(changed))); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options)).Message, Does.Contain("program.json bindings differ")); + + // A file replaced by a symbolic link is refused, not followed. + File.Delete(metadata); + File.CreateSymbolicLink(metadata, hash); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options)).Message, Does.Contain("unsupported filesystem entry in prepared artifacts: program.json")); + Assert.That(fake.Runtime.CoreCommands, Is.EqualTo(commands)); + } + + private static void Replace(string file, string from, string to) + { + string text = File.ReadAllText(file); + Assert.That(text, Does.Contain(from), from + " is not in " + file); + File.WriteAllText(file, text.Replace(from, to, StringComparison.Ordinal)); + } + + /// + /// A new cache entry is owner-only throughout (Core's store included), + /// and the cache root, the entry, or any file or directory in it that its + /// group may write is refused on load; restoring the permission makes the + /// entry load again. + /// + [Test] + public void AGroupWritableCacheEntryIsRefused() + { + using FakeCore fake = FakeCore.Programs(At("owner")); + string[] sources = { fake.Source("rules.jac") }; + PreparedProgram program = fake.Runtime.Prepare(sources, fake.Options); + string entry = program.Directory; + var targets = Directory.EnumerateFileSystemEntries(entry, "*", SearchOption.AllDirectories).Append(entry).ToList(); + Assert.That(targets, Does.Contain(Path.Combine(entry, "store", "objects"))); + foreach (string path in targets) + { + UnixFileMode mode = File.GetUnixFileMode(path); + Assert.That(mode & ~(UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute), Is.EqualTo(UnixFileMode.None), path + " is not owner-only: " + mode); + } + + string[] checkedPaths = + { + Path.GetFullPath(fake.Options.CacheDir!), entry, Path.Combine(entry, "program.json"), Path.Combine(entry, Runtime.HashOutput), Path.Combine(entry, "core"), + Path.Combine(entry, "store"), Path.Combine(entry, "store", "objects"), + }; + foreach (string path in checkedPaths) + { + UnixFileMode original = File.GetUnixFileMode(path); + File.SetUnixFileMode(path, original | UnixFileMode.GroupWrite); + JacquardException error = Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options), "a group-writable " + path + " was accepted"); + Assert.That(error.Message, Does.Contain(path + " is group- or world-writable")); + File.SetUnixFileMode(path, original); + } + + Assert.That(fake.Runtime.Prepare(sources, fake.Options).Directory, Is.EqualTo(entry)); + } + + /// + /// A cache root this user does not own is refused before anything is + /// written to it. The root directory stands in for a foreign owner; a + /// foreign-owned entry inside an owned root would need another account, + /// so it is not tested. + /// + [Test] + public void AForeignOwnedCacheRootIsRefused() + { + Assume.That(Unix.EffectiveUser(), Is.Not.EqualTo(0u), "running as root"); + using FakeCore fake = FakeCore.Programs(At("foreign")); + JacquardException error = Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = "/" })); + Assert.That(error.Message, Does.Contain("cache path / is not owned by the current user")); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + private static Runtime RuntimeFor(FakeCore fake, string binary) => new(new RuntimeOptions { Binary = binary, Prelude = Path.Combine(fake.Root, "prelude") }); + + /// An executable over the 512 MiB fingerprint ceiling is refused from its size: it is sparse and unreadable, so a read would fail with "cannot read" instead. + [Test] + public void AnExecutableOverTheFingerprintCeilingIsRefusedBeforeItIsRead() + { + using FakeCore fake = FakeCore.Failing(At("huge"), Compat.CoreVersion); + string huge = SdkSupport.Sparse(Path.Combine(fake.Root, "huge-jacquard"), Core.ExecutableLimit + 1, UnixFileMode.UserWrite | UnixFileMode.UserExecute); + string message = Support.Throws(() => RuntimeFor(fake, huge).Prepare(new[] { fake.Source("rules.jac") }, fake.Options)).Message; + Assert.That(message, Does.StartWith("Core executable ").And.EndWith(" exceeds 512 MiB")); + } + + /// A cancellation during fingerprinting stops it after the piece being read (a test seam counts the pieces of a 16 MiB executable). + [Test] + public void AStopDuringFingerprintingIsObserved() + { + using FakeCore fake = FakeCore.Failing(At("digest"), Compat.CoreVersion); + string big = SdkSupport.Sparse(Path.Combine(fake.Root, "big-jacquard"), 16L << 20, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + int pieces = 0; + using var cancel = new CancellationTokenSource(); + Core.DigestChunkHook = () => + { + pieces++; + cancel.Cancel(); + }; + Assert.That(Support.Throws(() => RuntimeFor(fake, big).Prepare(new[] { fake.Source("rules.jac") }, fake.Options, cancel.Token)).Message, Is.EqualTo(Core.Aborted)); + Assert.That(pieces, Is.EqualTo(1)); + Assert.That(fake.Calls("--version"), Is.EqualTo(0)); + } + + /// + /// A cancellation after the first of two empty prelude files stops the + /// walk before the second, whichever comes first: an empty file has no + /// piece to read, so only the check before each entry can see the stop. + /// + [Test] + public void AStopBetweenEmptyFilesIsObserved() + { + using FakeCore fake = FakeCore.Failing(At("empty"), Compat.CoreVersion); + File.WriteAllText(Path.Combine(fake.Root, "prelude", "01-prim.jqd"), string.Empty); + File.WriteAllText(Path.Combine(fake.Root, "prelude", "02-empty.jqd"), string.Empty); + var digested = new List(); + using var cancel = new CancellationTokenSource(); + Core.TreeFileHook = path => + { + digested.Add(path); + cancel.Cancel(); + }; + Assert.That(fake.PrepareFails("rules.jac", cancel.Token).Message, Does.EndWith(Core.Aborted)); + Assert.That(digested, Has.Count.EqualTo(1)); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + /// A tree over the entry cap is refused, zero-byte files counted (a test seam lowers the cap to 3). + [Test] + public void ATreeOverTheEntryCapIsRefused() + { + Assert.That(Core.TreeEntries, Is.EqualTo(100_000)); + using FakeCore fake = FakeCore.Failing(At("entries"), Compat.CoreVersion); + foreach (string name in new[] { "a.jqd", "b.jqd", "c.jqd" }) + { + File.WriteAllText(Path.Combine(fake.Root, "prelude", name), string.Empty); + } + + Core.TreeEntryCap = 3; + string prelude = Path.Combine(fake.Root, "prelude"); + Assert.That(fake.PrepareFails().Message, Does.StartWith("cannot read prelude").And.EndWith("more than 3 entries under " + prelude)); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + /// A source list that counts the elements read, and fails a read past limit. + private sealed class CountingList : IReadOnlyList + { + private readonly string element; + private readonly int limit; + private readonly Action onRead; + + internal CountingList(int count, string element, int limit, Action onRead) + { + Count = count; + this.element = element; + this.limit = limit; + this.onRead = onRead; + } + + internal List Indices { get; } = new(); + + public int Count { get; } + + public string this[int index] + { + get + { + if (Indices.Count >= limit) + { + throw new InvalidOperationException("read past the source cap"); + } + + Indices.Add(index); + onRead(index); + return element; + } + } + + public IEnumerator GetEnumerator() => throw new InvalidOperationException("the list was enumerated"); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// A source list over 1,024 is refused from its count, before any element is read. A list of exactly 1,024 passes that check. + [Test] + public void AHugeSourceListIsRefusedBeforeAnyElementIsRead() + { + Assert.That(Runtime.SourceFiles, Is.EqualTo(1024)); + using FakeCore fake = FakeCore.Programs(At("count")); + string source = fake.Source("rules.jac"); + var huge = new CountingList(int.MaxValue, source, 2048, _ => { }); + JacquardException error = Support.Throws(() => fake.Runtime.Prepare(huge, fake.Options)); + Assert.That(error.Message, Is.EqualTo("more than 1024 sources")); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Preparation)); + Assert.That(huge.Indices, Is.Empty); + var over = new CountingList(1025, source, 2048, _ => { }); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(over, fake.Options)).Message, Is.EqualTo("more than 1024 sources")); + Assert.That(over.Indices, Is.Empty); + var most = new CountingList(1024, source, 2048, _ => { }); + fake.Runtime.Prepare(most, fake.Options); + Assert.That(most.Indices, Has.Count.EqualTo(1024)); + } + + /// A cancellation while the list is validated stops it before the next element is read. + [Test] + public void AStopDuringSourceValidationIsObserved() + { + using FakeCore fake = FakeCore.Programs(At("validate")); + string source = fake.Source("rules.jac"); + using var cancel = new CancellationTokenSource(); + var cancelled = new CountingList(3, source, 3, index => + { + if (index == 1) + { + cancel.Cancel(); + } + }); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(cancelled, fake.Options, cancel.Token)).Message, Is.EqualTo(Core.Aborted)); + Assert.That(cancelled.Indices, Is.EqualTo(new[] { 0, 1 })); + Assert.That(fake.Calls("--version"), Is.EqualTo(0)); + } + + /// A fake whose run leaves ten files in its store and fails. + private static FakeCore FailingStore(string root) => FakeCore.Create(root, Compat.CoreVersion, null, + "check) echo 'first : () ->{} Int' ;;\nhash) echo '0:first " + FakeCore.First + "' ;;\n" + + "run) mkdir -p \"$4\" && for i in 0 1 2 3 4 5 6 7 8 9; do : > \"$4/f$i\"; done; exit 4 ;;"); + + /// + /// A failed preparation's cleanup removes its staging directory, but a + /// cancellation during the cleanup ends it at once (a test seam sets the + /// stop after the second removal), leaving the unpublished staging + /// directory; nothing is published either way. + /// + [Test] + public void AStopDuringTheFailureCleanupEndsIt() + { + using FakeCore fake = FailingStore(At("cleanup")); + var removed = new List(); + Runtime.CleanupHook = removed.Add; + Assert.That(fake.PrepareFails().Message, Does.EndWith("Core run failed: exit status 4")); + Assert.That(removed, Has.Count.EqualTo(13)); + Assert.That(fake.CacheEntries(), Is.Empty); + removed.Clear(); + using var cancel = new CancellationTokenSource(); + Runtime.CleanupHook = path => + { + removed.Add(path); + if (removed.Count == 2) + { + cancel.Cancel(); + } + }; + Assert.That(fake.PrepareFails("rules.jac", cancel.Token).Message, Is.EqualTo(Core.Aborted)); + Assert.That(removed, Has.Count.EqualTo(2)); + List left = fake.CacheEntries(); + Assert.That(left, Has.Count.EqualTo(1)); + Assert.That(left[0], Does.StartWith("prepare-")); + Assert.That(Directory.Exists(Path.Combine(fake.Root, "cache", left[0], "store")), Is.True); + } + + /// The failure cleanup removes at most the tree cap's entries (a test seam lowers it to 3), leaving the rest unpublished. + [Test] + public void TheFailureCleanupStopsAtTheCap() + { + using FakeCore fake = FailingStore(At("capped")); + var removed = new List(); + Runtime.CleanupHook = removed.Add; + Core.TreeEntryCap = 3; + Assert.That(fake.PrepareFails().Message, Does.EndWith("Core run failed: exit status 4")); + Assert.That(removed, Has.Count.EqualTo(3)); + List left = fake.CacheEntries(); + Assert.That(left, Has.Count.EqualTo(1)); + Assert.That(left[0], Does.StartWith("prepare-")); + } + + /// + /// A cache root reached through a symbolic link is checked as the + /// directory it names (the link's own mode reads 0777), and its entries + /// live there; a group-writable real root is refused with a hint. + /// + [Test] + public void ASymlinkedCacheRootIsFollowedToItsRealDirectory() + { + using FakeCore fake = FakeCore.Programs(At("link")); + string real = Directory.CreateDirectory(Path.Combine(fake.Root, "real-cache"), UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute).FullName; + string link = Path.Combine(fake.Root, "linked-cache"); + Directory.CreateSymbolicLink(link, real); + var options = new PrepareOptions { CacheDir = link }; + PreparedProgram program = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options); + Assert.That(Path.GetDirectoryName(program.Directory), Is.EqualTo(real)); + Assert.That(Directory.EnumerateFileSystemEntries(real).Count(), Is.EqualTo(1)); + Assert.That(fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options).Directory, Is.EqualTo(program.Directory)); + File.SetUnixFileMode(real, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options)).Message, + Is.EqualTo("cache path " + real + " is group- or world-writable; only its owner may write the cache (make it owner-only, for example chmod 700 " + real + ")")); + } + + /// A preparation removes only its own staging directory: directories it did not make, whatever their names or ages, are left alone. + [Test] + public void APreparationLeavesDirectoriesItDidNotMake() + { + using FakeCore fake = FakeCore.Programs(At("leaves")); + string cache = Directory.CreateDirectory(Path.Combine(fake.Root, "cache"), UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute).FullName; + string notes = Directory.CreateDirectory(Path.Combine(cache, "prepare-release-notes")).FullName; + File.WriteAllText(Path.Combine(notes, "draft.txt"), "keep"); + string left = Directory.CreateDirectory(Path.Combine(cache, "prepare-1-00000000-0")).FullName; + Directory.SetLastWriteTimeUtc(notes, DateTime.UtcNow.AddDays(-1)); + Directory.SetLastWriteTimeUtc(left, DateTime.UtcNow.AddDays(-1)); + string entry = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options).Directory; + Assert.That(File.ReadAllText(Path.Combine(notes, "draft.txt")), Is.EqualTo("keep")); + Assert.That(Directory.Exists(left), Is.True, "a directory the preparation did not make was removed"); + Assert.That(fake.CacheEntries(), Is.EqualTo(new[] { Path.GetFileName(entry), Path.GetFileName(left), Path.GetFileName(notes) }.OrderBy(name => name, StringComparer.Ordinal))); + } + + /// Once an entry is published its staging path is vacated, and whatever appears there afterwards is not the preparation's: it is left alone. + [Test] + public void ADirectoryAtTheVacatedStagingPathIsLeftAlone() + { + using FakeCore fake = FakeCore.Programs(At("vacated")); + string? replacement = null; + Runtime.PublishedHook = staging => + { + replacement = Directory.CreateDirectory(staging).FullName; + File.WriteAllText(Path.Combine(staging, "unrelated.txt"), "keep"); + }; + fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + Assert.That(File.ReadAllText(Path.Combine(replacement!, "unrelated.txt")), Is.EqualTo("keep"), "the preparation removed a directory it did not make"); + } + + /// A cancellation that lands as a warm load finishes fails the preparation instead of returning a program. + [Test] + public void AStopAsAWarmLoadFinishesIsReported() + { + using FakeCore fake = FakeCore.Programs(At("loaded")); + string[] sources = { fake.Source("rules.jac") }; + fake.Runtime.Prepare(sources, fake.Options); + using var cancel = new CancellationTokenSource(); + Runtime.LoadedHook = cancel.Cancel; + Assert.That(Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options, cancel.Token)).Message, Is.EqualTo(Core.Aborted)); + Runtime.LoadedHook = null; + + // Without a stop the same entry loads. + fake.Runtime.Prepare(sources, fake.Options); + } + + /// Replaces with a new executable file holding , as an atomic rename does. + private static void Replace(string path, string text) + { + string next = path + ".next"; + File.WriteAllText(next, text); + File.SetUnixFileMode(next, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + File.Move(next, path, overwrite: true); + } + + private static string ChangedMessage(string what, string path) => what + " " + path + " changed during preparation; prepare again once it no longer changes"; + + /// + /// The cache root is checked once and used by its path, so its identity + /// is checked again before each use. A test seam renames the checked + /// root away and puts a new directory in its place (as another user who + /// may write its parent could) just before the staging directory is + /// made, before publishing, before a load, and once a load has read the + /// entry, on a cold and on a warm preparation; each is refused. + /// + [Test] + public void ACacheRootReplacedAfterItsCheckIsRefused() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string cache = Path.GetFullPath(fake.Options.CacheDir!); + string Swap(string use, string source) + { + bool swapped = false; + Runtime.RootHook = at => + { + if (at == use && !swapped) + { + swapped = true; + Directory.Move(cache, cache + "-" + use + "-" + Path.GetFileNameWithoutExtension(source)); + Directory.CreateDirectory(cache, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + }; + string message = fake.PrepareFails(source).Message; + Runtime.RootHook = null; + Assert.That(swapped, Is.True, use + " was never reached"); + return message; + } + + string refused = "cache root " + cache + " was replaced after it was checked; prepare again"; + foreach (string use in new[] { "stage", "publish", "load", "loaded" }) + { + Assert.That(Swap(use, "cold-" + use + ".jac"), Is.EqualTo(refused), "cold " + use); + } + + foreach (string use in new[] { "load", "loaded" }) + { + string source = "warm-" + use + ".jac"; + fake.Runtime.Prepare(new[] { fake.Source(source) }, fake.Options); + Assert.That(Swap(use, source), Is.EqualTo(refused), "warm " + use); + } + + // Unswapped, the same root is used. + fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + } + + /// + /// No other user may write a directory above the cache root, so none can + /// rename the root away and put another directory at its path: a parent + /// that its group or others may write is refused, naming it, unless it + /// has the sticky bit (in which only an entry's owner may rename it). + /// Links in the root's path are resolved once, so the mode of a link + /// itself (always 0777) does not matter and its target is checked. + /// + [Test] + public void ACacheParentOthersMayWriteIsRefusedUnlessItIsSticky() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent"), UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute).FullName; + NotPrivate(parent); + var options = new PrepareOptions { CacheDir = Path.Combine(parent, "cache") }; + string Refused(UnixFileMode mode) + { + File.SetUnixFileMode(parent, mode); + return Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options))!.Message; + } + + const UnixFileMode Owner = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute; + string cache = Path.Combine(parent, "cache"); + Assert.That(Refused(Owner | UnixFileMode.OtherWrite | UnixFileMode.OtherExecute), Is.EqualTo(Unsafe(cache, parent + " (writable by others)")), "others may write"); + Assert.That(Refused(Owner | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute), Is.EqualTo(Unsafe(cache, parent + " (" + NotPrivateGroup + ")")), "the group may write"); + + // A grandparent is checked as well. + string nested = Directory.CreateDirectory(Path.Combine(parent, "nested"), Owner).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1FF); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = Path.Combine(nested, "cache") }))!.Message, + Is.EqualTo(Unsafe(Path.Combine(nested, "cache"), parent + " (writable by others)"))); + + // Sticky and world-writable, as /tmp is: accepted. + File.SetUnixFileMode(parent, (UnixFileMode)0x3FF); + Assert.That(fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options).Directory, Does.StartWith(Path.Combine(parent, "cache") + "/")); + File.SetUnixFileMode(parent, Owner | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute); + fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options); + + // Through a link (mode 0777) to the parent: the real path is used. + string link = At("link"); + File.CreateSymbolicLink(link, parent); + PreparedProgram linked = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = Path.Combine(link, "cache") }); + Assert.That(linked.Directory, Does.StartWith(Path.Combine(parent, "cache") + "/")); + } + + private const string NotPrivateGroup = "writable by its group, which is not your private group"; + + /// The refusal of a cache at listing . + private static string Unsafe(string cache, params string[] levels) => + "cache directory " + cache + " could be replaced by another user through the directories above it." + + " Make each of these writable only by you (for example chmod go-w DIR), or use a location under directories only you or root may write: " + + string.Join("; ", levels); + + /// + /// Answers the private-group lookups as a user "me" whose primary group + /// 's group is, named + /// with , and the only account with it. + /// + private static void Accounts(string directory, string name, params string[] members) => SdkSupport.Accounts(directory, name, members); + + /// Answers the lookups so that 's group is not the user's private group. + private static void NotPrivate(string directory) => Accounts(directory, "staff"); + + /// + /// A group-writable directory above the cache is accepted when its group + /// is the user's private group (Debian OpenSSH's rule): the directory's + /// group is the user's primary group, the group's name is the user's + /// name, and it lists no members or only the user. The password and + /// group answers come from a seam; the directory is 0770. + /// + [Test] + public void ACacheParentWritableByThePrivateGroupIsAccepted() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent"), (UnixFileMode)0x1F8).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1F8); + var options = new PrepareOptions { CacheDir = Path.Combine(parent, "cache") }; + Accounts(parent, "me"); + Assert.That(fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options).Directory, Does.StartWith(Path.Combine(parent, "cache") + "/"), "no members"); + Accounts(parent, "me", "me"); + Assert.That(fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options).Directory, Does.StartWith(Path.Combine(parent, "cache") + "/"), "the user alone"); + + // The private group does not excuse others' write. + File.SetUnixFileMode(parent, (UnixFileMode)0x1FA); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options))!.Message, + Is.EqualTo(Unsafe(Path.Combine(parent, "cache"), parent + " (writable by others)"))); + } + + /// + /// Any other group-writable directory above the cache is refused: a + /// group that is not the user's primary group, a group named otherwise, + /// a group with another member (alone or beside the user), and a user or + /// group the databases do not know. + /// + [Test] + public void ACacheParentWritableByAnyOtherGroupIsRefused() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent"), (UnixFileMode)0x1F8).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1F8); + var options = new PrepareOptions { CacheDir = Path.Combine(parent, "cache") }; + uint group = Unix.Stat(parent, follow: false).Group; + string expected = Unsafe(Path.Combine(parent, "cache"), parent + " (" + NotPrivateGroup + ")"); + void Refused(string what) => + Assert.That(Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options))!.Message, Is.EqualTo(expected), what); + + Core.UserHook = uid => new Unix.Account("me", group + 1); + Core.GroupHook = gid => new Unix.GroupEntry("me", Array.Empty()); + Refused("another primary group"); + Accounts(parent, "staff"); + Refused("a group named otherwise"); + Accounts(parent, "me", "other"); + Refused("another member"); + Accounts(parent, "me", "me", "other"); + Refused("another member beside the user"); + Core.UserHook = uid => null; + Refused("an unknown user"); + Accounts(parent, "me"); + Core.GroupHook = gid => null; + Refused("an unknown group"); + Assert.That(Directory.Exists(Path.Combine(parent, "cache")), Is.False, "a refused preparation made the root"); + } + + /// + /// The group database's member list omits accounts whose primary group a + /// group is, so the password database is enumerated as well: a 0770 + /// parent whose group is the user's private group, listing no members, + /// is refused when another account ("attacker", another uid) has that + /// group as its primary group, and accepted when only the user has it + /// (other accounts with other primary groups do not matter). + /// + [Test] + public void ACacheParentIsRefusedWhenAnotherAccountHasThePrivateGroupAsItsPrimaryGroup() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent")).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1F8); + uint group = Unix.Stat(parent, follow: false).Group; + uint other = Unix.EffectiveUser() + 1; + string cache = Path.Combine(parent, "cache"); + var options = new PrepareOptions { CacheDir = cache }; + SdkSupport.Accounts(parent, "me", Array.Empty(), new Unix.PasswdEntry("attacker", other, group)); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options))!.Message, + Is.EqualTo(Unsafe(cache, parent + " (" + NotPrivateGroup + ")"))); + Assert.That(Directory.Exists(cache), Is.False, "a refused preparation made the root"); + + SdkSupport.Accounts(parent, "me", Array.Empty(), new Unix.PasswdEntry("daemon", other, group + 1)); + Assert.That(fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options).Directory, Does.StartWith(cache + "/")); + } + + /// + /// One preparation lists the password database at most once, though it + /// checks a group-writable private-group parent twice when the cache + /// root is missing (before making it and again once it is resolved); a + /// later preparation, here on the entry made, lists it again, since the + /// verdict is not kept after the call. + /// + [Test] + public void APreparationListsThePasswordDatabaseAtMostOnce() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent")).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1F8); + string cache = Path.Combine(parent, "cache"); + SdkSupport.Accounts(parent, "me", Array.Empty()); + Func listing = Core.GetentHook!; + int listed = 0; + Core.GetentHook = token => + { + listed++; + return listing(token); + }; + var options = new PrepareOptions { CacheDir = cache }; + Assert.That(fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options).Directory, Does.StartWith(cache + "/")); + Assert.That(listed, Is.EqualTo(1), "a preparation that made the root"); + Assert.That(fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options).Directory, Does.StartWith(cache + "/")); + Assert.That(listed, Is.EqualTo(2), "a later preparation lists it again"); + } + + /// + /// The listing runs at most once in one preparation whatever the account + /// lookups answer: when the account is "alice" for the check before the + /// root is made and "alice-renamed" (same user id and group, each the + /// name of its private group, both listed) for the check after, the + /// password database is listed once and the second check refuses the + /// parent rather than listing again. + /// + [Test] + public void APreparationWhoseAccountChangesBetweenChecksListsOnceAndRefuses() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent")).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1F8); + string cache = Path.Combine(parent, "cache"); + uint group = Unix.Stat(parent, follow: false).Group; + uint me = Unix.EffectiveUser(); + string[] names = { "alice", "alice-renamed" }; + int looked = 0; + string current = names[0]; + Core.UserHook = uid => + { + current = names[Math.Min(looked++, 1)]; + return uid == me ? new Unix.Account(current, group) : null; + }; + Core.GroupHook = gid => gid == group ? new Unix.GroupEntry(current, Array.Empty()) : null; + Func listing = SdkSupport.Getent(SdkSupport.Passwd(names.Select(n => new Unix.PasswdEntry(n, me, group)))); + int listed = 0; + Core.GetentHook = token => + { + listed++; + return listing(token); + }; + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = cache }))!.Message, + Is.EqualTo(Unsafe(cache, parent + " (" + NotPrivateGroup + ")"))); + Assert.That(looked, Is.EqualTo(2), "both checks looked the account up"); + Assert.That(listed, Is.EqualTo(1), "one listing"); + } + + /// + /// The preparation's token reaches the password database listing: a + /// listing that sees the token cancelled stops the preparation as + /// cancelled, and nothing is made. + /// + [Test] + public void APreparationCancelledWhileThePasswordDatabaseIsListedStops() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent")).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1F8); + string cache = Path.Combine(parent, "cache"); + SdkSupport.Accounts(parent, "me", Array.Empty()); + Func listing = Core.GetentHook!; + using var cancel = new CancellationTokenSource(); + var seen = new List(); + Core.GetentHook = token => + { + seen.Add(token); + cancel.Cancel(); + throw CarrierException.Aborted("command aborted"); + }; + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = cache }, cancel.Token))!.Message, + Is.EqualTo(Core.Aborted)); + Assert.That(seen, Is.EqualTo(new[] { cancel.Token })); + Assert.That(Directory.Exists(cache), Is.False, "a cancelled preparation made the root"); + Core.GetentHook = listing; + Assert.That(fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = cache }).Directory, Does.StartWith(cache + "/")); + } + + /// + /// The password database listing (getent passwd's output and exit + /// status, from a seam) is parsed strictly and fails closed: only the + /// user with the group is accepted, and exactly the entry cap; another + /// account with the group as its primary group, a nonzero exit, a + /// timeout, output over the limit or cut short, a malformed line, + /// invalid UTF-8, a name listed twice, entries past the cap, and the user + /// missing or listed with other ids each mean "not private". A + /// cancellation is a cancelled preparation. + /// + [Test] + public void ThePasswordDatabaseListingIsParsedStrictlyAndFailsClosed() + { + uint me = Unix.EffectiveUser(); + const uint Group = 4242; + string self = SdkSupport.Passwd(new[] { new Unix.PasswdEntry("me", me, Group) }); + string Others(int count) => SdkSupport.Passwd(Enumerable.Range(0, count).Select(i => new Unix.PasswdEntry("user" + i.ToString(CultureInfo.InvariantCulture), me + 1 + (uint)i, Group + 1))); + bool Verdict(Func getent) + { + Core.GetentHook = getent; + return Core.OnlyUserHasPrimaryGroup("me", me, Group, CancellationToken.None); + } + + bool Listing(string stdout) => Verdict(SdkSupport.Getent(stdout)); + + Assert.That(Listing(self), Is.True, "only the user"); + Assert.That(Listing(self + Others(Core.PasswdEntries - 1)), Is.True, "exactly the cap"); + Assert.That(Listing(self + "alias:x:" + me.ToString(CultureInfo.InvariantCulture) + ":4242::/:/bin/sh\n"), Is.True, "another name for the same user id"); + Assert.That(Listing(self + "attacker:x:" + (me + 1).ToString(CultureInfo.InvariantCulture) + ":4242::/home/attacker:/bin/sh\n"), Is.False, "another account with the group as its primary group"); + Assert.That(Listing(self + Others(Core.PasswdEntries)), Is.False, "one past the cap"); + Assert.That(Verdict(SdkSupport.Getent(self, status: 2)), Is.False, "a nonzero exit"); + Assert.That(Verdict(SdkSupport.Getent(self, overflow: true)), Is.False, "output over the limit"); + Assert.That(Verdict(_ => throw CarrierException.Timeout("command did not exit in time")), Is.False, "a timeout"); + Assert.That(Verdict(_ => throw CarrierException.Loss("could not start getent")), Is.False, "a start failure"); + Assert.That(Listing(self + "daemon:x:1:1:daemon:/usr/sbin"), Is.False, "output cut short within a line"); + Assert.That(Listing(self[..^1]), Is.False, "output cut short before the newline"); + Assert.That(Listing(string.Empty), Is.False, "no output"); + Assert.That(Listing(self + "\n"), Is.False, "an empty line"); + Assert.That(Listing(self + "daemon:x:1:1:daemon:/usr/sbin\n"), Is.False, "six fields"); + Assert.That(Listing(self + "daemon:x:1:1:daemon:/usr/sbin:/bin/sh:extra\n"), Is.False, "eight fields"); + Assert.That(Listing(self + ":x:1:1:daemon:/usr/sbin:/bin/sh\n"), Is.False, "an empty name"); + foreach (string id in new[] { string.Empty, "-1", "+1", "01", "1a", " 1", "4294967296", "0x1" }) + { + Assert.That(Listing(self + "daemon:x:" + id + ":1::/:/bin/sh\n"), Is.False, "user id \"" + id + "\""); + Assert.That(Listing(self + "daemon:x:1:" + id + "::/:/bin/sh\n"), Is.False, "group id \"" + id + "\""); + } + + Assert.That(Listing(self + "daemon:x:0:0::/:/bin/sh\n"), Is.True, "id zero"); + byte[] invalid = Encoding.UTF8.GetBytes(self + "da?mon:x:1:1::/:/bin/sh\n"); + invalid[Array.IndexOf(invalid, (byte)'?')] = 0xFF; + Assert.That(Verdict(_ => new CoreCommand.Output(0, invalid, false, Array.Empty())), Is.False, "invalid UTF-8"); + Assert.That(Listing(self + self), Is.False, "the user listed twice"); + Assert.That(Listing(self + "daemon:x:1:1::/:/bin/sh\ndaemon:x:2:2::/:/bin/sh\n"), Is.False, "another name listed twice"); + Assert.That(Listing(Others(3)), Is.False, "the user is missing"); + Assert.That(Listing(SdkSupport.Passwd(new[] { new Unix.PasswdEntry("me", me, Group + 1) })), Is.False, "the user with another primary group"); + Assert.That(Listing(SdkSupport.Passwd(new[] { new Unix.PasswdEntry("me", me + 1, Group) })), Is.False, "the user's name with another user id"); + + var cancel = new CancellationTokenSource(); + CancellationToken seen = default; + Core.GetentHook = token => + { + seen = token; + cancel.Cancel(); + throw CarrierException.Aborted("command aborted"); + }; + Assert.That(Support.Throws(() => Core.OnlyUserHasPrimaryGroup("me", me, Group, cancel.Token))!.Message, Is.EqualTo(Core.Aborted)); + Assert.That(seen, Is.EqualTo(cancel.Token), "getent runs with the caller's token"); + } + + /// + /// getent runs through the bounded command runner: a real command + /// in its place (/bin/sh, root's) that prints a valid listing but + /// exits 1 is refused, one that does not exit within the time limit + /// (here 100 ms) is refused, and one whose token is cancelled once it has + /// started stops the preparation as cancelled. No command it started is + /// left running. + /// + [Test] + [Platform("Linux")] + public void ThePasswordDatabaseListingIsBoundedAndStoppedByTheToken() + { + uint me = Unix.EffectiveUser(); + const uint Group = 4242; + string self = "me:x:" + me.ToString(CultureInfo.InvariantCulture) + ":4242::/:/bin/sh"; + Core.GetentPaths = new[] { "/bin/sh" }; + Core.GetentArguments = new[] { "-c", "echo " + self + "; exit 0" }; + Assert.That(Core.OnlyUserHasPrimaryGroup("me", me, Group, CancellationToken.None), Is.True, "the stand-in's listing"); + Core.GetentArguments = new[] { "-c", "echo " + self + "; exit 1" }; + Assert.That(Core.OnlyUserHasPrimaryGroup("me", me, Group, CancellationToken.None), Is.False, "a nonzero exit"); + + var started = new List(); + CoreCommand.StartedHook = started.Add; + try + { + Core.GetentArguments = new[] { "-c", "echo " + self + "; exec sleep 30" }; + Core.GetentTimeoutCap = 100; + Assert.That(Core.OnlyUserHasPrimaryGroup("me", me, Group, CancellationToken.None), Is.False, "a timeout"); + Core.GetentTimeoutCap = Core.GetentTimeoutMs; + var cancel = new CancellationTokenSource(); + CoreCommand.StartedHook = pid => + { + started.Add(pid); + cancel.Cancel(); + }; + Assert.That(Support.Throws(() => Core.OnlyUserHasPrimaryGroup("me", me, Group, cancel.Token))!.Message, Is.EqualTo(Core.Aborted)); + } + finally + { + CoreCommand.StartedHook = null; + } + + Assert.That(started, Has.Count.EqualTo(2)); + foreach (int pid in started) + { + Assert.That(Directory.Exists("/proc/" + pid.ToString(CultureInfo.InvariantCulture)), Is.False, "the stopped command " + pid.ToString(CultureInfo.InvariantCulture) + " is gone"); + } + } + + /// + /// getent is the first of the configured paths present, and it + /// must be root's and not writable by group or others: a missing path is + /// passed over, an executable this user owns (which would print a valid + /// listing) is refused without falling through to a later path, and + /// getent is refused when its file (not a regular file) or a directory + /// above it is another user's or writable by group or others (statuses + /// altered through a seam). + /// + [Test] + [Platform("Linux")] + public void TheGetentExecutableMustBeRootsAndNotWritable() + { + uint me = Unix.EffectiveUser(); + Unix.Account account = Unix.User(me) ?? throw new AssertionException("no passwd entry for this user"); + string missing = Path.Combine(scratch, "missing"); + string mine = Path.Combine(scratch, "getent"); + File.WriteAllText(mine, "#!/bin/sh\necho '" + account.Name + ":x:" + me.ToString(CultureInfo.InvariantCulture) + ":" + account.Group.ToString(CultureInfo.InvariantCulture) + "::/:/bin/sh'\n"); + File.SetUnixFileMode(mine, (UnixFileMode)0x1ED); + Core.GetentPaths = new[] { missing }; + Assert.That(Core.OnlyUserHasPrimaryGroup(account.Name, me, account.Group, CancellationToken.None), Is.False, "no getent"); + Core.GetentPaths = new[] { missing, mine, "/usr/bin/getent" }; + Assert.That(Core.OnlyUserHasPrimaryGroup(account.Name, me, account.Group, CancellationToken.None), Is.False, "a getent this user owns"); + bool machine = MachineVerdict(account.Name, me, account.Group); + Core.GetentPaths = new[] { missing, "/usr/bin/getent" }; + Assert.That(Core.OnlyUserHasPrimaryGroup(account.Name, me, account.Group, CancellationToken.None), Is.EqualTo(machine), "the first present is used"); + + // The real getent's file or a directory above it, as another user's or writable by group or others. + string getent = Unix.RealPath("/usr/bin/getent"); + bool Altered(string path, Func change) + { + Core.GetentPaths = new[] { "/usr/bin/getent" }; + Core.GetentStatHook = (_, p) => p == path ? change(Unix.Stat(p, follow: false)) : Unix.Stat(p, follow: false); + try + { + return Core.OnlyUserHasPrimaryGroup(account.Name, me, account.Group, CancellationToken.None); + } + finally + { + Core.GetentStatHook = null; + } + } + + Assert.That(Altered(getent, s => s), Is.EqualTo(machine), "unaltered"); + Assert.That(Altered(getent, s => s with { Owner = me == 0 ? 1u : me }), Is.False, "getent owned by another user"); + Assert.That(Altered(getent, s => s with { Mode = s.Mode | 0x10 }), Is.False, "getent writable by its group"); + Assert.That(Altered(getent, s => s with { Mode = s.Mode | 0x2 }), Is.False, "getent writable by others"); + Assert.That(Altered(getent, s => s with { Kind = Unix.Kind.Other }), Is.False, "getent not a regular file"); + Assert.That(Altered(Path.GetDirectoryName(getent)!, s => s with { Mode = s.Mode | 0x10 }), Is.False, "its directory writable by its group"); + Assert.That(Altered("/", s => s with { Owner = me == 0 ? 1u : me }), Is.False, "the root directory owned by another user"); + + Assert.That(Core.RootOnly(new Unix.Status(Unix.Kind.Regular, 0, 0, 0x1ED)), Is.True, "root's 0755"); + Assert.That(Core.RootOnly(new Unix.Status(Unix.Kind.Regular, 0, me == 0 ? 1u : me, 0x1ED)), Is.False, "another owner"); + Assert.That(Core.RootOnly(new Unix.Status(Unix.Kind.Regular, 0, 0, 0x1FD)), Is.False, "group-writable"); + Assert.That(Core.RootOnly(new Unix.Status(Unix.Kind.Regular, 0, 0, 0x1EF)), Is.False, "writable by others"); + } + + /// + /// Where commands run through setsid, getent runs as + /// setsid -- getent passwd, so setsid must be root's and + /// not writable by group or others as well, as must every directory + /// above its real path: with a stand-in getent (/bin/sh, + /// root's) printing a valid listing, the listing is accepted, and refused + /// when setsid's file is another user's, writable by its group or + /// others, or not a regular file, or a directory above it is not root's + /// alone (statuses altered through a seam, for setsid's check only, so + /// the stand-in getent's own check passes). Without groups + /// setsid is not run, so its status does not matter. + /// + [Test] + [Platform("Linux")] + public void TheSetsidGetentRunsThroughMustBeRootsAndNotWritable() + { + Assume.That(CoreCommand.Setsid, Is.Not.Null, "setsid is installed"); + uint me = Unix.EffectiveUser(); + const uint Group = 4242; + Core.GetentPaths = new[] { "/bin/sh" }; + Core.GetentArguments = new[] { "-c", "echo me:x:" + me.ToString(CultureInfo.InvariantCulture) + ":4242::/:/bin/sh" }; + string setsid = Unix.RealPath(CoreCommand.Setsid!); + // Only setsid's statuses are altered: the stand-in getent's real path + // may share a directory with setsid's (/usr/bin/dash beside /usr/bin/setsid). + bool Altered(string path, Func change) + { + Core.GetentStatHook = (file, p) => file == setsid && p == path ? change(Unix.Stat(p, follow: false)) : Unix.Stat(p, follow: false); + try + { + return Core.OnlyUserHasPrimaryGroup("me", me, Group, CancellationToken.None); + } + finally + { + Core.GetentStatHook = null; + } + } + + Assert.That(Altered(setsid, s => s), Is.True, "unaltered"); + Assert.That(Altered(setsid, s => s with { Owner = me == 0 ? 1u : me }), Is.False, "setsid owned by another user"); + Assert.That(Altered(setsid, s => s with { Mode = s.Mode | 0x10 }), Is.False, "setsid writable by its group"); + Assert.That(Altered(setsid, s => s with { Mode = s.Mode | 0x2 }), Is.False, "setsid writable by others"); + Assert.That(Altered(setsid, s => s with { Kind = Unix.Kind.Other }), Is.False, "setsid not a regular file"); + Assert.That(Altered(Path.GetDirectoryName(setsid)!, s => s with { Mode = s.Mode | 0x10 }), Is.False, "its directory writable by its group"); + CoreCommand.WithoutGroups = true; + try + { + Assert.That(Altered(setsid, s => s with { Mode = s.Mode | 0x10 }), Is.True, "setsid not used"); + } + finally + { + CoreCommand.WithoutGroups = false; + } + } + + /// + /// What run with + /// prints (here getent passwd, or a stand-in printing its listing + /// and more), run by the test itself rather than the SDK's runner: its + /// lines. + /// + private static string[] Listing(string file, string[] arguments) + { + var info = new System.Diagnostics.ProcessStartInfo(file) + { + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + }; + foreach (string argument in arguments) + { + info.ArgumentList.Add(argument); + } + + using System.Diagnostics.Process process = System.Diagnostics.Process.Start(info) ?? throw new AssertionException("could not start " + file); + process.StandardInput.Close(); + Task stdout = process.StandardOutput.ReadToEndAsync(); + if (!process.WaitForExit(30_000)) + { + process.Kill(entireProcessTree: true); + process.WaitForExit(); + Assert.Fail(file + " did not exit within 30 seconds"); + } + + Assert.That(process.ExitCode, Is.EqualTo(0), file + " exit status"); + return stdout.GetAwaiter().GetResult().Split('\n', StringSplitOptions.RemoveEmptyEntries); + } + + /// + /// The private-group rule's password database half applied by the test + /// to a listing: listed with exactly these ids, + /// no name listed twice, no malformed line, and no account of another + /// user id with as its primary group. + /// + private static bool Oracle(string[] lines, string name, uint uid, uint gid) + { + var names = new HashSet(StringComparer.Ordinal); + bool listed = false; + foreach (string line in lines) + { + string[] fields = line.Split(':'); + if (fields.Length != 7 || !names.Add(fields[0]) + || !uint.TryParse(fields[2], NumberStyles.None, CultureInfo.InvariantCulture, out uint entryUid) + || !uint.TryParse(fields[3], NumberStyles.None, CultureInfo.InvariantCulture, out uint entryGid)) + { + return false; + } + + if (fields[0] == name) + { + if (entryUid != uid || entryGid != gid) + { + return false; + } + + listed = true; + } + else if (entryGid == gid && entryUid != uid) + { + return false; + } + } + + return listed; + } + + /// The verdict gives for this machine's getent passwd listing. + private static bool MachineVerdict(string name, uint uid, uint gid) => + Oracle(Listing(Unix.RealPath(new[] { "/usr/bin/getent", "/bin/getent" }.First(File.Exists)), new[] { "passwd" }), name, uid, gid); + + /// + /// The SDK's verdict on this machine's real getent passwd listing + /// is the one the test reaches by running getent passwd itself and + /// applying the rule, whichever it is (private, or shared as where every + /// account has the primary group users), and so is its verdict on + /// that listing with another account of this user's primary group added + /// (a stand-in printing both), which the test's rule finds shared. + /// Another enumeration of the password database in this process, here a + /// full setpwent/getpwent/endpwent pass run while + /// getent runs, cannot hide an entry, as it could while the SDK + /// shared libc's process-global position: an account this machine lists + /// whose primary group another user id shares is refused too. + /// + [Test] + [Platform("Linux")] + public void ThePasswordDatabaseListingOfThisMachineIsReadInAnotherProcess() + { + uint me = Unix.EffectiveUser(); + Unix.Account account = Unix.User(me) ?? throw new AssertionException("no passwd entry for this user"); + string getent = Unix.RealPath(new[] { "/usr/bin/getent", "/bin/getent" }.First(File.Exists)); + string second = "jacquard-test-second:x:" + (me + 1).ToString(CultureInfo.InvariantCulture) + ":" + account.Group.ToString(CultureInfo.InvariantCulture) + "::/:/bin/sh"; + var listings = new (string What, string File, string[] Arguments)[] + { + ("this machine's listing", getent, new[] { "passwd" }), + ("this machine's listing and another account with this user's primary group", "/bin/sh", new[] { "-c", "\"$0\" passwd && echo '" + second + "'", getent }), + }; + var expected = new bool[listings.Length]; + int consumed = 0; + CoreCommand.StartedHook = _ => consumed += SdkSupport.Libc.ReadWholePasswordDatabase(); + try + { + for (int i = 0; i < listings.Length; i++) + { + (string what, string file, string[] arguments) = listings[i]; + expected[i] = Oracle(Listing(file, arguments), account.Name, me, account.Group); + Core.GetentPaths = new[] { file }; + Core.GetentArguments = arguments; + consumed = 0; + Assert.That(Core.OnlyUserHasPrimaryGroup(account.Name, me, account.Group, CancellationToken.None), Is.EqualTo(expected[i]), what); + Assert.That(consumed, Is.GreaterThan(0), what + ": the in-process enumeration ran while getent ran"); + } + + Assert.That(expected[1], Is.False, "the test's rule finds the group shared once another account has it"); + + string[][] entries = Listing(getent, new[] { "passwd" }).Select(l => l.Split(':')).Where(f => f.Length == 7).ToArray(); + string[]? shared = entries.FirstOrDefault(f => entries.Any(o => o[3] == f[3] && o[2] != f[2])); + if (shared is not null) + { + Core.GetentPaths = new[] { getent }; + Core.GetentArguments = new[] { "passwd" }; + Assert.That( + Core.OnlyUserHasPrimaryGroup(shared[0], uint.Parse(shared[2], CultureInfo.InvariantCulture), uint.Parse(shared[3], CultureInfo.InvariantCulture), CancellationToken.None), + Is.False, + shared[0] + " shares its primary group " + shared[3]); + } + } + finally + { + CoreCommand.StartedHook = null; + } + } + + /// + /// The account records libc fills are read only within the buffer the + /// SDK gave it: a name, a member array, a member slot, or a member name + /// outside it, or a string without its NUL inside it, is a lookup error + /// (null), never read through. Here the pointers are crafted; the + /// outside memory is another valid array, so reading it would succeed. + /// + [Test] + public unsafe void TheAccountRecordsAreReadOnlyWithinTheirBuffer() + { + byte[] buffer = new byte[64]; + byte[] outside = Encoding.ASCII.GetBytes("outside\0\0\0\0\0\0\0\0\0"); + byte[] entry = new byte[256]; + fixed (byte* data = buffer) + fixed (byte* far = outside) + fixed (byte* record = entry) + { + // "me\0" at 0, "you\0" at 8, the member array { 8, null } at 16. + Encoding.ASCII.GetBytes("me\0").CopyTo(buffer, 0); + Encoding.ASCII.GetBytes("you\0").CopyTo(buffer, 8); + *(IntPtr*)(record + 0) = (IntPtr)(data); + *(IntPtr*)(record + 24) = (IntPtr)(data + 16); + *(IntPtr*)(data + 16) = (IntPtr)(data + 8); + *(IntPtr*)(data + 24) = IntPtr.Zero; + *(uint*)(record + 16) = 7; + *(uint*)(record + 20) = 9; + Assert.That(Unix.ReadAccount(record, data, buffer.Length), Is.EqualTo(new Unix.PasswdEntry("me", 7, 9))); + Unix.GroupEntry group = Unix.ReadGroup(record, data, buffer.Length)!; + Assert.That((group.Name, string.Join(",", group.Members)), Is.EqualTo(("me", "you"))); + + *(IntPtr*)(record + 0) = (IntPtr)(far); + Assert.That(Unix.ReadAccount(record, data, buffer.Length), Is.Null, "a name outside the buffer"); + Assert.That(Unix.ReadGroup(record, data, buffer.Length), Is.Null, "a group name outside the buffer"); + *(IntPtr*)(record + 0) = (IntPtr)(data); + *(IntPtr*)(data + 16) = (IntPtr)(far); + Assert.That(Unix.ReadGroup(record, data, buffer.Length), Is.Null, "a member name outside the buffer"); + *(IntPtr*)(data + 16) = (IntPtr)(data + 8); + *(IntPtr*)(record + 24) = (IntPtr)(far); + Assert.That(Unix.ReadGroup(record, data, buffer.Length), Is.Null, "a member array outside the buffer"); + + // A member array whose terminating slot would lie past the end. + *(IntPtr*)(record + 24) = (IntPtr)(data + buffer.Length - 8); + *(IntPtr*)(data + buffer.Length - 8) = (IntPtr)(data + 8); + Assert.That(Unix.ReadGroup(record, data, buffer.Length), Is.Null, "a member array running past the buffer"); + + // A name with no NUL before the end of the buffer. + new Span(data + 40, 16).Fill((byte)'x'); + *(IntPtr*)(record + 0) = (IntPtr)(data + 40); + *(IntPtr*)(record + 24) = IntPtr.Zero; + Assert.That(Unix.ReadAccount(record, data, 56), Is.Null, "a name without its NUL in the buffer"); + } + } + + /// + /// An account, group, or member name that is not valid UTF-8 is a lookup + /// error (null), as distinct malformed names would otherwise decode to + /// the same replacement text and compare equal. + /// + [Test] + public unsafe void AccountNamesThatAreNotUtf8AreLookupErrors() + { + byte[] buffer = new byte[64]; + byte[] entry = new byte[256]; + fixed (byte* data = buffer) + fixed (byte* record = entry) + { + // "\xFFm\0" at 0, "me\0" at 4, "\xC3\0" (a cut sequence) at 8, the member array { 8, null } at 16. + buffer[0] = 0xFF; + buffer[1] = (byte)'m'; + Encoding.ASCII.GetBytes("me\0").CopyTo(buffer, 4); + buffer[8] = 0xC3; + *(IntPtr*)(record + 0) = (IntPtr)data; + *(IntPtr*)(record + 24) = IntPtr.Zero; + Assert.That(Unix.ReadAccount(record, data, buffer.Length), Is.Null, "an account name that is not UTF-8"); + Assert.That(Unix.ReadGroup(record, data, buffer.Length), Is.Null, "a group name that is not UTF-8"); + *(IntPtr*)(record + 0) = (IntPtr)(data + 4); + Assert.That(Unix.ReadAccount(record, data, buffer.Length)?.Name, Is.EqualTo("me"), "a valid name"); + *(IntPtr*)(record + 24) = (IntPtr)(data + 16); + *(IntPtr*)(data + 16) = (IntPtr)(data + 8); + *(IntPtr*)(data + 24) = IntPtr.Zero; + Assert.That(Unix.ReadGroup(record, data, buffer.Length), Is.Null, "a member name that is not UTF-8"); + } + } + + /// + /// The scratch directories the SDK tests use are made where no group or + /// other user may write any directory above them, so a group-writable + /// TMPDIR (here 0775) does not add levels to the exact refusals the + /// tests expect: /tmp is used instead. + /// + [Test] + public void TheScratchDirectoriesAvoidAGroupWritableTemporaryDirectory() + { + string shared = Directory.CreateDirectory(At("tmpdir")).FullName; + File.SetUnixFileMode(shared, (UnixFileMode)0x1FD); + string? saved = Environment.GetEnvironmentVariable("TMPDIR"); + string probe; + try + { + Environment.SetEnvironmentVariable("TMPDIR", shared); + Assert.That(Path.GetTempPath(), Does.StartWith(shared)); + probe = SdkSupport.Scratch("probe"); + } + finally + { + Environment.SetEnvironmentVariable("TMPDIR", saved); + } + + try + { + Assert.That(probe, Does.Not.StartWith(shared)); + Assert.That(SdkSupport.SafeAncestry(probe), Is.True, probe); + Assert.That(File.GetUnixFileMode(probe), Is.EqualTo(UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute)); + } + finally + { + SdkSupport.RemoveScratch(probe); + } + } + + /// + /// A directory writable by the private group is still refused when it + /// has an access ACL: on Linux the group bits show the ACL's mask, and + /// an entry may grant another user write (here nobody, set with + /// setfacl). + /// + [Test] + public void APrivateGroupParentWithAnAccessListIsRefused() + { + if (!File.Exists("/usr/bin/setfacl")) + { + Assert.Ignore("setfacl is not installed"); + } + + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent")).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1C0); + using (var setfacl = System.Diagnostics.Process.Start("/usr/bin/setfacl", new[] { "-m", "u:nobody:rwx", parent })) + { + Assert.That(setfacl.WaitForExit(30_000), Is.True); + Assert.That(setfacl.ExitCode, Is.EqualTo(0), "setfacl"); + } + + Assert.That(File.GetUnixFileMode(parent), Is.EqualTo((UnixFileMode)0x1F8), "the mask shows in the group bits"); + Accounts(parent, "me"); + string cache = Path.Combine(parent, "cache"); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = cache }))!.Message, + Is.EqualTo(Unsafe(cache, parent + " (writable by its group, with an access control list)"))); + } + + /// + /// The real password and group databases answer for this user: its name + /// is the one .NET reports, and its primary group has an entry. + /// + [Test] + public void TheAccountLookupsReadTheRealDatabases() + { + Unix.Account account = Unix.User(Unix.EffectiveUser()) ?? throw new AssertionException("no passwd entry for this user"); + Assert.That(account.Name, Is.EqualTo(Environment.UserName)); + Unix.GroupEntry group = Unix.GroupOf(account.Group) ?? throw new AssertionException("no group entry for the primary group"); + Assert.That(group.Name, Is.Not.Empty); + Assert.That(group.Members.Count, Is.LessThan(Unix.AccountBytes)); + } + + /// + /// A refusal names every unsafe directory above the cache, nearest + /// first, with its reason, not only the first found; beyond 16 it counts + /// the rest. + /// + [Test] + public void ARefusalListsEveryUnsafeDirectoryAboveTheCache() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string a = Directory.CreateDirectory(At("a")).FullName; + string b = Directory.CreateDirectory(Path.Combine(a, "b")).FullName; + string c = Directory.CreateDirectory(Path.Combine(b, "c")).FullName; + File.SetUnixFileMode(a, (UnixFileMode)0x1FF); + File.SetUnixFileMode(b, (UnixFileMode)0x1F8); + File.SetUnixFileMode(c, (UnixFileMode)0x1C0); + NotPrivate(b); + string cache = Path.Combine(c, "cache"); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = cache }))!.Message, + Is.EqualTo(Unsafe(cache, b + " (" + NotPrivateGroup + ")", a + " (writable by others)"))); + + // Eighteen unsafe levels: sixteen are named and two counted. + string deep = Directory.CreateDirectory(At("deep"), UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute).FullName; + var levels = new List(); + for (int i = 0; i < 18; i++) + { + deep = Directory.CreateDirectory(Path.Combine(deep, "d")).FullName; + levels.Add(deep); + } + + foreach (string level in levels) + { + File.SetUnixFileMode(level, (UnixFileMode)0x1FF); + } + + string deepCache = Path.Combine(deep, "cache"); + levels.Reverse(); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = deepCache }))!.Message, + Is.EqualTo(Unsafe(deepCache, levels.Take(16).Select(level => level + " (writable by others)").ToArray()) + "; and 2 more")); + } + + /// + /// The directories that exist above a missing cache root are checked + /// before any missing level is made, so a refused preparation leaves + /// nothing behind. + /// + [Test] + public void ARefusedPreparationMakesNoDirectory() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent")).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1FF); + string cache = Path.Combine(parent, "x", "y", "cache"); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = cache }))!.Message, + Is.EqualTo(Unsafe(cache, parent + " (writable by others)"))); + Assert.That(Directory.EnumerateFileSystemEntries(parent), Is.Empty); + } + + /// + /// The directories a preparation makes for a missing cache root are each + /// owner-only (0700), whatever the umask, so a cache under new + /// directories is not refused for parents the SDK itself made writable. + /// + [Test] + public void TheDirectoriesMadeForACacheRootAreOwnerOnly() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string top = At("made"); + string cache = Path.Combine(top, "a", "b", "cache"); + fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = cache }); + foreach (string level in new[] { top, Path.Combine(top, "a"), Path.Combine(top, "a", "b"), cache }) + { + Assert.That(File.GetUnixFileMode(level), Is.EqualTo(UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute), level); + } + } + + /// + /// A prepared program's workers run on its store by path, so the cache + /// root's device and inode are checked again before each worker starts. + /// After preparation the root is renamed away and back (the same + /// directory: calls still start a worker), then renamed away and + /// replaced by a new owner-only directory, as another user who could + /// write a parent could: the next call is refused before any worker + /// starts, for a function bound before the swap and one bound after it. + /// + [Test] + public void ACacheRootReplacedAfterPreparationIsRefusedAtTheNextCall() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string cache = Path.GetFullPath(fake.Options.CacheDir!); + PreparedProgram program = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + Function before = program.Function("first"); + long Workers() => File.ReadAllLines(Path.Combine(fake.Root, "log")).Length; + + long started = Workers(); + Assert.That(before.Invoke().Ok, Is.False, "the fake worker answers nothing"); + Assert.That(Workers(), Is.EqualTo(started + 1), "a worker started"); + + Directory.Move(cache, cache + "-away"); + Directory.Move(cache + "-away", cache); + before.Invoke(); + Assert.That(Workers(), Is.EqualTo(started + 2), "the same root renamed back is used"); + + Directory.Move(cache, cache + "-away"); + Directory.CreateDirectory(cache, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + string refused = "cache root " + cache + " was replaced after it was checked; prepare again"; + Assert.That(Support.Throws(() => before.Invoke())!.Message, Is.EqualTo(refused)); + Assert.That(Support.Throws(() => program.Function("second").Invoke())!.Message, Is.EqualTo(refused)); + Assert.That(Workers(), Is.EqualTo(started + 2), "no worker started from the replacement"); + } + + /// + /// The executable Core runs must be the one fingerprinted: a test seam + /// replaces it (a new file renamed over it) after each kind of Core + /// command, and the preparation is refused instead of keying artifacts + /// made by other bytes under the old fingerprint. + /// + [Test] + public void AnExecutableReplacedDuringPreparationIsRefused() + { + using FakeCore fake = FakeCore.Programs(At("swap")); + string script = File.ReadAllText(fake.Binary); + foreach (string after in new[] { "--version", "check", "hash", "run" }) + { + bool replaced = false; + Runtime.CommandHook = command => + { + if (command == after && !replaced) + { + replaced = true; + Replace(fake.Binary, script + "# replaced after " + after + "\n"); + } + }; + string message = fake.PrepareFails("rules" + after.Trim('-') + ".jac").Message; + Assert.That(replaced, Is.True, after + " never ran"); + Assert.That(message, Is.EqualTo(ChangedMessage("Core executable", fake.Binary)), "after " + after); + Assert.That( + Directory.Exists(Path.Combine(fake.Root, "cache")) ? fake.CacheEntries() : new List(), + Is.Empty, + "an entry was published or a staging directory left after " + after); + } + + Runtime.CommandHook = null; + fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + } + + /// + /// The prelude Core reads must be the one fingerprinted: a prelude file + /// rewritten in place, or a file added to the prelude directory, while + /// the preparation runs is refused. + /// + [Test] + public void APreludeChangedDuringPreparationIsRefused() + { + using FakeCore fake = FakeCore.Programs(At("prelude-swap")); + string prelude = Path.Combine(fake.Root, "prelude"); + string file = Path.Combine(prelude, "01-prim.jqd"); + Runtime.CommandHook = command => + { + if (command == "check") + { + File.AppendAllText(file, "changed\n"); + } + }; + Assert.That(fake.PrepareFails("first.jac").Message, Is.EqualTo(ChangedMessage("prelude entry", file))); + Runtime.CommandHook = command => + { + if (command == "run" && !File.Exists(Path.Combine(prelude, "added.jqd"))) + { + File.WriteAllText(Path.Combine(prelude, "added.jqd"), "added\n"); + } + }; + Assert.That(fake.PrepareFails("second.jac").Message, Is.EqualTo(ChangedMessage("prelude", prelude))); + Runtime.CommandHook = null; + fake.Runtime.Prepare(new[] { fake.Source("third.jac") }, fake.Options); + } + + /// + /// The version check runs once per executable fingerprint, which + /// includes the file's identity: the same bytes with a new modification + /// time (as a replaced file restored would have) are checked again. + /// + [Test] + public void TheVersionCheckIsKeyedOnTheExecutablesIdentity() + { + using FakeCore fake = FakeCore.Programs(At("version-key")); + fake.Runtime.Prepare(new[] { fake.Source("one.jac") }, fake.Options); + fake.Runtime.Prepare(new[] { fake.Source("two.jac") }, fake.Options); + Assert.That(fake.Calls("--version"), Is.EqualTo(1)); + File.SetLastWriteTimeUtc(fake.Binary, File.GetLastWriteTimeUtc(fake.Binary).AddSeconds(-10)); + fake.Runtime.Prepare(new[] { fake.Source("three.jac") }, fake.Options); + Assert.That(fake.Calls("--version"), Is.EqualTo(2)); + } + + /// + /// The executable must be one this user may run (access(X_OK)), + /// not merely one with some execute bit: a file only its group and + /// others may execute is refused with a clear message before it runs. + /// + [Test] + public void AnExecutableTheUserMayNotRunIsRefused() + { + Assume.That(Unix.EffectiveUser(), Is.Not.EqualTo(0u), "running as root"); + using FakeCore fake = FakeCore.Programs(At("noexec")); + File.SetUnixFileMode(fake.Binary, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherExecute); + Assert.That( + fake.PrepareFails().Message, + Is.EqualTo("Core executable " + fake.Binary + " may not be executed by the current user; give it execute permission or configure JACQUARD_BIN")); + Assert.That(fake.Calls("--version"), Is.EqualTo(0)); + } + + /// A source list whose count, or else whose indexer, throws Error. + private sealed class ThrowingSources(bool count) : IReadOnlyList + { + internal static readonly InvalidOperationException Error = new("list failed"); + + public int Count => count ? throw Error : 1; + + public string this[int index] => throw Error; + + public IEnumerator GetEnumerator() => throw Error; + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// A bindings dictionary whose count, or else whose enumerator, throws . + private sealed class ThrowingBindings(bool count) : IReadOnlyDictionary + { + public int Count => count ? throw ThrowingSources.Error : 1; + + public IEnumerable Keys => throw new NotSupportedException(); + + public IEnumerable Values => throw new NotSupportedException(); + + public Operations this[string key] => throw new NotSupportedException(); + + public bool ContainsKey(string key) => throw new NotSupportedException(); + + public bool TryGetValue(string key, out Operations value) => throw new NotSupportedException(); + + public IEnumerator> GetEnumerator() => throw ThrowingSources.Error; + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// + /// An exception from the caller's source list (its count or an element) + /// or bindings dictionary (its count or enumerator) is wrapped in the + /// SDK's exception, keeping the cause, as other caller code is. + /// + [Test] + public void ACallersFailingSourceListOrBindingsIsWrappedWithTheCause() + { + using FakeCore fake = FakeCore.Effects(At("throwing")); + foreach (bool count in new[] { true, false }) + { + JacquardException sources = Support.Throws(() => fake.Runtime.Prepare(new ThrowingSources(count), fake.Options))!; + Assert.That(sources.Kind, Is.EqualTo(JacquardErrorKind.Preparation)); + Assert.That(sources.Message, Is.EqualTo("reading the source list failed: System.InvalidOperationException: list failed")); + Assert.That(sources.InnerException, Is.SameAs(ThrowingSources.Error)); + } + + PreparedProgram program = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + foreach (bool count in new[] { true, false }) + { + JacquardException bindings = Support.Throws(() => program.Function("first", new ThrowingBindings(count)))!; + Assert.That(bindings.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(bindings.Message, Is.EqualTo("first: reading the bindings failed: System.InvalidOperationException: list failed")); + Assert.That(bindings.InnerException, Is.SameAs(ThrowingSources.Error)); + } + } + + /// A bindings dictionary that records whether the program's lock was held while it was read. + private sealed class Watching : IReadOnlyDictionary + { + private readonly PreparedProgram program; + private readonly Operations service; + + internal Watching(PreparedProgram program, Operations service) + { + this.program = program; + this.service = service; + } + + internal bool? LockHeld { get; private set; } + + public int Count => 1; + + public IEnumerable Keys => throw new NotSupportedException(); + + public IEnumerable Values => throw new NotSupportedException(); + + public Operations this[string key] => throw new NotSupportedException(); + + public bool ContainsKey(string key) => throw new NotSupportedException(); + + public bool TryGetValue(string key, out Operations value) => throw new NotSupportedException(); + + public IEnumerator> GetEnumerator() + { + LockHeld = program.LockHeld; + yield return new("Inventory", service); + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// The caller's bindings dictionary is read before the program's lock is taken, so its code cannot hold the lock. + [Test] + public void TheCallersBindingsAreReadOutsideTheProgramsLock() + { + using FakeCore fake = FakeCore.Effects(At("bindings")); + PreparedProgram program = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + var bindings = new Watching(program, new Operations().On("lookup", Codecs.Int, () => 1L)); + Assert.That(program.Function("first", bindings).Name, Is.EqualTo("first")); + Assert.That(bindings.LockHeld, Is.False); + } + + /// + /// A function, effect, or operation name over 4,096 chars is refused + /// before it is looked up, quoting an excerpt; one of exactly 4,096 + /// chars is looked up (and names nothing). + /// + [Test] + public void OverlongProgramNamesAreRefusedBeforeTheyAreLookedUp() + { + using FakeCore fake = FakeCore.Effects(At("names")); + PreparedProgram program = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + string huge = new('X', 1_000_000); + Operations service = new Operations().On("lookup", Codecs.Int, () => 1L); + Assert.That(Support.Throws(() => program.Function(huge)).Message, Is.EqualTo("function name " + Excerpt('X', 1_000_000) + " exceeds 4096 chars")); + Assert.That(Support.Throws(() => program.Provide(huge, service)).Message, Is.EqualTo("effect name " + Excerpt('X', 1_000_000) + " exceeds 4096 chars")); + Assert.That( + Support.Throws(() => program.Function("first", new Dictionary { [huge] = service })).Message, + Is.EqualTo("effect name " + Excerpt('X', 1_000_000) + " exceeds 4096 chars")); + Assert.That(Support.Throws(() => new Operations().On(huge, Codecs.Int, () => 1L)).Message, Is.EqualTo("operation name " + Excerpt('X', 1_000_000) + " exceeds 4096 chars")); + string longest = new('x', 4096); + Assert.That(Support.Throws(() => program.Function(longest)).Message, Does.EndWith(": absent or unsupported closed v0 signature")); + Assert.That(Support.Throws(() => program.Provide(longest, service)).Message, Does.StartWith("unknown effect ")); + Assert.That(program.Function("first", new Dictionary { ["Inventory"] = service }).Name, Is.EqualTo("first")); + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/SdkLiveTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/SdkLiveTests.cs new file mode 100644 index 0000000..bd1444f --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/SdkLiveTests.cs @@ -0,0 +1,698 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; +using JacquardLang.Host.Tests.Readme; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// +/// Live SDK tests against the real Core named by JACQUARD_BIN and +/// JACQUARD_PRELUDE. Each is ignored without them, and fails instead +/// when JACQUARD_REQUIRE_LIVE is set, as CI does. +/// +[TestFixture] +[Platform("Linux,MacOsX")] +public sealed class SdkLiveTests +{ + private string root = string.Empty; + private Runtime runtime = null!; + + internal static void RequireLive() + { + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("JACQUARD_BIN")) || string.IsNullOrEmpty(Environment.GetEnvironmentVariable("JACQUARD_PRELUDE"))) + { + if (Environment.GetEnvironmentVariable("JACQUARD_REQUIRE_LIVE") is not null) + { + Assert.Fail("live tests need JACQUARD_BIN and JACQUARD_PRELUDE"); + } + + Assert.Ignore("live tests need JACQUARD_BIN and JACQUARD_PRELUDE"); + } + } + + [SetUp] + public void Live() + { + RequireLive(); + root = SdkSupport.Scratch("live"); + runtime = new Runtime(); + } + + [TearDown] + public void CleanUp() + { + Operations.AfterDecodeHook = null; + if (root.Length > 0) + { + SdkSupport.RemoveScratch(root); + } + } + + /// The dotnet/ directory this test assembly was built from. + internal static string DotnetRoot() => + typeof(SdkLiveTests).Assembly.GetCustomAttributes().Single(attribute => attribute.Key == "DotnetRoot").Value!; + + internal static string Example(string project) => Path.GetFullPath(Path.Combine(DotnetRoot(), "..", "examples", project, "rules.jac")); + + private PrepareOptions Options => new() { CacheDir = Path.Combine(root, "cache") }; + + private PreparedProgram Prepare(string source) => runtime.Prepare(new[] { source }, Options); + + private string Source(string name, string text) + { + string path = Path.Combine(root, name); + File.WriteAllText(path, text); + return path; + } + + private static Dynamic Policy(PreparedProgram program) => program.Data.Make("TriagePolicy", Dynamic.Of(4L), Dynamic.Of(60L)); + + [Test] + public void QuickstartCallsInvocationsAndSavedFrames() + { + Function shipping = Prepare(Example("quickstart")).Function("shipping-cost"); + Assert.That(shipping.Call(Codecs.Int, 35), Is.EqualTo(5L)); + Assert.That(shipping.Call(Codecs.Int, 75L), Is.EqualTo(0L)); + Invocation invocation = shipping.Invoke(35); + Assert.That(invocation.ToDynamic(), Is.EqualTo(Dynamic.Of(5L))); + Assert.That(invocation.Render(), Is.EqualTo("shipping-cost: ok")); + Assert.That(invocation.Error, Is.Null); + string path = Path.Combine(root, "inspection.json"); + invocation.Save(path); + byte[] raw = File.ReadAllBytes(path); + Assert.That(raw[^1], Is.EqualTo((byte)'\n')); + JsonObject saved = JsonCodec.DecodeObject(raw.AsSpan(0, raw.Length - 1), 128); + Assert.That(saved["schema"], Is.EqualTo(Json.Of("jacquard-host-inspection-v1"))); + Assert.That(saved["core_frames"].AsArray()!, Has.Count.EqualTo(invocation.Raw.CoreFrames.Count)); + Assert.That(File.GetUnixFileMode(path), Is.EqualTo(UnixFileMode.UserRead | UnixFileMode.UserWrite)); + } + + /// The README's SDK examples, compiled from Readme/, against Core. + [Test] + public void TheReadmeExamplesRun() + { + string saved = Path.Combine(root, "shipping.json"); + Assert.That(Shipping.Cost(Example("quickstart"), Path.Combine(root, "cache"), saved), Is.EqualTo(5L)); + Assert.That(File.Exists(saved), Is.True); + PreparedProgram documents = Prepare(Source( + "documents.jac", + "once effect Documents where {\n read-document : (Text) -> Text\n}\n\nsummarize : (Text) ->{Documents} Text\nsummarize(key) = read-document(key)\n")); + Function summarize = Documents.Summarize(documents); + Assert.That(summarize.Call(Codecs.Text, "example"), Is.EqualTo("A synthetic document supplied by .NET.")); + JacquardException refused = Support.Throws(() => summarize.Call(Codecs.Text, "other")); + Assert.That(refused.Kind, Is.EqualTo(JacquardErrorKind.Invocation)); + Assert.That(refused.Failure, Is.EqualTo(Failure.OperationFailed), refused.Message); + PreparedProgram triage = Prepare(Example("triage")); + IReadOnlyList routed = Incidents.Route( + triage.Function("triage.batch"), Policy(triage), new[] { new Incidents.Incident("inc-1", "checkout", 5, 3, true, null) }); + Assert.That(routed, Is.EqualTo(new[] { new Incidents.Routed("inc-1", "checkout-oncall", Incidents.Urgency.PageNow, new[] { "unowned", "customer-impact" }) }).Using(new RoutedComparer())); + } + + /// Compares results by value, with their reason lists by items. + private sealed class RoutedComparer : IEqualityComparer + { + public bool Equals(Incidents.Triage? x, Incidents.Triage? y) => + (x, y) switch + { + (Incidents.Routed a, Incidents.Routed b) => a.Id == b.Id && a.Team == b.Team && a.Urgency == b.Urgency && a.Reasons.SequenceEqual(b.Reasons), + _ => Equals(x, y), + }; + + public int GetHashCode(Incidents.Triage obj) => 0; + } + + private static Incidents.Incident Incident(long i) => + new(i.ToString(CultureInfo.InvariantCulture), "api", (i % 5) + 1, i * 3, i % 2 == 0, i % 3 != 0 ? "ops" : null); + + [Test] + public void RecordsEnumsOptionsAndExplicitBatches() + { + PreparedProgram program = Prepare(Example("triage")); + Function route = program.Function("triage.batch"); + Argument[] policy = { Policy(program) }; + List rows = Enumerable.Range(0, 64).Select(i => Incident(i)).ToList(); + IReadOnlyList results = route.MapBatches(Incidents.TriageCodec, Incidents.IncidentCodec, rows, 8, policy); + Assert.That(results, Has.Count.EqualTo(rows.Count)); + + // An item codec that throws on the second batch: the failure names that batch, keeps the first, and keeps the codec's exception. + int decoded = 0; + Codec failing = Codecs.Create( + (encoder, value) => Incidents.TriageCodec.Encode(encoder, value), + (decoder, value) => ++decoded > 8 ? throw new InvalidOperationException("second batch") : Incidents.TriageCodec.Decode(decoder, value)); + JacquardException batch = Support.Throws(() => route.MapBatches(failing, Incidents.IncidentCodec, rows.GetRange(0, 16), 8, policy)); + Assert.That(batch.BatchIndex, Is.EqualTo(1)); + Assert.That(batch.Completed, Has.Count.EqualTo(8)); + Assert.That(batch.InnerException, Is.InstanceOf(), "the codec's exception is not kept"); + for (int i = 0; i < rows.Count; i++) + { + Incidents.Incident row = rows[i]; + bool page = row.Visible && row.Severity >= 4; + var reasons = new List(); + if (row.Owner is null) + { + reasons.Add("unowned"); + } + + if (row.Age >= 60) + { + reasons.Add("ageing"); + } + + if (page) + { + reasons.Add("customer-impact"); + } + + Incidents.Urgency urgency = page ? Incidents.Urgency.PageNow : row.Age >= 60 || row.Severity == 5 ? Incidents.Urgency.WorkToday : Incidents.Urgency.WatchOnly; + Assert.That(results[i], Is.EqualTo(new Incidents.Routed(row.Id, row.Owner ?? "api-oncall", urgency, reasons)).Using(new RoutedComparer()), "row " + row.Id); + } + + Assert.That(route.MapBatches(Incidents.TriageCodec, Incidents.IncidentCodec, Array.Empty(), 8, policy), Is.Empty); + } + + [Test] + public void InvalidInputIsRefusedBeforeAnyWorker() + { + PreparedProgram program = Prepare(Example("triage")); + + // A spawn would now fail as worker_spawn_failed, not as a local refusal. + program.SetWorkerCommand(new[] { "/no/such/jacquard" }); + Function route = program.Function("triage.batch"); + Argument[] policy = { Policy(program) }; + List rows = Enumerable.Range(0, 17).Select(i => Incident(i)).ToList(); + rows[16] = rows[16] with { Severity = Json.IntMax + 1 }; + JacquardException batch = Support.Throws(() => route.MapBatches(Incidents.TriageCodec, Incidents.IncidentCodec, rows, 8, policy)); + Assert.That(batch.BatchIndex, Is.EqualTo(-1)); + Assert.That(batch.Completed, Is.Empty); + Assert.That(batch.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(batch.Message, Does.Contain("argument[1][0].severity")); + Incidents.Incident[] many = Enumerable.Repeat(Incident(1), 64).ToArray(); + Codec> results = Codecs.List(Incidents.TriageCodec); + Assert.That( + Support.Throws(() => route.Call(results, policy[0], Codecs.List(Incidents.IncidentCodec).Arg(many))).Kind, + Is.EqualTo(JacquardErrorKind.InvalidArgument), + "over-deep single call"); + JacquardException notList = Support.Throws(() => route.Call(results, policy[0], "not a list")); + Assert.That(notList.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(notList.Message, Does.Contain("argument[1]")); + Assert.That(Support.Throws(() => route.Call(results, policy[0])).Message, Does.Contain("expected 2 arguments")); + } + + [Test] + public void MissingHandlersRefuseBindingAndPureCallsNeedNoServices() + { + PreparedProgram fulfillment = Prepare(Example("fulfillment")); + int calls = 0; + Operations inventory = new Operations().On("booking-status", Codecs.Dynamic, Codecs.Dynamic, order => + { + calls++; + return Dynamic.Unit; + }); + JacquardException missing = Support.Throws(() => fulfillment.Function("fulfillment.place", new Dictionary { ["Inventory"] = inventory })); + Assert.That(missing.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(missing.Message, Does.Contain("inventory.lookup-stock").And.Contain("dispatch.publish-booking")); + Assert.That(calls, Is.EqualTo(0), "a handler ran during binding"); + Operations typo = new Operations().On("lookup_stok", Codecs.Text, Codecs.Unit, sku => Unit.Value); + Assert.That( + Support.Throws(() => fulfillment.Function("fulfillment.place", new Dictionary { ["Inventory"] = typo })).Message, + Does.Contain("inventory has no operation lookup-stok")); + + // A service binding more names than the program has operations is refused from its count. + var wide = new Operations(); + for (int i = 0; i < 100; i++) + { + wide.On("not-an-operation-" + i.ToString(CultureInfo.InvariantCulture), Codecs.Text, Codecs.Unit, sku => Unit.Value); + } + + Assert.That( + Support.Throws(() => fulfillment.Function("fulfillment.place", new Dictionary { ["Inventory"] = wide })).Message, + Does.Contain("the service for inventory binds more operations than the program's ")); + fulfillment.Provide("Inventory", inventory); + Function valid = fulfillment.Function("order.valid-lines"); + Assert.That(valid.Call(Codecs.Bool, Codecs.List(Codecs.Dynamic).Arg(Array.Empty()), Codecs.List(Codecs.Text).Arg(Array.Empty())), Is.True); + Assert.That(Support.Throws(() => fulfillment.Provide("Unknown", new Operations())).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(Support.Throws(() => fulfillment.Function("no-such-function")).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(fulfillment.Functions, Does.Contain("fulfillment.place").And.Not.Contain("app.any")); + } + + private static Operations Warehouse(Action reserved) => new Operations() + .On("booking-status", Codecs.Dynamic, Codecs.Dynamic, order => Dynamic.VariantOf("BookingPending")) + .On("lookup_stock", Codecs.Text, Codecs.Dynamic, sku => Dynamic.VariantOf("Stock", Dynamic.Of(20L))) + .On("reserve-order", Codecs.Dynamic, Codecs.Dynamic, (context, order) => + { + Assert.That(context.IsCancelled, Is.False); + reserved(); + return Dynamic.VariantOf("Booked", Dynamic.Of("receipt-1")); + }) + .On("publish-booking", Codecs.Text, Codecs.Text, Codecs.Unit, (string id, string receipt) => throw new IOException("synthetic after commit")); + + private static Dynamic Order(ProgramData data) => + data.Make("Order", Dynamic.Of("one"), Dynamic.ListOf(data.Make("Line", Dynamic.Of("sensor"), Dynamic.Of(2L)))); + + private static Dictionary Both(Operations service) => new() { ["Inventory"] = service, ["Dispatch"] = service }; + + [Test] + public void UnknownCompletionHappensOnceWithRawEvidence() + { + PreparedProgram fulfillment = Prepare(Example("fulfillment")); + int reservations = 0; + Function place = fulfillment.Function("fulfillment.place", Both(Warehouse(() => Interlocked.Increment(ref reservations)))); + Dynamic order = Order(fulfillment.Data); + JacquardException error = Support.Throws(() => place.Call(Codecs.Dynamic, order)); + Assert.That(error.Failure, Is.EqualTo(Failure.OperationFailed), error.Message); + InvocationResult result = error.Result!; + Assert.That(result.Code, Is.EqualTo("E1612")); + Assert.That(result.HostFrames.Any(frame => Json.Of("effect_ok").Equals(frame.Get("kind"))), Is.True); + Assert.That(reservations, Is.EqualTo(1)); + string rendered = place.Invoke(order).Render(); + Assert.That(rendered, Does.StartWith("fulfillment.place: E1612").And.Contain(". publish-booking")); + } + + /// + /// In a real invocation, a cancellation that lands after a request's + /// arguments are decoded (through the test seam) keeps the handler from + /// starting, and the invocation reports the abort. + /// + [Test] + public void NoHandlerStartsOnceTheInvocationStoppedDuringDecoding() + { + PreparedProgram fulfillment = Prepare(Example("fulfillment")); + int started = 0; + Operations service = new Operations() + .On("booking-status", Codecs.Dynamic, Codecs.Dynamic, order => + { + started++; + return Dynamic.VariantOf("BookingPending"); + }) + .On("lookup-stock", Codecs.Text, Codecs.Dynamic, sku => Dynamic.VariantOf("Stock", Dynamic.Of(20L))) + .On("reserve-order", Codecs.Dynamic, Codecs.Dynamic, order => Dynamic.VariantOf("Booked", Dynamic.Of("r"))) + .On("publish-booking", Codecs.Text, Codecs.Text, Codecs.Unit, (id, receipt) => Unit.Value); + Function place = fulfillment.Function("fulfillment.place", Both(service)); + using var cancel = new CancellationTokenSource(); + Operations.AfterDecodeHook = cancel.Cancel; + Invocation invocation = place.Invoke(new Argument[] { Order(fulfillment.Data) }, cancel.Token); + Operations.AfterDecodeHook = null; + Assert.That(started, Is.EqualTo(0), "the handler started after the invocation stopped"); + Assert.That(Support.Reason(invocation.Raw), Is.EqualTo("aborted"), invocation.ToString()); + Assert.That(invocation.Raw.Requests, Has.Count.EqualTo(1)); + } + + /// A handler's context token is cancelled when the caller cancels while it runs; the handler sees it. + [Test] + public void AHandlersTokenIsCancelledWhenTheCallerCancels() + { + PreparedProgram fulfillment = Prepare(Example("fulfillment")); + using var cancel = new CancellationTokenSource(); + using var entered = new ManualResetEventSlim(); + bool sawCancellation = false; + Operations service = new Operations() + .On("booking-status", Codecs.Dynamic, Codecs.Dynamic, (context, order) => + { + entered.Set(); + sawCancellation = context.CancellationToken.WaitHandle.WaitOne(TimeSpan.FromMinutes(1)); + return Dynamic.VariantOf("BookingPending"); + }) + .On("lookup-stock", Codecs.Text, Codecs.Dynamic, sku => Dynamic.VariantOf("Stock", Dynamic.Of(20L))) + .On("reserve-order", Codecs.Dynamic, Codecs.Dynamic, order => Dynamic.VariantOf("Booked", Dynamic.Of("r"))) + .On("publish-booking", Codecs.Text, Codecs.Text, Codecs.Unit, (id, receipt) => Unit.Value); + Function place = fulfillment.Function("fulfillment.place", Both(service)); + Thread canceller = Support.After(TimeSpan.Zero, () => + { + entered.Wait(TimeSpan.FromMinutes(1)); + cancel.Cancel(); + }); + Invocation invocation = place.Invoke(new Argument[] { Order(fulfillment.Data) }, cancel.Token); + canceller.Join(); + Assert.That(Support.Reason(invocation.Raw), Is.EqualTo("aborted")); + Assert.That(Support.Eventually(() => sawCancellation, TimeSpan.FromSeconds(10)), Is.True, "the handler's token was not cancelled"); + } + + internal sealed record Job(string Id, long Effort, long Value, long Latest); + + internal sealed record Assignment(string Id, long Slot); + + internal sealed record Plan(long Score, IReadOnlyList Assignments); + + internal abstract record Planning; + + internal sealed record Planned(Plan Plan) : Planning; + + internal sealed record InvalidPlan(string Reason) : Planning; + + private static readonly RecordCodec JobCodec = Codecs.Record( + "Job", + Codecs.Field("id", Codecs.Text, (Job job) => job.Id), + Codecs.Field("effort", Codecs.Int, (Job job) => job.Effort), + Codecs.Field("value", Codecs.Int, (Job job) => job.Value), + Codecs.Field("latest", Codecs.Int, (Job job) => job.Latest), + (id, effort, value, latest) => new Job(id, effort, value, latest)); + + private static readonly UnionCodec PlanningCodec = Codecs.Union( + Codecs.Record( + "Planned", + Codecs.Field( + "plan", + Codecs.Record( + "Plan", + Codecs.Field("score", Codecs.Int, (Plan plan) => plan.Score), + Codecs.Field( + "assignments", + Codecs.List(Codecs.Record("Assignment", Codecs.Field("id", Codecs.Text, (Assignment a) => a.Id), Codecs.Field("slot", Codecs.Int, (Assignment a) => a.Slot), (id, slot) => new Assignment(id, slot))), + (Plan plan) => plan.Assignments), + (score, assignments) => new Plan(score, assignments)), + (Planned planned) => planned.Plan), + plan => new Planned(plan)), + Codecs.Record("InvalidPlan", Codecs.Field("reason", Codecs.Text, (InvalidPlan invalid) => invalid.Reason), reason => new InvalidPlan(reason))); + + private static long Optimum(IReadOnlyList jobs, long[] caps) + { + if (jobs.Count == 0) + { + return 0; + } + + Job job = jobs[0]; + List rest = jobs.Skip(1).ToList(); + long best = Optimum(rest, caps); + for (int slot = 0; slot < Math.Min(job.Latest, caps.Length); slot++) + { + if (caps[slot] >= job.Effort) + { + long[] next = (long[])caps.Clone(); + next[slot] -= job.Effort; + best = Math.Max(best, job.Value + Optimum(rest, next)); + } + } + + return best; + } + + [Test] + public void PlannerMatchesAnIndependentExhaustiveOracle() + { + Function solve = Prepare(Example("planner")).Function("planner.plan"); + for (long seed = 0; seed < 25; seed++) + { + var jobs = new List(); + for (long i = 0; i < (seed % 6) + 1; i++) + { + jobs.Add(new Job("job-" + i.ToString(CultureInfo.InvariantCulture), ((seed + (i * 3)) % 4) + 1, ((seed * 7) + (i * 11)) % 15, ((seed + i) % 3) + 1)); + } + + long[] all = { (seed % 5) + 1, (seed % 3) + 2, 3 }; + long[] caps = all.Take((int)(seed % 3) + 1).ToArray(); + if (solve.Call(PlanningCodec, Codecs.List(JobCodec).Arg(jobs), Codecs.List(Codecs.Int).Arg(caps)) is not Planned(Plan plan)) + { + Assert.Fail("seed " + seed.ToString(CultureInfo.InvariantCulture) + ": not planned"); + return; + } + + Assert.That(plan.Score, Is.EqualTo(Optimum(jobs, caps)), "seed " + seed.ToString(CultureInfo.InvariantCulture)); + long[] loads = new long[caps.Length]; + var seen = new HashSet(); + long score = 0; + foreach (Assignment assignment in plan.Assignments) + { + Job job = jobs.Single(candidate => candidate.Id == assignment.Id); + Assert.That(seen.Add(assignment.Id) && assignment.Slot >= 1 && assignment.Slot <= Math.Min(job.Latest, caps.Length), Is.True, assignment.ToString()); + loads[assignment.Slot - 1] += job.Effort; + score += job.Value; + } + + for (int slot = 0; slot < caps.Length; slot++) + { + Assert.That(loads[slot], Is.LessThanOrEqualTo(caps[slot]), "seed " + seed.ToString(CultureInfo.InvariantCulture) + ": overloaded"); + } + + Assert.That(score, Is.EqualTo(plan.Score)); + } + } + + [Test] + public void CacheReuseInvalidationAndCorruption() + { + PreparedProgram quick = Prepare(Example("quickstart")); + int commands = runtime.CoreCommands; + PreparedProgram repeated = Prepare(Example("quickstart")); + Assert.That(runtime.CoreCommands, Is.EqualTo(commands), "a warm preparation ran Core"); + Assert.That(repeated.Directory, Is.EqualTo(quick.Directory)); + string original = File.ReadAllText(Example("quickstart")); + string changed = Source("changed.jac", original.Replace("\n", "\r\n", StringComparison.Ordinal) + "\r\n-- changed\r\n"); + PreparedProgram program = Prepare(changed); + string snapshot = File.ReadAllText(Path.Combine(program.Directory, "source.jac")); + Assert.That(program.Directory != quick.Directory && snapshot.Contains("\r\n", StringComparison.Ordinal), Is.True, "not a byte-exact new entry"); + File.WriteAllText(Path.Combine(program.Directory, "store", "unexpected"), "corrupt"); + Assert.That(Support.Throws(() => Prepare(changed), "a corrupt cache was accepted").Kind, Is.EqualTo(JacquardErrorKind.Preparation)); + Assert.That(Support.Throws(() => Prepare(Source("rules.txt", string.Empty))).Kind, Is.EqualTo(JacquardErrorKind.Preparation)); + } + + /// + /// Core accepts a parameter nesting 2,500 Options. That function is + /// unavailable, refused with a binding failure that names the depth + /// limit, while safe works on both the cold and warm paths. + /// + [Test] + public void ASignatureTooDeepToBindLeavesTheOtherFunctionsWorking() + { + const int depth = 2500; + string source = Source("deep.jac", "deep : (" + string.Concat(Enumerable.Repeat("Option (", depth)) + "Int" + new string(')', depth) + ") ->{} Int\ndeep(x) = 0\nsafe : () ->{} Int\nsafe() = 1\n"); + int afterCold = -1; + foreach (string path in new[] { "cold", "warm" }) + { + PreparedProgram program = Prepare(source); + if (afterCold < 0) + { + afterCold = runtime.CoreCommands; + } + else + { + Assert.That(runtime.CoreCommands, Is.EqualTo(afterCold), "the warm path ran Core"); + } + + Assert.That(program.Function("safe").Call(Codecs.Int), Is.EqualTo(1L), path); + JacquardException refused = Support.Throws(() => program.Function("deep"), path); + Assert.That(refused.Kind, Is.EqualTo(JacquardErrorKind.Binding), path); + Assert.That(refused.Message, Does.Contain("deeper than 64 levels, the protocol's value-depth limit")); + } + } + + /// Pointing first at second's identity in program.json would make it return 2. + [Test] + public void ARedirectedNameInProgramJsonIsRefused() + { + string source = Source("pair.jac", "first : () ->{} Int\nfirst() = 1\n\nsecond : () ->{} Int\nsecond() = 2\n"); + PreparedProgram program = Prepare(source); + Assert.That(program.Function("first").Call(Codecs.Int), Is.EqualTo(1L)); + string metadata = Path.Combine(program.Directory, "program.json"); + string original = File.ReadAllText(metadata); + string from = "[\"first\",\"" + program.Data.Identity("first").Hex + "\"]"; + Assert.That(original, Does.Contain(from)); + File.WriteAllText(metadata, original.Replace(from, "[\"first\",\"" + program.Data.Identity("second").Hex + "\"]", StringComparison.Ordinal)); + JacquardException error = Support.Throws(() => Prepare(source).Function("first").Call(Codecs.Int)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Preparation), error.Message); + Assert.That(error.Message, Does.Contain("program.json bindings differ")); + File.WriteAllText(metadata, original); + Assert.That(Prepare(source).Function("first").Call(Codecs.Int), Is.EqualTo(1L)); + } + + [Test] + public void FullIntRangeRealsAndNestedOptions() + { + PreparedProgram program = Prepare(Source( + "values.jac", + "value.int : (Int) ->{} Int\nvalue.int(x) = x\nvalue.real : (Real) ->{} Real\nvalue.real(x) = x\n" + + "value.option : (Option (Option Int)) ->{} Option (Option Int)\nvalue.option(x) = x\n")); + Function integer = program.Function("value.int"); + foreach (long value in new[] { Json.IntMin, Json.IntMax }) + { + Assert.That(integer.Call(Codecs.Int, value), Is.EqualTo(value)); + } + + Assert.That(Support.Throws(() => integer.Call(Codecs.Int, Json.IntMax + 1)).Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(Support.Throws(() => integer.Call(Codecs.Int, 1.5)).Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + JacquardException narrow = Support.Throws(() => integer.Call(Codecs.Int32, 1L << 40)); + Assert.That(narrow.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(narrow.Result, Is.Not.Null); + Function real = program.Function("value.real"); + foreach (double value in new[] { 1.0, -0.0, double.PositiveInfinity, double.NaN }) + { + Assert.That(BitConverter.DoubleToInt64Bits(real.Call(Codecs.Real, value)), Is.EqualTo(BitConverter.DoubleToInt64Bits(value))); + } + + Function option = program.Function("value.option"); + Codec>> nested = Codecs.Option(Codecs.Option(Codecs.Int)); + foreach (Option> value in new[] { Option>.None, Option.Some(Option.None), Option.Some(Option.Some(3L)) }) + { + Assert.That(option.Call(nested, nested.Arg(value)), Is.EqualTo(value)); + } + + Dynamic inner = Dynamic.Some(Dynamic.None()); + Assert.That(option.Call(Codecs.Dynamic, inner), Is.EqualTo(inner)); + } + + [Test] + public void SpawnFailuresAndCancellation() + { + PreparedProgram quick = Prepare(Example("quickstart")); + Function shipping = quick.Function("shipping-cost"); + quick.SetWorkerCommand(new[] { "/no/such/jacquard" }); + JacquardException spawn = Support.Throws(() => quick.Function("shipping-cost").Call(Codecs.Int, 10)); + Assert.That(spawn.Failure, Is.EqualTo(Failure.WorkerLost), spawn.Message); + Assert.That(Support.Reason(spawn.Result!), Is.EqualTo("worker_spawn_failed")); + using var cancel = new CancellationTokenSource(); + cancel.Cancel(); + Invocation invocation = shipping.Invoke(new Argument[] { 10 }, cancel.Token); + Assert.That(Support.Reason(invocation.Raw), Is.EqualTo("aborted")); + Assert.That(invocation.Error, Is.Not.Null); + Assert.That(Support.Throws(() => shipping.Call(Codecs.Int, new Argument[] { 10 }, cancel.Token)).Failure, Is.EqualTo(Failure.WorkerLost)); + } + + [Test] + public void OutputOverflowFailsOnceWithoutAdaptiveBatching() + { + PreparedProgram program = Prepare(Example("triage")); + Function route = program.Function("triage.batch"); + List rows = Enumerable.Range(0, 29).Select(i => new Incidents.Incident(i.ToString(CultureInfo.InvariantCulture), "api", 5, 90, true, null)).ToList(); + JacquardException error = Support.Throws(() => route.MapBatches(Incidents.TriageCodec, Incidents.IncidentCodec, rows, 29, new Argument[] { Policy(program) })); + Assert.That(error.BatchIndex, Is.EqualTo(0), error.Message); + Assert.That(error.Completed, Is.Empty); + Assert.That(error.Result!.Code, Is.EqualTo("E1602")); + } + + /// Batch 0 decodes to one unit; batch 1's result exceeds the limits (E1602), and the failure keeps batch 0's result. + [Test] + public void ALaterBatchFailureKeepsEarlierUnitResults() + { + string many = string.Join(",", Enumerable.Repeat("()", 40)); + Function units = Prepare(Source( + "units.jac", + "units : (List Int) ->{} List ()\nunits(xs) = match xs { | Cons(n, _) -> if eq(n, 1) then [()] else [" + many + "] | Nil -> [] }\n")).Function("units"); + Assert.That(units.MapBatches(Codecs.Unit, Codecs.Int, new[] { 1L }, 1), Is.EqualTo(new[] { Unit.Value })); + JacquardException error = Support.Throws(() => units.MapBatches(Codecs.Unit, Codecs.Int, new[] { 1L, 2L }, 1)); + Assert.That(error.BatchIndex, Is.EqualTo(1), error.Message); + Assert.That(error.Completed, Is.EqualTo(new object[] { Unit.Value })); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Invocation)); + Assert.That(error.Result!.Code, Is.EqualTo("E1602")); + } + + /// A dictionary that reports a huge count and fails when it is read. + private sealed class HugeBindings : IReadOnlyDictionary + { + public int Count => 1_000_000; + + public IEnumerable Keys => throw new InvalidOperationException("a binding was read"); + + public IEnumerable Values => throw new InvalidOperationException("a binding was read"); + + public Operations this[string key] => throw new InvalidOperationException("a binding was read"); + + public bool ContainsKey(string key) => throw new InvalidOperationException("a binding was read"); + + public bool TryGetValue(string key, out Operations value) => throw new InvalidOperationException("a binding was read"); + + public IEnumerator> GetEnumerator() => throw new InvalidOperationException("a binding was read"); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// More bindings than the function has effects are refused from the count, before any binding is read. + [Test] + public void TooManyBindingsAreRefusedBeforeAnyIsRead() + { + PreparedProgram program = Prepare(Example("quickstart")); + JacquardException error = Support.Throws(() => program.Function("shipping-cost", new HugeBindings())); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(error.Message, Is.EqualTo("shipping-cost: 1000000 bindings given, but it uses 0 effects")); + } + + /// A result codec that throws with a huge message gives a short binding failure that keeps the raw result and the exception. + [Test] + public void AThrowingResultCodecKeepsTheResultAndQuotesAnExcerpt() + { + Function shipping = Prepare(Example("quickstart")).Function("shipping-cost"); + Invocation invocation = shipping.Invoke(35); + Assert.That(invocation.Ok, Is.True); + Codec throwing = Codecs.Create((encoder, value) => Codecs.Int.Encode(encoder, value), (decoder, value) => throw new InvalidOperationException(new string('X', 1_000_000))); + JacquardException error = Support.Throws(() => invocation.Decode(throwing)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(error.Message, Has.Length.LessThan(400)); + Assert.That(error.Message, Does.EndWith("... (1000000 chars)")); + Assert.That(error.Result!.Ok, Is.True, "the raw result was not kept"); + Assert.That(error.InnerException, Is.InstanceOf(), "the codec's exception is not kept"); + Assert.That(Support.Throws(() => shipping.Call(throwing, 35)).InnerException, Is.InstanceOf()); + + // A mapping failure with a null message is still a binding failure keeping the result. + Codec nullMessage = Codecs.Create((encoder, value) => Codecs.Int.Encode(encoder, value), (decoder, value) => throw decoder.Mapping(null)); + JacquardException nulled = Support.Throws(() => invocation.Decode(nullMessage)); + Assert.That(nulled.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(nulled.Result!.Ok, Is.True); + + // A save path longer than any path is refused before the file system sees it. + IOException save = Support.Throws(() => invocation.Save("/" + new string('v', 1_000_000))); + Assert.That(save.Message, Has.Length.LessThan(200)); + } + + [Test] + public void ResultMappingFailureRetainsTheSuccessfulResult() + { + Invocation invocation = Prepare(Example("quickstart")).Function("shipping-cost").Invoke(35); + Assert.That(invocation.Ok, Is.True); + Assert.That(invocation.Error, Is.Null); + JacquardException error = Support.Throws(() => invocation.Decode(Codecs.Text)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(error.Result!.Ok, Is.True, error.Message); + } + + private static void CopyTree(string from, string to) + { + Directory.CreateDirectory(to); + foreach (string entry in Directory.EnumerateFileSystemEntries(from)) + { + string target = Path.Combine(to, Path.GetFileName(entry)); + if (Directory.Exists(entry)) + { + CopyTree(entry, target); + } + else + { + File.Copy(entry, target); + } + } + } + + [Test] + public void PreludeContentsKeyTheCacheAndConcurrentPreparationPublishesOnce() + { + PreparedProgram quick = Prepare(Example("quickstart")); + string copy = Path.Combine(root, "prelude"); + CopyTree(Environment.GetEnvironmentVariable("JACQUARD_PRELUDE")!, copy); + var copied = new Runtime(new RuntimeOptions { Prelude = copy }); + PreparedProgram before = copied.Prepare(new[] { Example("quickstart") }, Options); + Assert.That(before.Directory, Is.EqualTo(quick.Directory), "an identical prelude copy should share the entry"); + string license = Path.Combine(copy, "LICENSE.md"); + File.AppendAllText(license, "\n"); + PreparedProgram after = copied.Prepare(new[] { Example("quickstart") }, Options); + Assert.That(after.Directory, Is.Not.EqualTo(before.Directory), "a prelude change did not change the cache key"); + string source = Source("concurrent.jac", "shipping-cost : (Int) ->{} Int\nshipping-cost(total) = if lt(total, 50) then 5 else 0\n-- concurrent\n"); + Task[] workers = Enumerable.Range(0, 2) + .Select(_ => Task.Factory.StartNew(() => runtime.Prepare(new[] { source }, Options), CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)) + .ToArray(); + Assert.That(Task.WaitAll(workers, TimeSpan.FromMinutes(2)), Is.True); + Assert.That(workers[1].Result.Directory, Is.EqualTo(workers[0].Result.Directory)); + Assert.That(workers[1].Result.Function("shipping-cost").Call(Codecs.Int, 35), Is.EqualTo(5L)); + foreach (string entry in Directory.EnumerateFileSystemEntries(Path.Combine(root, "cache"))) + { + Assert.That(Path.GetFileName(entry), Does.Not.StartWith("prepare-"), "staging directory left behind: " + entry); + } + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/SdkSupport.cs b/dotnet/test/JacquardHost.Tests/Sdk/SdkSupport.cs new file mode 100644 index 0000000..5a61362 --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/SdkSupport.cs @@ -0,0 +1,208 @@ +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// Helpers for the SDK tests. +internal static partial class SdkSupport +{ + /// Waits up to a minute for to hold a complete pid line and returns it. + internal static int PidFile(string path) + { + DateTime deadline = DateTime.UtcNow.AddMinutes(1); + while (true) + { + string text = File.Exists(path) ? File.ReadAllText(path) : string.Empty; + if (text.EndsWith('\n') && text.Trim().Length > 0) + { + return int.Parse(text.Trim().Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)[0], CultureInfo.InvariantCulture); + } + + if (DateTime.UtcNow > deadline) + { + Assert.Fail(path + " was never written"); + } + + Thread.Sleep(5); + } + } + + /// Waits up to ten seconds for to exit, killing it and failing otherwise; the tests' children live for five minutes. + internal static void Dies(string what, int pid) + { + if (!Support.Eventually(() => !Support.Alive(pid), TimeSpan.FromSeconds(10))) + { + Support.Kill(pid); + Assert.Fail(what + " is still running"); + } + } + + /// A shell command line. + internal static string[] Sh(string script) => new[] { "/bin/sh", "-c", script }; + + /// + /// A fresh owner-only scratch directory, under the temporary directory + /// when no other user may write it or a directory above it, and + /// otherwise under /tmp when that is so, so the SDK's directory + /// rule finds nothing unsafe above the scratch directory and the tests + /// that expect its exact refusals see only the levels they make. Fails + /// naming the requirement when neither is. + /// + internal static string Scratch(string prefix) + { + string name = "jacquard-dotnet-sdk-" + prefix + "-" + Convert.ToHexString(System.Security.Cryptography.RandomNumberGenerator.GetBytes(6)).ToLowerInvariant(); + if (!Unix.Supported) + { + return Directory.CreateTempSubdirectory("jacquard-dotnet-sdk-" + prefix + "-").FullName; + } + + foreach (string candidate in new[] { Path.GetTempPath(), "/tmp" }) + { + if (Directory.Exists(candidate) && SafeAncestry(Unix.RealPath(candidate))) + { + return Directory.CreateDirectory(Path.Combine(Unix.RealPath(candidate), name), UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute).FullName; + } + } + + throw new AssertionException("the SDK tests need a temporary directory that no other user, nor any group, may write, nor any directory above it (unless sticky); set TMPDIR to one"); + } + + /// + /// Whether , a real path, and every directory + /// above it is owned by this user or root and neither its group nor + /// others may write it unless it is sticky (without the private-group + /// exception, which tests answer through seams). + /// + internal static bool SafeAncestry(string directory) + { + for (string? level = directory; level is not null; level = Path.GetDirectoryName(level)) + { + Unix.Status status = Unix.Stat(level, follow: false); + if (status.Kind != Unix.Kind.Directory || (status.Owner != Unix.EffectiveUser() && status.Owner != 0) || ((status.Mode & 0x12) != 0 && (status.Mode & 0x200) == 0)) + { + return false; + } + } + + return true; + } + + /// + /// Answers the private-group lookups as a user "me" whose primary group + /// is 's group, named + /// with , and a + /// password database listing "me" with that primary group followed by + /// . + /// + internal static void Accounts(string directory, string name, string[] members, params Unix.PasswdEntry[] others) + { + uint group = Unix.Stat(directory, follow: false).Group; + uint me = Unix.EffectiveUser(); + Core.UserHook = uid => uid == me ? new Unix.Account("me", group) : null; + Core.GroupHook = gid => gid == group ? new Unix.GroupEntry(name, members) : null; + Core.GetentHook = Getent(Passwd(new[] { new Unix.PasswdEntry("me", me, group) }.Concat(others))); + } + + /// The getent passwd lines of . + internal static string Passwd(System.Collections.Generic.IEnumerable entries) => + string.Concat(entries.Select(e => string.Format(CultureInfo.InvariantCulture, "{0}:x:{1}:{2}:{0}:/home/{0}:/bin/sh\n", e.Name, e.Uid, e.Group))); + + /// A getent stand-in that prints and exits with . + internal static Func Getent(string stdout, int status = 0, bool overflow = false) => + _ => new CoreCommand.Output(status, Encoding.UTF8.GetBytes(stdout), overflow, Array.Empty()); + + /// libc's own password database enumeration, as other native code in the process would run it. + internal static partial class Libc + { + /// Reads every entry with setpwent, getpwent, and endpwent; how many. + internal static int ReadWholePasswordDatabase() + { + int count = 0; + Setpwent(); + while (Getpwent() != IntPtr.Zero && count < 1_000_000) + { + count++; + } + + Endpwent(); + return count; + } + + [System.Runtime.InteropServices.LibraryImport("libc", EntryPoint = "setpwent")] + private static partial void Setpwent(); + + [System.Runtime.InteropServices.LibraryImport("libc", EntryPoint = "getpwent")] + private static partial IntPtr Getpwent(); + + [System.Runtime.InteropServices.LibraryImport("libc", EntryPoint = "endpwent")] + private static partial void Endpwent(); + } + + /// Clears the private-group seams. + internal static void NoAccounts() + { + Core.UserHook = null; + Core.GroupHook = null; + Core.GetentHook = null; + Core.GetentStatHook = null; + Core.GetentPaths = new[] { "/usr/bin/getent", "/bin/getent" }; + Core.GetentArguments = new[] { "passwd" }; + Core.GetentTimeoutCap = Core.GetentTimeoutMs; + } + + /// + /// Removes a scratch directory, first restoring owner permissions on + /// everything in it (tests make some files unreadable). + /// + internal static void RemoveScratch(string root) + { + if (!Directory.Exists(root)) + { + return; + } + + foreach (string entry in Directory.EnumerateFileSystemEntries(root, "*", SearchOption.AllDirectories)) + { + try + { + if (new FileInfo(entry).LinkTarget is null) + { + File.SetUnixFileMode(entry, File.GetUnixFileMode(entry) | UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + } + catch (IOException) + { + // Gone already. + } + } + + Directory.Delete(root, recursive: true); + } + + /// A file of bytes that takes no space (sparse), with . + internal static string Sparse(string path, long length, UnixFileMode mode = UnixFileMode.None) + { + using (var stream = new FileStream(path, FileMode.CreateNew)) + { + stream.SetLength(length); + } + + File.SetUnixFileMode(path, mode); + return path; + } + + internal static void MakeFifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("mkfifo", path) ?? throw new InvalidOperationException("mkfifo"); + Assert.That(mkfifo.WaitForExit(30_000) && mkfifo.ExitCode == 0, "mkfifo failed"); + } + + /// The text of . + internal static string Text(byte[] bytes) => Encoding.UTF8.GetString(bytes); +} diff --git a/dotnet/test/JacquardHost.Tests/Support.cs b/dotnet/test/JacquardHost.Tests/Support.cs index 34ed3a8..6109817 100644 --- a/dotnet/test/JacquardHost.Tests/Support.cs +++ b/dotnet/test/JacquardHost.Tests/Support.cs @@ -60,7 +60,7 @@ internal static CallInterface Echo(bool withWorld = true) } /// The dotnet executable that runs the stand-in. - private static string Muxer() + internal static string Muxer() { string? host = Environment.GetEnvironmentVariable("DOTNET_HOST_PATH"); if (!string.IsNullOrEmpty(host) && File.Exists(host)) diff --git a/examples/README.md b/examples/README.md index f52318a..6eb0351 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,13 +1,14 @@ # Example projects -The same Jacquard rules run through all five SDKs. The commands below use +The same Jacquard rules run through all six SDKs. The commands below use Python; the [TypeScript walkthrough](../typescript/README.md#run-the-projects) includes compiled Node.js front ends for all four projects, the [Go walkthrough](../go/README.md#run-the-projects) a Go command with the rules and inputs embedded, the [Rust walkthrough](../rust/README.md#run-the-projects) a Cargo example that reads them from this directory, and the -[Java walkthrough](../java/README.md#run-the-projects) a Java command that reads -them from this directory too. Each includes an +[Java walkthrough](../java/README.md#run-the-projects) a Java command and the +[.NET walkthrough](../dotnet/README.md#run-the-projects) a .NET command that +read them from this directory too. Each includes an independent fulfillment recovery example. Start with [setup](../docs/SETUP.md), then run these from the repository root. diff --git a/scripts/check_distribution.py b/scripts/check_distribution.py index 15d963c..2f6b2ca 100644 --- a/scripts/check_distribution.py +++ b/scripts/check_distribution.py @@ -36,7 +36,7 @@ def main() -> None: # Only our freshly built archive is extracted, never an external input. archive.extractall(root / "source", filter="data") source = next((root / "source").iterdir()) - for relative in ("LICENSE", "NOTICE", "go/NOTICE", "rust/NOTICE", "java/NOTICE", "java/pom.xml", "dotnet/NOTICE", "dotnet/src/JacquardHost/JacquardHost.csproj", "dotnet/src/JacquardHost/Compat.cs", "dotnet/src/JacquardHost/kit/host-protocol-v0/kit/transcripts.json", "CONTRIBUTING.md", "CHANGELOG.md", "docs/SETUP.md", "scripts/check_examples.py", + for relative in ("LICENSE", "NOTICE", "go/NOTICE", "rust/NOTICE", "java/NOTICE", "java/pom.xml", "dotnet/NOTICE", "dotnet/src/JacquardHost/JacquardHost.csproj", "dotnet/src/JacquardHost/Compat.cs", "dotnet/src/JacquardHost/Sdk/Runtime.cs", "dotnet/examples/JacquardExamples/Examples.cs", "dotnet/src/JacquardHost/kit/host-protocol-v0/kit/transcripts.json", "CONTRIBUTING.md", "CHANGELOG.md", "docs/SETUP.md", "scripts/check_examples.py", "examples/python_sdk/app.py", "examples/python_sdk/exercise.py"): assert (source / relative).is_file(), f"sdist missing {relative}" for project in ("quickstart", "triage", "planner", "fulfillment"):