From c3d19d193568cf1e8b5fd3aa648dc572fc5d2159 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 07:04:14 -0400 Subject: [PATCH 01/52] Add bounded Core commands for the .NET SDK CoreCommand runs one of Core's public commands to completion on the caller's thread, reading its stdout and stderr only when the adapter's poll(2) reports them ready, bounded by a timeout, output limits, and a CancellationToken. On Linux with setsid at /usr/bin or /bin the command leads its own process group, which is killed with kill(-pgid) when the command exits or is stopped; a reaped leader's pid held by a live process is treated as reuse and nothing is killed. Without setsid, descendants recorded from /proc by pid and start time are killed at the exit, and Process.Kill(true) on a stop. Also add the SDK's one exception type, with its 8,192-char message cap and 64-char excerpts of caller text, Core's hash and signature parsers with depth, type, and token bounds, and the native calls the SDK needs (kill, geteuid, mkdir, statx or stat). --- dotnet/src/JacquardHost/Sdk/CoreCommand.cs | 716 ++++++++++++++++++ dotnet/src/JacquardHost/Sdk/Identities.cs | 494 ++++++++++++ .../src/JacquardHost/Sdk/JacquardException.cs | 188 +++++ dotnet/src/JacquardHost/Sdk/Unix.cs | 177 +++++ .../JacquardHost.Tests.csproj | 6 +- .../Sdk/CoreCommandTests.cs | 321 ++++++++ .../test/JacquardHost.Tests/Sdk/SdkSupport.cs | 99 +++ 7 files changed, 1999 insertions(+), 2 deletions(-) create mode 100644 dotnet/src/JacquardHost/Sdk/CoreCommand.cs create mode 100644 dotnet/src/JacquardHost/Sdk/Identities.cs create mode 100644 dotnet/src/JacquardHost/Sdk/JacquardException.cs create mode 100644 dotnet/src/JacquardHost/Sdk/Unix.cs create mode 100644 dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs create mode 100644 dotnet/test/JacquardHost.Tests/Sdk/SdkSupport.cs diff --git a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs new file mode 100644 index 0000000..ce0d153 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs @@ -0,0 +1,716 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.IO.Pipes; +using System.Threading; +using JacquardLang.Host.Carrier; + +namespace JacquardLang.Host.Sdk; + +/// +/// Runs one short command to completion, bounded in time and output, the way +/// the SDK runs Core's --version, check, hash, and +/// run commands. +/// +/// +/// +/// As in the adapter's worker carrier, the command's stdout and stderr are +/// read only after poll(2) reports them ready, on the caller's thread, +/// in waits of 1 to 5 ms that recheck the timeout and the +/// ; .NET's own stream reads, which cannot be +/// interrupted, are never left waiting. Its stdin is a pipe the host closes +/// at once, so it reads end of file. +/// +/// +/// On Linux, when a setsid is installed at /usr/bin/setsid or +/// /bin/setsid (util-linux's, or a compatible one such as BusyBox's), +/// the command runs as setsid -- <command>. .NET's child is never +/// a process group leader, so setsid starts a new session and execs +/// the command without forking: the command keeps the child's pid and leads +/// a new process group whose id is that pid, shared by every descendant that +/// does not leave it. The host kills that group with kill(-pgid, +/// SIGKILL) when the command exits, and on timeout or cancellation. +/// After the leader exits and .NET has reaped it, a live process holding the +/// leader's pid means that pid was reused, so the old group is empty and +/// nothing is killed; the window between that check and the kill is the same +/// as Rust's kill(-pgid) after a full pid wraparound. When +/// setsid cannot exec the command (exit status 126 or 127 with its own +/// "setsid: " message), the run is reported as a start failure. A +/// descendant that leaves the group (calls setsid itself) is neither +/// detected nor killed; the host closes its ends of the output after the +/// exit, so its next write fails. +/// +/// +/// Elsewhere (Linux without setsid), the command's descendants are +/// recorded from /proc about every 10 ms, each by pid and start time, +/// and every recorded descendant still alive with that start time is killed +/// when the command returns; on timeout or cancellation .NET's +/// kills the command's tree too. A +/// descendant started and reparented between two records keeps running. On +/// macOS, which has no /proc, nothing is recorded: only the tree kill +/// on a stop applies. +/// +/// +internal static class CoreCommand +{ + /// The most of each stream read after the command has exited. + internal const int ExitDrain = 4 << 20; + + /// How long the host waits for a killed command to exit. + private const int ExitGraceMs = 1000; + + /// The longest single wait between checks. + private const int PollSliceMs = 5; + + /// The most of each stream one step reads, so a fast writer cannot hold the caller. + private const int Slice = 1 << 20; + + /// How often descendants are recorded where there are no process groups. + private const long SampleIntervalMs = 10; + + private const short Ready = Posix.PollIn | Posix.PollHup | Posix.PollErr | Posix.PollNval; + + /// Test seam: when set, commands run without a process group, as where setsid is absent. + internal static volatile bool WithoutGroups; + + /// Test seam: whether descendants are recorded where there are no groups. + internal static volatile bool RecordDescendants = true; + + /// Test seam: runs after each record with every pid recorded so far. Null outside tests. + internal static volatile Action>? RecordedHook; + + /// Test seam: receives the pid of each command once it has started. Null outside tests. + internal static volatile Action? StartedHook; + + /// Test seam: runs after the cleanup that follows the exit, before the stop is checked again. Null outside tests. + internal static volatile Action? ExitedHook; + + /// Test seam: whether the group is killed when the command exits. + internal static volatile bool KillGroupAtExit = true; + + /// Test seam: whether the group is killed on a stop. + internal static volatile bool KillGroupOnStop = true; + + /// The setsid that starts each command in its own group, or null where groups are not used. + internal static string? Setsid { get; } = FindSetsid(); + + /// Whether the next command runs in its own process group. + internal static bool Grouped => Setsid is not null && !WithoutGroups; + + private static string? FindSetsid() + { + if (!OperatingSystem.IsLinux() || !File.Exists("/proc/self/stat")) + { + return null; + } + + foreach (string candidate in new[] { "/usr/bin/setsid", "/bin/setsid" }) + { + if (IsExecutable(candidate)) + { + return candidate; + } + } + + return null; + } + + private static bool IsExecutable(string path) + { + try + { + Unix.Status status = Unix.Stat(path, follow: true); + return status.Kind == Unix.Kind.Regular && (status.Mode & 0x49) != 0; + } + catch (IOException) + { + return false; + } + } + + /// + /// The command's executable, resolved on the PATH when it names no + /// directory, so that one setsid cannot exec is reported as a + /// start failure, as without setsid. + /// + private static string Executable(string name) + { + if (name.Contains('/', StringComparison.Ordinal)) + { + if (IsExecutable(name)) + { + return name; + } + } + else + { + string path = Environment.GetEnvironmentVariable("PATH") ?? string.Empty; + foreach (string directory in path.Split(':')) + { + string candidate = Path.Combine(directory.Length == 0 ? "." : directory, name); + if (IsExecutable(candidate)) + { + return candidate; + } + } + } + + throw CarrierException.Loss("could not start " + Identities.Excerpt(name) + " (not an executable file)"); + } + + /// A command that exited. + /// The exit status; .NET reports a signal N as 128 + N. + /// At most the requested limit of stdout. + /// Whether stdout was longer than its limit. + /// At most the requested limit of stderr. + internal sealed record Output(int ExitStatus, byte[] Stdout, bool StdoutOverflow, byte[] Stderr); + + /// A bounded prefix of one stream, read only when poll reports it ready. + private sealed class Capture + { + private readonly Stream stream; + private readonly int limit; + private readonly byte[] chunk = new byte[8192]; + private byte[] kept = Array.Empty(); + private int length; + + internal Capture(Stream stream, int fd, int limit) + { + this.stream = stream; + Fd = fd; + this.limit = limit; + } + + internal int Fd { get; } + + internal bool Overflow { get; private set; } + + internal bool Ended { get; private set; } + + /// Reads at most bytes already in the pipe; returns how many. Never blocks. + internal int Step(int budget) + { + int total = 0; + if (Ended) + { + return 0; + } + + try + { + Span fds = stackalloc Posix.PollFd[1]; + while (total < budget) + { + fds[0] = new Posix.PollFd { Fd = Fd, Events = Posix.PollIn }; + Posix.Poll(fds, 0); + if ((fds[0].Revents & Ready) == 0) + { + break; + } + + int n = stream.Read(chunk, 0, Math.Min(chunk.Length, budget - total)); + if (n <= 0) + { + Ended = true; + break; + } + + Keep(n); + total += n; + } + } + catch (Exception error) when (error is IOException or ObjectDisposedException or CarrierException) + { + Ended = true; + } + + return total; + } + + private void Keep(int n) + { + int room = limit - length; + if (n > room) + { + Overflow = true; + } + + int take = Math.Min(n, room); + if (take > 0) + { + if (length + take > kept.Length) + { + Array.Resize(ref kept, Math.Min(limit, Math.Max(length + take, kept.Length * 2))); + } + + Array.Copy(chunk, 0, kept, length, take); + length += take; + } + } + + internal byte[] Bytes() => kept.AsSpan(0, length).ToArray(); + } + + /// The command's descendants seen while it ran, by pid, with their start times (Linux). + private sealed class Recorded + { + private readonly Dictionary started = new(); + private long next = Stopwatch.GetTimestamp(); + + /// Records the command's current descendants, at most once per . + internal void Sample(int root) + { + long now = Stopwatch.GetTimestamp(); + if (!RecordDescendants || now < next || !OperatingSystem.IsLinux()) + { + return; + } + + next = now + (Stopwatch.Frequency * SampleIntervalMs / 1000); + var children = new Dictionary>(); + foreach (string entry in ProcEntries()) + { + if (ReadStat(entry) is { } stat) + { + if (!children.TryGetValue(stat.Parent, out List<(int, string)>? list)) + { + list = new List<(int, string)>(); + children[stat.Parent] = list; + } + + list.Add((stat.Pid, stat.Start)); + } + } + + var queue = new Queue(); + var seen = new HashSet { root }; + queue.Enqueue(root); + while (queue.Count > 0) + { + if (!children.TryGetValue(queue.Dequeue(), out List<(int Pid, string Start)>? list)) + { + continue; + } + + foreach ((int pid, string start) in list) + { + if (seen.Add(pid)) + { + started.TryAdd(pid, start); + queue.Enqueue(pid); + } + } + } + + RecordedHook?.Invoke(new List(started.Keys)); + } + + /// + /// Kills every recorded descendant still alive with the start time it + /// was recorded with, so a reused pid is not killed (the start time is + /// read just before the kill, as the JDK's process handles do). + /// + internal void KillAlive() + { + foreach ((int pid, string start) in started) + { + if (ReadStat("/proc/" + pid.ToString(CultureInfo.InvariantCulture)) is { } stat && stat.Start == start && !stat.Zombie) + { + Unix.Kill(pid); + } + } + } + } + + private static IEnumerable ProcEntries() + { + IEnumerable entries; + try + { + entries = Directory.EnumerateDirectories("/proc"); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + yield break; + } + + using IEnumerator cursor = entries.GetEnumerator(); + while (true) + { + try + { + if (!cursor.MoveNext()) + { + yield break; + } + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + yield break; + } + + string name = Path.GetFileName(cursor.Current); + if (name.Length > 0 && name.Length <= 10 && IsDigits(name)) + { + yield return cursor.Current; + } + } + } + + private static bool IsDigits(string text) + { + foreach (char c in text) + { + if (c < '0' || c > '9') + { + return false; + } + } + + return true; + } + + /// + /// A process's pid, parent, start time, and whether it is a zombie, from + /// /proc/<pid>/stat, read after the last ) (the + /// command name may hold any byte); null when it cannot be read. + /// + private static (int Pid, int Parent, int Group, string Start, bool Zombie)? ReadStat(string directory) + { + try + { + string stat = File.ReadAllText(Path.Combine(directory, "stat")); + int close = stat.LastIndexOf(')'); + if (close < 0 || !int.TryParse(Path.GetFileName(directory), NumberStyles.None, CultureInfo.InvariantCulture, out int pid)) + { + return null; + } + + // state ppid pgrp session tty_nr tpgid flags minflt cminflt majflt + // cmajflt utime stime cutime cstime priority nice num_threads + // itrealvalue starttime ... + string[] fields = stat[(close + 1)..].Trim().Split(' '); + if (fields.Length < 20 + || !int.TryParse(fields[1], NumberStyles.None, CultureInfo.InvariantCulture, out int parent) + || !int.TryParse(fields[2], NumberStyles.None, CultureInfo.InvariantCulture, out int group)) + { + return null; + } + + return (pid, parent, group, fields[19], fields[0] is "Z" or "X"); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + return null; + } + } + + /// + /// Kills group , the pid of , + /// unless the leader has exited and a live process now holds that pid: + /// while a group has members its id cannot be reused, so a reaped + /// leader's pid held by a live process means the old group is empty. + /// + private static void KillGroup(int group, Process leader) + { + if (HasExited(leader) && ReadStat("/proc/" + group.ToString(CultureInfo.InvariantCulture)) is { Zombie: false }) + { + return; + } + + Unix.KillGroup(group); + } + + private static bool HasExited(Process process) + { + try + { + return process.HasExited; + } + catch (Exception error) when (error is InvalidOperationException or Win32Exception) + { + return true; + } + } + + /// + /// Whether setsid itself failed to exec the command: it exits 126 + /// (cannot execute) or 127 (not found) and its message, unlike the + /// command's, starts with "setsid: " (util-linux and BusyBox both + /// print that prefix). The command never ran then. + /// + internal static bool SetsidFailed(Output output) => + output.ExitStatus is 126 or 127 && System.Text.Encoding.UTF8.GetString(output.Stderr).StartsWith("setsid: ", StringComparison.Ordinal); + + /// + /// Runs , adding + /// to the inherited environment, and returns once it has exited. + /// + /// The executable and its arguments. + /// Variables added to the inherited environment. + /// Bounds the whole run, from the start to the exit. + /// How much stdout is kept; the rest is read and counted as overflow. + /// How much stderr is kept. + /// Stops the command; it is then killed. + /// + /// A loss when the command cannot start; a timeout when it does not exit + /// in time; aborted when the token is cancelled. The command is killed in + /// the last two cases. + /// + internal static Output Run( + IReadOnlyList command, + IReadOnlyDictionary? environment, + TimeSpan timeout, + int stdoutLimit, + int stderrLimit, + CancellationToken cancellationToken) + { + if (command.Count == 0 || command[0].Length == 0) + { + throw new ArgumentException("command must be a nonempty executable argument list", nameof(command)); + } + + if (timeout <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(timeout), "timeout must be positive"); + } + + long expires = Worker.Expires(Stopwatch.GetTimestamp(), timeout); + if (Stopped(expires, cancellationToken) is { } early) + { + throw early; + } + + bool grouped = Grouped; + var info = new ProcessStartInfo(grouped ? Setsid! : command[0]) + { + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + if (grouped) + { + info.ArgumentList.Add("--"); + info.ArgumentList.Add(Executable(command[0])); + } + + for (int i = 1; i < command.Count; i++) + { + info.ArgumentList.Add(command[i]); + } + + if (environment is not null) + { + foreach (KeyValuePair variable in environment) + { + info.Environment[variable.Key] = variable.Value; + } + } + + Process process; + try + { + process = Process.Start(info) ?? throw new InvalidOperationException("no process was started"); + } + catch (Exception error) when (error is Win32Exception or InvalidOperationException or IOException or PlatformNotSupportedException) + { + throw CarrierException.Loss("could not start " + Identities.Excerpt(command[0]) + " (" + error.Message + ")"); + } + + int pid = process.Id; + StartedHook?.Invoke(pid); + Recorded? recorded = grouped ? null : new Recorded(); + Stream? stdout = null; + Stream? stderr = null; + bool cleaned = false; + try + { + try + { + // Its stdin reads end of file at once. + process.StandardInput.Close(); + } + catch (Exception error) when (error is IOException or ObjectDisposedException) + { + // The command already closed its end. + } + + stdout = process.StandardOutput.BaseStream; + stderr = process.StandardError.BaseStream; + var output = new Capture(stdout, Descriptor(stdout), stdoutLimit); + var errors = new Capture(stderr, Descriptor(stderr), stderrLimit); + Output result = Await(process, pid, grouped, recorded, output, errors, expires, cancellationToken, ref cleaned); + if (grouped && SetsidFailed(result)) + { + string detail = System.Text.Encoding.UTF8.GetString(result.Stderr).Trim(); + int line = detail.IndexOf('\n', StringComparison.Ordinal); + throw CarrierException.Loss("could not start " + Identities.Excerpt(command[0]) + " (" + (line < 0 ? detail : detail[..line]) + ")"); + } + + return result; + } + finally + { + if (!cleaned) + { + if (grouped) + { + // While the command is alive, or unreaped, its group id is reserved. + if (KillGroupOnStop) + { + KillGroup(pid, process); + } + + KillQuietly(process, entireTree: false); + AwaitExit(process); + if (KillGroupOnStop) + { + // Members forked while the first kill ran. + KillGroup(pid, process); + } + } + else + { + if (!HasExited(process)) + { + KillQuietly(process, entireTree: true); + AwaitExit(process); + } + + recorded!.KillAlive(); + } + } + + // A descendant still writing has no reader now, so its next write fails. + stdout?.Dispose(); + stderr?.Dispose(); + process.Dispose(); + } + } + + private static int Descriptor(Stream stream) => + stream is PipeStream pipe + ? (int)pipe.SafePipeHandle.DangerousGetHandle() + : throw CarrierException.Loss("the command's output is " + stream.GetType().Name + ", not a pipe"); + + private static Output Await( + Process process, + int pid, + bool grouped, + Recorded? recorded, + Capture stdout, + Capture stderr, + long expires, + CancellationToken cancellationToken, + ref bool cleaned) + { + int idle = 1; + Span fds = stackalloc Posix.PollFd[2]; + while (true) + { + // The exit is observed before the pipes are read, so everything + // the command itself wrote is read below. + bool exited = HasExited(process); + if (!exited) + { + recorded?.Sample(pid); + } + + int read = stdout.Step(Slice) + stderr.Step(Slice); + if (exited) + { + // The cleanup always runs to the end, as Rust kills the group + // when the command exits. + if (grouped) + { + if (KillGroupAtExit) + { + KillGroup(pid, process); + } + } + else + { + recorded!.KillAlive(); + } + + cleaned = true; + stdout.Step(ExitDrain); + stderr.Step(ExitDrain); + ExitedHook?.Invoke(); + + // The cleanup may have outlived the deadline or a stop. + if (Stopped(expires, cancellationToken) is { } late) + { + throw late; + } + + return new Output(process.ExitCode, stdout.Bytes(), stdout.Overflow, stderr.Bytes()); + } + + if (Stopped(expires, cancellationToken) is { } stopped) + { + throw stopped; + } + + if (read > 0) + { + idle = 1; + continue; + } + + int count = 0; + foreach (Capture capture in new[] { stdout, stderr }) + { + if (!capture.Ended) + { + fds[count++] = new Posix.PollFd { Fd = capture.Fd, Events = Posix.PollIn }; + } + } + + Posix.Poll(fds[..count], Math.Min(idle, Worker.RemainingMs(expires, Stopwatch.GetTimestamp()))); + idle = Math.Min(idle * 2, PollSliceMs); + } + } + + /// Why the run must stop now, or null. + private static CarrierException? Stopped(long expires, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + { + return CarrierException.Aborted("command aborted"); + } + + return Stopwatch.GetTimestamp() >= expires ? CarrierException.Timeout("command did not exit in time") : null; + } + + private static void KillQuietly(Process process, bool entireTree) + { + try + { + process.Kill(entireTree); + } + catch (Exception error) when (error is InvalidOperationException or Win32Exception or AggregateException or NotSupportedException) + { + // Already gone, or a descendant could not be signalled. + } + } + + /// + /// Waits up to one second for a killed command to exit, in short + /// poll(2) sleeps, so the wait neither blocks on a .NET wait handle + /// nor consumes a . + /// + private static void AwaitExit(Process process) + { + long end = Stopwatch.GetTimestamp() + (Stopwatch.Frequency * ExitGraceMs / 1000); + while (!HasExited(process) && Stopwatch.GetTimestamp() < end) + { + Posix.Poll(Span.Empty, 1); + } + } +} diff --git a/dotnet/src/JacquardHost/Sdk/Identities.cs b/dotnet/src/JacquardHost/Sdk/Identities.cs new file mode 100644 index 0000000..5255754 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Identities.cs @@ -0,0 +1,494 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// Public names and identities printed by Core's hash and check --print-sigs commands, and the SDK's text bounds. +internal static class Identities +{ + /// The longest name, in chars, that is converted between spellings. + internal const int NameLimit = 4096; + + /// The longest excerpt of caller text quoted in a message, in chars. + internal const int ExcerptLimit = 64; + + /// + /// The deepest type a signature may nest (a type with no arguments has + /// depth 1): the protocol's value-depth limit (max_json_depth, 64), + /// since no deeper value could be passed. + /// + internal const int SignatureDepth = 64; + + /// The most types one signature may name. + internal const int SignatureNodes = 4096; + + /// The most tokens one signature may have, which bounds the parser's work before any type is counted. + internal const int SignatureTokens = 4 * SignatureNodes; + + /// One public name and identity printed by jacquard hash. + internal sealed record Entry(string Name, Hash Hash); + + /// for a message: whole when short, otherwise its first 64 chars and its length. + internal static string Excerpt(string? text) + { + if (text is null) + { + return "null"; + } + + if (text.Length <= ExcerptLimit) + { + return text; + } + + int end = char.IsHighSurrogate(text[ExcerptLimit - 1]) ? ExcerptLimit - 1 : ExcerptLimit; + return string.Concat(text.AsSpan(0, end), "... (" + text.Length.ToString(CultureInfo.InvariantCulture) + " chars)"); + } + + /// + /// for a message: its type and an excerpt of its + /// message, never its whole text. Reading a caller's exception message + /// runs the caller's code; a failure there is described by type alone. + /// + [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1031", Justification = "A caller's Message property may throw anything; the description falls back to the type.")] + internal static string Describe(Exception error) + { + string? message; + try + { + message = error.Message; + } + catch (Exception) + { + message = null; + } + + string type = error.GetType().FullName ?? error.GetType().Name; + return message is null ? type : type + ": " + Excerpt(message); + } + + /// Converts a display name such as WorkToday to its store spelling, work-today; lowercase names are unchanged. + internal static string DisplayToStore(string name) + { + // A name longer than any the SDK maps is returned as given, so a huge + // one is never expanded; it then names nothing. + if (name.Length == 0 || name.Length > NameLimit) + { + return name; + } + + // A lone surrogate decodes as U+FFFD, which is not upper case. + _ = Rune.DecodeFromUtf16(name, out Rune first, out _); + if (!Rune.IsUpper(first)) + { + return name; + } + + var output = new StringBuilder(name.Length + 8); + int index = 0; + foreach (Rune rune in name.EnumerateRunes()) + { + if (index > 0 && Rune.IsUpper(rune)) + { + output.Append('-'); + } + + output.Append(Rune.ToLowerInvariant(rune).ToString()); + index++; + } + + return output.ToString(); + } + + /// The form used to look up effects and operations: store spelling, with _ accepted for -. + internal static string Normal(string name) => name.Length > NameLimit ? name : DisplayToStore(name).Replace('_', '-'); + + /// Parses one N:name hash line, or returns null. + private static (string Index, Entry Entry)? HashLine(string line) + { + int colon = line.IndexOf(':', StringComparison.Ordinal); + if (colon <= 0) + { + return null; + } + + string index = line[..colon]; + string rest = line[(colon + 1)..]; + int space = rest.IndexOf(' ', StringComparison.Ordinal); + if (space < 0) + { + return null; + } + + string name = rest[..space]; + foreach (char c in index) + { + if (c < '0' || c > '9') + { + return null; + } + } + + if (name.Length == 0) + { + return null; + } + + foreach (Rune rune in name.EnumerateRunes()) + { + if (Rune.IsWhiteSpace(rune)) + { + return null; + } + } + + string hex = rest[(space + 1)..]; + return Hash.LowerHex(hex, 64) ? (index, new Entry(name, new Hash(hex))) : null; + } + + /// + /// Parses jacquard hash output into declaration groups: a type or + /// effect identity followed by its constructors or operations, in Core's + /// printed order. + /// + /// A binding failure when the output names no identity. + internal static List> HashGroups(string output) + { + var order = new List(); + var groups = new Dictionary>(StringComparer.Ordinal); + foreach (string line in output.Split('\n')) + { + (string Index, Entry Entry)? parsed = HashLine(line.Trim()); + if (parsed is { } found) + { + if (!groups.TryGetValue(found.Index, out List? group)) + { + group = new List(); + groups[found.Index] = group; + order.Add(found.Index); + } + + group.Add(found.Entry); + } + } + + if (order.Count == 0) + { + throw JacquardException.Binding("Core hash command returned no public identities"); + } + + var result = new List>(order.Count); + foreach (string index in order) + { + result.Add(groups[index]); + } + + return result; + } + + /// Refuses a public name that denotes different identities in different declaration groups. + /// A binding failure naming the ambiguous name. + internal static void CheckGroups(List> groups) + { + var seen = new Dictionary>(StringComparer.Ordinal); + foreach (List group in groups) + { + var current = new Dictionary>(StringComparer.Ordinal); + var names = new List(); + foreach (Entry entry in group) + { + if (!current.TryGetValue(entry.Name, out HashSet? set)) + { + set = new HashSet(); + current[entry.Name] = set; + names.Add(entry.Name); + } + + set.Add(entry.Hash); + } + + foreach (string name in names) + { + if (!seen.TryGetValue(name, out HashSet? previous)) + { + previous = new HashSet(); + seen[name] = previous; + } + + if (previous.Count > 0 && !previous.IsSupersetOf(current[name])) + { + throw JacquardException.Binding("ambiguous public name " + Excerpt(name) + + " occurs in different declaration groups; use distinct names or explicit low-level identities"); + } + + previous.UnionWith(current[name]); + } + } + } + + private static JacquardException TooDeep() => + JacquardException.Binding("signature nests types deeper than " + SignatureDepth.ToString(CultureInfo.InvariantCulture) + + " levels, the protocol's value-depth limit (max_json_depth)"); + + /// Tokens of a printed signature, at most a few per type, so the token count is bounded too. + private static List Tokens(string text) + { + var tokens = new List(); + int at = 0; + int end = text.Length; + while (true) + { + while (at < end && char.IsWhiteSpace(text[at])) + { + at++; + } + + if (at == end) + { + return tokens; + } + + if (tokens.Count >= SignatureTokens) + { + throw JacquardException.Binding("signature is longer than " + SignatureTokens.ToString(CultureInfo.InvariantCulture) + " tokens"); + } + + int length; + char first = text[at]; + if (string.CompareOrdinal(text, at, "->{", 0, 3) == 0) + { + length = 3; + } + else if ("(),{}".Contains(first, StringComparison.Ordinal)) + { + length = 1; + } + else if ((first >= 'a' && first <= 'z') || (first >= 'A' && first <= 'Z')) + { + length = 1; + while (at + length < end) + { + char c = text[at + length]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || ".?!-".Contains(c, StringComparison.Ordinal))) + { + break; + } + + length++; + } + } + else + { + throw JacquardException.Binding("unsupported signature near " + Excerpt(text.Substring(at, Math.Min(12, end - at)))); + } + + tokens.Add(text.Substring(at, length)); + at += length; + } + } + + /// + /// Reads a closed v0 signature printed by check --print-sigs, such as + /// (List Line, Int) ->{Inventory} Option Text. Polymorphic and + /// effect-polymorphic signatures are refused, and so is one with more than + /// tokens, one whose types nest deeper than + /// levels, or one with more than + /// types. The nesting is refused before the + /// parser recurses past it, so no signature can exhaust the stack. + /// + /// A binding failure naming why the signature has no binding. + internal static CallInterface ParseSignature(string text, IReadOnlyDictionary names) => new Parser(Tokens(text), names).Parse(); + + /// + /// Refuses a type deeper than or, with + /// , more than types. + /// It walks with an explicit stack, so a deep type cannot exhaust it. + /// + private static void Measure(BoundaryType type, ref int nodes) + { + var stack = new Stack<(BoundaryType Type, int Level)>(); + stack.Push((type, 1)); + while (stack.Count > 0) + { + (BoundaryType current, int level) = stack.Pop(); + if (level > SignatureDepth) + { + throw TooDeep(); + } + + if (++nodes > SignatureNodes) + { + throw JacquardException.Binding("signature names more than " + SignatureNodes.ToString(CultureInfo.InvariantCulture) + " types"); + } + + IReadOnlyList children = current is NominalType nominal ? nominal.Arguments : ((TupleType)current).Items; + foreach (BoundaryType child in children) + { + stack.Push((child, level + 1)); + } + } + } + + private sealed class Parser + { + private static readonly HashSet Ends = new(StringComparer.Ordinal) { string.Empty, ",", ")", "}", "->{" }; + + private readonly List tokens; + private readonly IReadOnlyDictionary names; + private int index; + private int frames; + + internal Parser(List tokens, IReadOnlyDictionary names) + { + this.tokens = tokens; + this.names = names; + } + + private string Peek() => index < tokens.Count ? tokens[index] : string.Empty; + + private string Take(string expected) + { + string token = Peek(); + if (token.Length == 0 || (expected.Length > 0 && !string.Equals(token, expected, StringComparison.Ordinal))) + { + throw JacquardException.Binding("expected \"" + expected + "\", found \"" + Excerpt(token) + "\""); + } + + index++; + return token; + } + + private Hash Identity(string display) + { + if (names.TryGetValue(DisplayToStore(display), out Hash? hash) || names.TryGetValue(display, out hash)) + { + return hash; + } + + throw JacquardException.Binding("no identity for " + Excerpt(display)); + } + + private List Items(string close) + { + var output = new List(); + if (string.Equals(Peek(), close, StringComparison.Ordinal)) + { + index++; + return output; + } + + while (true) + { + output.Add(Type()); + if (string.Equals(Peek(), ",", StringComparison.Ordinal)) + { + index++; + if (string.Equals(Peek(), close, StringComparison.Ordinal)) + { + index++; + return output; + } + + continue; + } + + Take(close); + return output; + } + } + + private BoundaryType Atom() + { + string token = Take(string.Empty); + if (token == "(") + { + List items = Items(")"); + if (items.Count == 1 && !string.Equals(tokens[index - 2], ",", StringComparison.Ordinal)) + { + return items[0]; + } + + return new TupleType(items); + } + + if (!char.IsLetter(token[0])) + { + throw JacquardException.Binding("unsupported type \"" + Excerpt(token) + "\""); + } + + return new NominalType(Identity(token)); + } + + private BoundaryType Type() + { + // Each nested parse is inside a type one level shallower (Core + // prints no redundant parentheses), so this bounds the recursion + // before it goes further. + if (++frames > SignatureDepth) + { + throw TooDeep(); + } + + BoundaryType head = Atom(); + if (head is NominalType nominal) + { + var arguments = new List(); + while (!Ends.Contains(Peek())) + { + arguments.Add(Atom()); + } + + head = new NominalType(nominal.Identity, arguments); + } + + frames--; + return head; + } + + internal CallInterface Parse() + { + Take("("); + List parameters = Items(")"); + Take("->{"); + var effects = new List(); + if (!string.Equals(Peek(), "}", StringComparison.Ordinal)) + { + while (true) + { + Hash identity = Identity(Take(string.Empty)); + if (!effects.Contains(identity)) + { + effects.Add(identity); + } + + if (!string.Equals(Peek(), ",", StringComparison.Ordinal)) + { + break; + } + + index++; + } + } + + Take("}"); + BoundaryType result = Type(); + if (Peek().Length != 0) + { + throw JacquardException.Binding("trailing signature tokens"); + } + + effects.Sort(); + int nodes = 0; + foreach (BoundaryType parameter in parameters) + { + Measure(parameter, ref nodes); + } + + Measure(result, ref nodes); + return new CallInterface(parameters, result, effects); + } + } +} diff --git a/dotnet/src/JacquardHost/Sdk/JacquardException.cs b/dotnet/src/JacquardHost/Sdk/JacquardException.cs new file mode 100644 index 0000000..8d9b3ce --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/JacquardException.cs @@ -0,0 +1,188 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Globalization; + +namespace JacquardLang.Host.Sdk; + +/// What an SDK failure is about. +public enum JacquardErrorKind +{ + /// The program could not be checked, hashed, installed, or loaded from its cache. + Preparation, + + /// + /// A mapping or binding problem: an unknown name, a missing handler, or a + /// successful Core result that does not decode into the requested type + /// ( is then set). + /// + Binding, + + /// A local input that cannot be sent; no worker started. + InvalidArgument, + + /// The invocation ran and did not succeed; classifies it. + Invocation, +} + +/// +/// Every SDK failure. says what failed; an invocation that +/// ran keeps its raw . A failure caused by the caller's +/// own code (a codec, a record accessor or constructor) keeps that exception +/// as . The message is at most +/// chars: a longer one keeps its start and notes +/// its whole length. +/// +public sealed class JacquardException : Exception +{ + /// The longest message kept, in chars, its noted length included. + public const int MessageLimit = 8192; + + private static readonly IReadOnlyList Nothing = new ReadOnlyCollection(Array.Empty()); + + private JacquardException(JacquardErrorKind kind, string message, InvocationResult? result, Exception? cause, int batchIndex, IReadOnlyList completed) + : base(Bounded(message), cause) + { + Kind = kind; + Result = result; + BatchIndex = batchIndex; + Completed = completed; + } + + /// What failed. + public JacquardErrorKind Kind { get; } + + /// + /// The raw result, when a worker ran: always for + /// , and for a successful result + /// that did not decode; otherwise null. + /// + public InvocationResult? Result { get; } + + /// How the invocation failed, for ; otherwise null. + public Failure? Failure => InnerException is InvocationException error ? error.Failure : null; + + /// + /// For Function.MapBatches: + /// the index of the batch that failed, or -1 when a batch was refused + /// before any worker started (and for every other failure). + /// + public int BatchIndex { get; } + + /// + /// For MapBatches: the decoded values of the batches that completed + /// before the failed one, in order, read-only. Nothing is retried. + /// + public IReadOnlyList Completed { get; } + + /// within chars, its start kept and its whole length noted. + internal static string Bounded(string? message) => Joined(string.Empty, message); + + /// + /// followed by , in at + /// most chars, noting the whole length once. A + /// message that was already cut keeps its own noted length, plus the + /// prefix, so it is never cut twice or noted wrongly. Nothing longer than + /// the limit is copied. + /// + internal static string Joined(string prefix, string? message) + { + message ??= "null"; + long noted = NotedLength(message); + long total = prefix.Length + (noted < 0 ? message.Length : noted); + if (noted < 0 && total <= MessageLimit) + { + return prefix + message; + } + + int bodyLength = noted < 0 ? message.Length : message.LastIndexOf("... (", StringComparison.Ordinal); + string suffix = "... (" + total.ToString(CultureInfo.InvariantCulture) + " chars)"; + int room = MessageLimit - suffix.Length - prefix.Length; + if (room <= 0) + { + // A prefix past the limit (not one the SDK builds): keep its start. + int keep = MessageLimit - suffix.Length; + if (char.IsHighSurrogate(prefix[keep - 1])) + { + keep--; + } + + return string.Concat(prefix.AsSpan(0, keep), suffix); + } + + int end = Math.Min(room, bodyLength); + if (end > 0 && end < bodyLength && char.IsHighSurrogate(message[end - 1])) + { + end--; + } + + return string.Concat(prefix, message.AsSpan(0, end), suffix); + } + + /// The noted length of a message already cut by , or -1. + private static long NotedLength(string message) + { + // A cut that dropped a high surrogate is one char shorter. + if (message.Length < MessageLimit - 1 || message.Length > MessageLimit || !message.EndsWith(" chars)", StringComparison.Ordinal)) + { + return -1; + } + + int open = message.LastIndexOf("... (", StringComparison.Ordinal); + if (open < 0) + { + return -1; + } + + ReadOnlySpan digits = message.AsSpan(open + 5, message.Length - 7 - (open + 5)); + if (!long.TryParse(digits, NumberStyles.None, CultureInfo.InvariantCulture, out long noted)) + { + return -1; + } + + // No string is longer than int.MaxValue, so a larger note is not the SDK's. + return noted < message.Length || noted > int.MaxValue ? -1 : noted; + } + + internal static JacquardException Preparation(string message) => new(JacquardErrorKind.Preparation, message, null, null, -1, Nothing); + + internal static JacquardException Binding(string message) => new(JacquardErrorKind.Binding, message, null, null, -1, Nothing); + + /// A binding failure caused by , such as a caller's codec that threw. + internal static JacquardException Binding(string message, Exception? cause) => new(JacquardErrorKind.Binding, message, null, cause, -1, Nothing); + + /// A successful Core result that could not be mapped; the result is kept. + internal static JacquardException Binding(string message, InvocationResult result, Exception? cause) => + new(JacquardErrorKind.Binding, message, result, cause, -1, Nothing); + + internal static JacquardException Invalid(string message) => new(JacquardErrorKind.InvalidArgument, message, null, null, -1, Nothing); + + /// An invalid argument at a value's location, such as shipping-cost.argument[0]. + internal static JacquardException InvalidAt(string path, string? message, Exception? cause = null) => + new(JacquardErrorKind.InvalidArgument, Joined(path + ": ", message), null, cause, -1, Nothing); + + /// A failed invocation; is not ok. + internal static JacquardException Invocation(InvocationResult result) => + new(JacquardErrorKind.Invocation, result.Message, result, new InvocationException(result), -1, Nothing); + + /// + /// as the failure of batch , + /// after the decoded values of the batches before it, copied read-only. + /// + internal static JacquardException Batch(int index, IReadOnlyList completed, JacquardException error) + { + var copy = new object?[completed.Count]; + for (int i = 0; i < copy.Length; i++) + { + copy[i] = completed[i]; + } + + return new JacquardException( + error.Kind, + Joined("batch " + index.ToString(CultureInfo.InvariantCulture) + ": ", error.Message), + error.Result, + error.InnerException, + index, + new ReadOnlyCollection(copy)); + } +} diff --git a/dotnet/src/JacquardHost/Sdk/Unix.cs b/dotnet/src/JacquardHost/Sdk/Unix.cs new file mode 100644 index 0000000..7e5e6c5 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Unix.cs @@ -0,0 +1,177 @@ +using System; +using System.Globalization; +using System.IO; +using System.Runtime.InteropServices; + +namespace JacquardLang.Host.Sdk; + +/// +/// The C library calls the SDK makes beyond the carrier's poll(2): +/// kill(2) to stop a Core command's process group, geteuid(2) +/// and a status call (statx(2) on Linux, stat(2) and +/// lstat(2) on macOS) for the cache's owner check and for file kinds +/// and sizes, and mkdir(2) to create a staging directory exclusively. +/// +/// +/// The macOS calls use the 64-bit-inode struct stat layout; they are +/// compiled and selected at run time but, like the rest of the .NET host on +/// macOS, not exercised by the tests or CI. +/// +internal static partial class Unix +{ + internal const int Sigkill = 9; + + private const int Enoent = 2; + private const int Enotdir = 20; + private const int Eexist = 17; + private const int AtFdcwd = -100; + private const int AtSymlinkNofollow = 0x100; + private const uint StatxBasicStats = 0x7ff; + private const int FileTypeMask = 0xF000; + private const int RegularType = 0x8000; + private const int DirectoryType = 0x4000; + private const int LinkType = 0xA000; + + /// A file's kind. + internal enum Kind + { + Missing, + Regular, + Directory, + Link, + Other, + } + + /// What the status call reports: the kind, size, owner, and permission bits. + internal readonly record struct Status(Kind Kind, long Size, uint Owner, int Mode); + + [LibraryImport("libc", EntryPoint = "kill", SetLastError = true)] + private static partial int NativeKill(int pid, int signal); + + [LibraryImport("libc", EntryPoint = "geteuid")] + private static partial uint NativeGeteuid(); + + [LibraryImport("libc", EntryPoint = "mkdir", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static partial int NativeMkdir(string path, uint mode); + + [LibraryImport("libc", EntryPoint = "statx", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static unsafe partial int NativeStatx(int dirfd, string path, int flags, uint mask, byte* buffer); + + [LibraryImport("libc", EntryPoint = "stat", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static unsafe partial int MacStatArm(string path, byte* buffer); + + [LibraryImport("libc", EntryPoint = "lstat", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static unsafe partial int MacLstatArm(string path, byte* buffer); + + [LibraryImport("libc", EntryPoint = "stat$INODE64", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static unsafe partial int MacStatIntel(string path, byte* buffer); + + [LibraryImport("libc", EntryPoint = "lstat$INODE64", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static unsafe partial int MacLstatIntel(string path, byte* buffer); + + /// Whether the SDK's file and process checks can run here: Linux or macOS. + internal static bool Supported => OperatingSystem.IsLinux() || OperatingSystem.IsMacOS(); + + /// This process's effective user id. + internal static uint EffectiveUser() => NativeGeteuid(); + + /// + /// Sends SIGKILL to every process in group . + /// A group id of one or less is never signalled (kill(-1) and + /// kill(0) mean every process and the caller's own group). + /// + internal static void KillGroup(int group) + { + if (group > 1) + { + _ = NativeKill(-group, Sigkill); + } + } + + /// Sends SIGKILL to , never to a pid of one or less. + internal static void Kill(int pid) + { + if (pid > 1) + { + _ = NativeKill(pid, Sigkill); + } + } + + /// Creates with ; false when it already exists. + /// The directory cannot be created for another reason. + internal static bool MakeDirectory(string path, int mode) + { + if (NativeMkdir(path, (uint)mode) == 0) + { + return true; + } + + int error = Marshal.GetLastPInvokeError(); + if (error == Eexist) + { + return false; + } + + throw new IOException("mkdir failed with errno " + error.ToString(CultureInfo.InvariantCulture)); + } + + /// + /// The status of , following a final symbolic link + /// when is set; when + /// it does not exist. + /// + /// The status cannot be read. + internal static unsafe Status Stat(string path, bool follow) + { + byte* buffer = stackalloc byte[256]; + new Span(buffer, 256).Clear(); + int result; + int modeOffset; + int ownerOffset; + int sizeOffset; + if (OperatingSystem.IsMacOS()) + { + bool arm = RuntimeInformation.ProcessArchitecture == Architecture.Arm64; + result = (follow, arm) switch + { + (true, true) => MacStatArm(path, buffer), + (false, true) => MacLstatArm(path, buffer), + (true, false) => MacStatIntel(path, buffer), + _ => MacLstatIntel(path, buffer), + }; + modeOffset = 4; + ownerOffset = 16; + sizeOffset = 96; + } + else + { + result = NativeStatx(AtFdcwd, path, follow ? 0 : AtSymlinkNofollow, StatxBasicStats, buffer); + modeOffset = 28; + ownerOffset = 20; + sizeOffset = 40; + } + + if (result != 0) + { + int error = Marshal.GetLastPInvokeError(); + if (error == Enoent || error == Enotdir) + { + return new Status(Kind.Missing, 0, 0, 0); + } + + throw new IOException("cannot read its status (errno " + error.ToString(CultureInfo.InvariantCulture) + ")"); + } + + int mode = *(ushort*)(buffer + modeOffset); + uint owner = *(uint*)(buffer + ownerOffset); + long size = *(long*)(buffer + sizeOffset); + Kind kind = (mode & FileTypeMask) switch + { + RegularType => Kind.Regular, + DirectoryType => Kind.Directory, + LinkType => Kind.Link, + _ => Kind.Other, + }; + return new Status(kind, size, owner, mode & 0xFFF); + } +} diff --git a/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj b/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj index 15e13ae..6ae2d10 100644 --- a/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj +++ b/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj @@ -9,8 +9,10 @@ false true 8.0 - - $(NoWarn);CA1707 + + $(NoWarn);CA1707;CA1416 diff --git a/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs new file mode 100644 index 0000000..63f2c92 --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs @@ -0,0 +1,321 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using JacquardLang.Host.Carrier; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// +/// Core commands are bounded: their exit status, bounded output, and added +/// environment are reported; a timeout or a cancellation kills the command +/// and its process group; a background child is killed when the command +/// exits, and an unrelated process holding the output is not; nothing a +/// descendant holds can hold the caller; and no descriptor is left behind. +/// +[TestFixture] +[Platform("Linux")] +public sealed class CoreCommandTests +{ + private static readonly TimeSpan Minute = TimeSpan.FromMinutes(1); + private string root = string.Empty; + + [SetUp] + public void Prepare() => root = SdkSupport.Scratch("command"); + + [TearDown] + public void CleanUp() + { + CoreCommand.WithoutGroups = false; + CoreCommand.RecordDescendants = true; + CoreCommand.RecordedHook = null; + CoreCommand.StartedHook = null; + CoreCommand.ExitedHook = null; + CoreCommand.KillGroupAtExit = true; + CoreCommand.KillGroupOnStop = true; + foreach (string file in Directory.EnumerateFiles(root, "*.pid")) + { + Support.Kill(int.Parse(File.ReadAllText(file).Trim(), CultureInfo.InvariantCulture)); + } + + SdkSupport.RemoveScratch(root); + } + + private static bool HasSetsid => CoreCommand.Setsid is not null; + + private string PidPath(string name) => Path.Combine(root, name + ".pid"); + + private static CoreCommand.Output Run(string script, TimeSpan? timeout = null, CancellationToken cancellationToken = default, int stdoutLimit = 1 << 20, int stderrLimit = 4096) => + CoreCommand.Run(SdkSupport.Sh(script), null, timeout ?? Minute, stdoutLimit, stderrLimit, cancellationToken); + + [Test] + public void TheExitStatusOutputAndAddedEnvironmentAreReported() + { + CoreCommand.Output output = CoreCommand.Run( + SdkSupport.Sh("echo out; echo err >&2; echo \"$JQ_ADDED\"; exit 3"), + new Dictionary { ["JQ_ADDED"] = "added" }, + Minute, + 1 << 20, + 4096, + default); + Assert.That(output.ExitStatus, Is.EqualTo(3)); + Assert.That(SdkSupport.Text(output.Stdout), Is.EqualTo("out\nadded\n")); + Assert.That(SdkSupport.Text(output.Stderr), Is.EqualTo("err\n")); + Assert.That(output.StdoutOverflow, Is.False); + + // The command's stdin reads end of file at once. + Assert.That(SdkSupport.Text(Run("cat; echo done").Stdout), Is.EqualTo("done\n")); + } + + [Test] + public void OutputBeyondTheLimitsIsReadAndCountedButNotKept() + { + CoreCommand.Output output = Run("head -c 3000000 /dev/zero; head -c 100000 /dev/zero >&2", stdoutLimit: 1000, stderrLimit: 10); + Assert.That(output.ExitStatus, Is.EqualTo(0)); + Assert.That(output.Stdout, Has.Length.EqualTo(1000)); + Assert.That(output.StdoutOverflow, Is.True); + Assert.That(output.Stderr, Has.Length.EqualTo(10)); + } + + [Test] + public void ACommandThatDoesNotExitInTimeIsKilledWithItsChild() + { + CarrierException error = Support.Throws(() => + Run("sleep 300 & echo $! > '" + PidPath("child") + "'; wait", TimeSpan.FromMilliseconds(500)))!; + Assert.That(error.IsTimeout, Is.True, error.Message); + SdkSupport.Dies("the timed-out command's child", SdkSupport.PidFile(PidPath("child"))); + } + + [Test] + public void CancellationKillsTheCommandAndItsChild() + { + using var cancel = new CancellationTokenSource(); + Thread canceller = Support.After(TimeSpan.Zero, () => + { + SdkSupport.PidFile(PidPath("child")); + cancel.Cancel(); + }); + CarrierException error = Support.Throws(() => + Run("sleep 300 & echo $! > '" + PidPath("child") + "'; wait", cancellationToken: cancel.Token))!; + Assert.That(error.IsAborted, Is.True, error.Message); + SdkSupport.Dies("the cancelled command's child", SdkSupport.PidFile(PidPath("child"))); + canceller.Join(); + } + + /// Under setsid the command keeps its pid (no fork) and leads its own process group. + [Test] + public void TheCommandKeepsItsPidAndLeadsItsOwnGroup() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + int started = 0; + CoreCommand.StartedHook = pid => started = pid; + string[] fields = SdkSupport.Text(Run("echo $$; cut -d')' -f2 /proc/$$/stat | cut -d' ' -f4,5").Stdout).Split('\n'); + Assert.That(int.Parse(fields[0], CultureInfo.InvariantCulture), Is.EqualTo(started), "setsid forked"); + string[] ids = fields[1].Split(' '); + Assert.That(ids, Is.EqualTo(new[] { started.ToString(CultureInfo.InvariantCulture), started.ToString(CultureInfo.InvariantCulture) }), "pgid and sid"); + } + + /// + /// sh -c 'sleep 300 & echo $!' exits at once, leaving a child + /// that holds its output; the group kill at the exit ends it, and the + /// command still succeeds, as in Rust. With that kill disabled (a test + /// seam) the child survives: the kill is what ends it. + /// + [Test] + public void ABackgroundChildIsKilledWhenTheCommandExits() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + CoreCommand.Output output = Run("sleep 300 & echo $!"); + Assert.That(output.ExitStatus, Is.EqualTo(0)); + SdkSupport.Dies("the background child", int.Parse(SdkSupport.Text(output.Stdout).Trim(), CultureInfo.InvariantCulture)); + + CoreCommand.KillGroupAtExit = false; + output = Run("sleep 300 & echo $!"); + int survivor = int.Parse(SdkSupport.Text(output.Stdout).Trim(), CultureInfo.InvariantCulture); + File.WriteAllText(PidPath("survivor"), survivor + "\n"); + Assert.That(Support.Alive(survivor), Is.True, "without the exit kill the child should survive"); + } + + /// A child that closed its output and keeps running is in the group, so it is killed at the exit too. + [Test] + public void ABackgroundChildThatClosedItsOutputIsKilled() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + CoreCommand.Output output = Run("sleep 300 /dev/null 2>&1 & echo $! > '" + PidPath("quiet") + "'"); + Assert.That(output.ExitStatus, Is.EqualTo(0)); + SdkSupport.Dies("the quiet child", SdkSupport.PidFile(PidPath("quiet"))); + } + + /// + /// A process outside the command's group that holds the command's + /// output is neither killed nor waited for: the command returns, and the + /// holder lives on. + /// + [Test] + public void AnUnrelatedProcessHoldingTheOutputIsNotKilled() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + Process? holder = null; + CoreCommand.StartedHook = pid => + { + // Started by the test, not the command: it opens the command's stdout pipe through /proc. + holder = Process.Start(new ProcessStartInfo("/bin/sh", new[] { "-c", "exec sleep 300 >>/proc/" + pid.ToString(CultureInfo.InvariantCulture) + "/fd/1" }) + { + UseShellExecute = false, + }); + }; + try + { + CoreCommand.Output output = Support.Within(Minute, () => Run("sleep 0.5; echo done")); + Assert.That(SdkSupport.Text(output.Stdout), Is.EqualTo("done\n")); + Assert.That(holder, Is.Not.Null); + Assert.That(Support.Alive(holder!.Id), Is.True, "an unrelated holder was killed"); + } + finally + { + if (holder is not null) + { + Support.Kill(holder.Id); + holder.WaitForExit(10_000); + holder.Dispose(); + } + } + } + + /// A stop during a long command kills its whole group: a child and a grandchild. + [Test] + public void AStopKillsTheWholeGroup() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + using var cancel = new CancellationTokenSource(); + string grandchild = PidPath("grandchild"); + string child = PidPath("child"); + Thread canceller = Support.After(TimeSpan.Zero, () => + { + SdkSupport.PidFile(grandchild); + cancel.Cancel(); + }); + Support.Throws(() => Run( + "(sleep 300 & echo $! > '" + grandchild + "'; wait) & echo $! > '" + child + "'; wait", + cancellationToken: cancel.Token)); + canceller.Join(); + SdkSupport.Dies("the child", SdkSupport.PidFile(child)); + SdkSupport.Dies("the grandchild", SdkSupport.PidFile(grandchild)); + } + + /// + /// Without a group (a test seam), a background child recorded while the + /// command ran is killed when it exits; with recording off it survives, + /// so the recorded kill is what ends it. + /// + [Test] + public void WithoutGroupsARecordedDescendantIsKilledAtTheExit() + { + CoreCommand.WithoutGroups = true; + var recorded = new List(); + CoreCommand.RecordedHook = pids => + { + lock (recorded) + { + recorded.Clear(); + recorded.AddRange(pids); + } + }; + string script = "sleep 300 /dev/null 2>&1 & echo $! > '" + PidPath("recorded") + "'; sleep 0.3"; + Assert.That(Run(script).ExitStatus, Is.EqualTo(0)); + int pid = SdkSupport.PidFile(PidPath("recorded")); + lock (recorded) + { + Assert.That(recorded, Does.Contain(pid), "the child was never recorded"); + } + + SdkSupport.Dies("the recorded child", pid); + + CoreCommand.RecordDescendants = false; + File.Delete(PidPath("recorded")); + Run(script); + Assert.That(Support.Alive(SdkSupport.PidFile(PidPath("recorded"))), Is.True, "without recording the child should survive"); + } + + [Test] + public void WithoutGroupsAStopKillsTheCommandAndItsChild() + { + CoreCommand.WithoutGroups = true; + CarrierException error = Support.Throws(() => + Run("sleep 300 & echo $! > '" + PidPath("child") + "'; wait", TimeSpan.FromMilliseconds(500)))!; + Assert.That(error.IsTimeout, Is.True); + SdkSupport.Dies("the child", SdkSupport.PidFile(PidPath("child"))); + } + + /// A cancellation that lands during the exit cleanup is reported, not the output. + [Test] + public void AStopDuringTheExitCleanupIsReported() + { + using var cancel = new CancellationTokenSource(); + CoreCommand.ExitedHook = cancel.Cancel; + CarrierException error = Support.Throws(() => Run("echo hi", cancellationToken: cancel.Token))!; + Assert.That(error.IsAborted, Is.True); + } + + [Test] + public void ACommandThatCannotStartIsALoss() + { + CarrierException error = Support.Throws(() => + CoreCommand.Run(new[] { Path.Combine(root, "missing") }, null, Minute, 1024, 1024, default))!; + Assert.That(error.IsLoss && !error.IsTimeout && !error.IsAborted, Is.True); + Assert.That(error.Message, Does.StartWith("could not start ")); + CoreCommand.WithoutGroups = true; + Assert.That(Support.Throws(() => + CoreCommand.Run(new[] { Path.Combine(root, "missing") }, null, Minute, 1024, 1024, default))!.Message, Does.StartWith("could not start ")); + } + + /// An executable setsid cannot exec (its interpreter is missing) is a start failure, not the command's exit status. + [Test] + public void ACommandSetsidCannotExecIsALoss() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + string broken = Path.Combine(root, "broken"); + File.WriteAllText(broken, "#!/no/such/interpreter\n"); + File.SetUnixFileMode(broken, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + CarrierException error = Support.Throws(() => CoreCommand.Run(new[] { broken }, null, Minute, 1024, 1024, default))!; + Assert.That(error.IsLoss, Is.True); + Assert.That(error.Message, Does.StartWith("could not start " + broken + " (setsid: "), error.Message); + } + + /// + /// A descendant that escapes the group and floods the output holds + /// neither the caller nor the heap: the command returns once it exits, + /// with at most the kept limit of output. + /// + [Test] + public void AnEscapedFlooderHoldsNeitherTheCallerNorTheHeap() + { + string flooder = PidPath("flooder"); + CoreCommand.Output output = Support.Within(Minute, () => + Run("setsid sh -c 'echo $$ > \"" + flooder + "\"; exec yes' & sleep 0.2", stdoutLimit: 4096)); + Assert.That(output.Stdout.Length, Is.LessThanOrEqualTo(4096)); + // It is not the host's to kill (it left the group); its next write + // after the host closed the pipe fails, and the test ends it. + Support.Kill(SdkSupport.PidFile(flooder)); + } + + /// Commands release every descriptor they opened by the time they return (counted at once, with no wait a finalizer could satisfy). + [Test] + public void CommandsLeaveNoDescriptorsBehind() + { + Run("true"); + int before = Support.Descriptors(); + for (int i = 0; i < 20; i++) + { + Run("echo $i; echo e >&2"); + Support.Throws(() => Run("sleep 300", TimeSpan.FromMilliseconds(50))); + } + + Assert.That(Support.Descriptors(), Is.EqualTo(before)); + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/SdkSupport.cs b/dotnet/test/JacquardHost.Tests/Sdk/SdkSupport.cs new file mode 100644 index 0000000..1b0eb0f --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/SdkSupport.cs @@ -0,0 +1,99 @@ +using System; +using System.Globalization; +using System.IO; +using System.Text; +using System.Threading; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// Helpers for the SDK tests. +internal static class SdkSupport +{ + /// Waits up to a minute for to hold a complete pid line and returns it. + internal static int PidFile(string path) + { + DateTime deadline = DateTime.UtcNow.AddMinutes(1); + while (true) + { + string text = File.Exists(path) ? File.ReadAllText(path) : string.Empty; + if (text.EndsWith('\n') && text.Trim().Length > 0) + { + return int.Parse(text.Trim().Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)[0], CultureInfo.InvariantCulture); + } + + if (DateTime.UtcNow > deadline) + { + Assert.Fail(path + " was never written"); + } + + Thread.Sleep(5); + } + } + + /// Waits up to ten seconds for to exit, killing it and failing otherwise; the tests' children live for five minutes. + internal static void Dies(string what, int pid) + { + if (!Support.Eventually(() => !Support.Alive(pid), TimeSpan.FromSeconds(10))) + { + Support.Kill(pid); + Assert.Fail(what + " is still running"); + } + } + + /// A shell command line. + internal static string[] Sh(string script) => new[] { "/bin/sh", "-c", script }; + + /// A fresh scratch directory. + internal static string Scratch(string prefix) => Directory.CreateTempSubdirectory("jacquard-dotnet-sdk-" + prefix + "-").FullName; + + /// + /// Removes a scratch directory, first restoring owner permissions on + /// everything in it (tests make some files unreadable). + /// + internal static void RemoveScratch(string root) + { + if (!Directory.Exists(root)) + { + return; + } + + foreach (string entry in Directory.EnumerateFileSystemEntries(root, "*", SearchOption.AllDirectories)) + { + try + { + if (new FileInfo(entry).LinkTarget is null) + { + File.SetUnixFileMode(entry, File.GetUnixFileMode(entry) | UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + } + catch (IOException) + { + // Gone already. + } + } + + Directory.Delete(root, recursive: true); + } + + /// A file of bytes that takes no space (sparse), with . + internal static string Sparse(string path, long length, UnixFileMode mode = UnixFileMode.None) + { + using (var stream = new FileStream(path, FileMode.CreateNew)) + { + stream.SetLength(length); + } + + File.SetUnixFileMode(path, mode); + return path; + } + + internal static void MakeFifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("mkfifo", path) ?? throw new InvalidOperationException("mkfifo"); + Assert.That(mkfifo.WaitForExit(30_000) && mkfifo.ExitCode == 0, "mkfifo failed"); + } + + /// The text of . + internal static string Text(byte[] bytes) => Encoding.UTF8.GetString(bytes); +} From 5f2ad2d484dc029d75ad6a132fbefef209033d6e Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 07:04:33 -0400 Subject: [PATCH 02/52] Add .NET SDK data mapping Codec objects map .NET values to Jacquard values: long (Core's 63-bit Int), int, double, string, bool, Hash, Unit, lists, Option (which nests) and nullable codecs, tuples, protocol Values, records and enumerations and type hierarchies as constructors, and Dynamic by public name. Records are mapped through explicit field readers and a constructor function rather than reflection, so mapping works under trimming and ahead-of-time compilation; Codecs.Lazy supports recursive types. There is no decimal mapping: money crosses as integer minor units. Every encoding is bounded as it goes by the selected limits; nesting is checked before each nested encoder exists, raw values, decoded values, nested unions, lazy chains, and Dynamic equality and text are walked iteratively, and caller exceptions from codecs, field readers, and record constructors are wrapped with the exception kept. --- dotnet/.editorconfig | 4 + dotnet/src/JacquardHost/Sdk/Builtins.cs | 760 ++++++++++++++++++ dotnet/src/JacquardHost/Sdk/Codec.cs | 328 ++++++++ dotnet/src/JacquardHost/Sdk/Dynamic.cs | 358 +++++++++ dotnet/src/JacquardHost/Sdk/Encoder.cs | 375 +++++++++ dotnet/src/JacquardHost/Sdk/ProgramData.cs | 374 +++++++++ dotnet/src/JacquardHost/Sdk/Values.cs | 183 +++++ .../test/JacquardHost.Tests/Sdk/DataTests.cs | 738 +++++++++++++++++ 8 files changed, 3120 insertions(+) create mode 100644 dotnet/src/JacquardHost/Sdk/Builtins.cs create mode 100644 dotnet/src/JacquardHost/Sdk/Codec.cs create mode 100644 dotnet/src/JacquardHost/Sdk/Dynamic.cs create mode 100644 dotnet/src/JacquardHost/Sdk/Encoder.cs create mode 100644 dotnet/src/JacquardHost/Sdk/ProgramData.cs create mode 100644 dotnet/src/JacquardHost/Sdk/Values.cs create mode 100644 dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs diff --git a/dotnet/.editorconfig b/dotnet/.editorconfig index f9fd46f..b08abcc 100644 --- a/dotnet/.editorconfig +++ b/dotnet/.editorconfig @@ -6,3 +6,7 @@ dotnet_diagnostic.CA1036.severity = none # JsonObject is named for JSON's object, not as a collection. dotnet_diagnostic.CA1710.severity = none +# The SDK's types are named for Jacquard's concepts, as in the other SDKs: +# Function, Option, and Int are Jacquard names, not .NET keywords or types. +dotnet_diagnostic.CA1716.severity = none +dotnet_diagnostic.CA1720.severity = none diff --git a/dotnet/src/JacquardHost/Sdk/Builtins.cs b/dotnet/src/JacquardHost/Sdk/Builtins.cs new file mode 100644 index 0000000..7cd2217 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Builtins.cs @@ -0,0 +1,760 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// The codec implementations behind 's properties and factories. +internal static class Builtins +{ + /// One field of a record codec. + internal interface IField + { + string Name { get; } + + Value Encode(Encoder encoder, TRecord record); + + object? Decode(Decoder decoder, Value value); + } + + /// A union, whose cases a flattening walk enters. + internal interface IUnion + { + IReadOnlyList Cases { get; } + } + + /// A lazy codec, which a walk resolves to what it stands for. + internal interface ILazy + { + object Target { get; } + } + + internal static IField[] Fields(params IField[] fields) => fields; + + /// + /// What stands for once every lazy codec on the + /// way is made, in a loop (so a chain of any length cannot exhaust the + /// stack) that refuses a lazy codec standing for itself. + /// + internal static object Resolve(object codec) + { + HashSet? seen = null; + while (codec is ILazy lazy) + { + seen ??= new HashSet(ReferenceEqualityComparer.Instance); + if (!seen.Add(codec)) + { + throw new InvalidOperationException("a lazy codec stands for itself"); + } + + codec = lazy.Target; + } + + return codec; + } + + // Built-in types. + internal sealed class IntCodec : Codec + { + public override Value Encode(Encoder encoder, long value) => encoder.Integer(value); + + public override long Decode(Decoder decoder, Value value) => value is IntValue number ? number.Value : throw decoder.Mapping("expected Int"); + } + + internal sealed class Int32Codec : Codec + { + public override Value Encode(Encoder encoder, int value) => encoder.Integer(value); + + public override int Decode(Decoder decoder, Value value) => + value is IntValue number && number.Value >= int.MinValue && number.Value <= int.MaxValue + ? (int)number.Value + : throw decoder.Mapping("expected Int that fits int"); + } + + internal sealed class RealCodec : Codec + { + public override Value Encode(Encoder encoder, double value) => encoder.Real(value); + + public override double Decode(Decoder decoder, Value value) => value is RealValue real ? real.ToDouble() : throw decoder.Mapping("expected Real"); + } + + internal sealed class TextCodec : Codec + { + public override Value Encode(Encoder encoder, string value) => encoder.Text(value); + + public override string Decode(Decoder decoder, Value value) => value is TextValue text ? text.Value : throw decoder.Mapping("expected Text"); + } + + internal sealed class BoolCodec : Codec + { + public override Value Encode(Encoder encoder, bool value) => encoder.Bool(value); + + public override bool Decode(Decoder decoder, Value value) => + decoder.Data.IsNullary(value, "true") || (decoder.Data.IsNullary(value, "false") ? false : throw decoder.Mapping("expected Bool")); + } + + internal sealed class HashCodec : Codec + { + public override Value Encode(Encoder encoder, Hash value) => encoder.Hash(value); + + public override Hash Decode(Decoder decoder, Value value) => value is HashValue hash ? hash.Hash : throw decoder.Mapping("expected Hash"); + } + + internal sealed class UnitCodec : Codec + { + public override Value Encode(Encoder encoder, Unit value) => encoder.Tuple(Array.Empty()); + + public override Unit Decode(Decoder decoder, Value value) => + value is TupleValue { Items.Count: 0 } ? default : throw decoder.Mapping("expected unit"); + } + + internal sealed class ValueCodec : Codec + { + public override Value Encode(Encoder encoder, Value value) => encoder.Raw(value); + + public override Value Decode(Decoder decoder, Value value) => value; + } + + internal sealed class CustomCodec : Codec + { + private readonly Func encode; + private readonly Func decode; + + internal CustomCodec(Func encode, Func decode) + { + this.encode = encode; + this.decode = decode; + } + + public override Value Encode(Encoder encoder, T value) => encode(encoder, value); + + public override T Decode(Decoder decoder, Value value) => decode(decoder, value); + } + + internal sealed class ListCodec : Codec> + { + private readonly Codec item; + + internal ListCodec(Codec item) + { + this.item = item; + } + + public override Value Encode(Encoder encoder, IReadOnlyList value) => encoder.List(item, value, 0, value.Count); + + public override IReadOnlyList Decode(Decoder decoder, Value value) + { + List items = decoder.Data.ListItems(value); + var output = new T[items.Count]; + for (int i = 0; i < output.Length; i++) + { + output[i] = item.Decode(decoder, items[i]); + } + + return output; + } + } + + /// The value inside Some, null for None, or a mapping failure for anything else. + private static Value? Some(Decoder decoder, Value value) + { + if (decoder.Data.IsNullary(value, "none")) + { + return null; + } + + if (value is ConstructorValue some && some.Arguments.Count == 1 && decoder.Data.Label(some.Identity) == "some") + { + return some.Arguments[0]; + } + + throw decoder.Mapping("expected Option"); + } + + internal sealed class OptionCodec : Codec> + { + private readonly Codec inner; + + internal OptionCodec(Codec inner) + { + this.inner = inner; + } + + public override Value Encode(Encoder encoder, Option value) => + encoder.Option(inner, value.HasValue, value.HasValue ? value.Value : default!); + + public override Option Decode(Decoder decoder, Value value) => + Some(decoder, value) is { } some ? Option.Some(inner.Decode(decoder, some)) : Option.None; + } + + internal sealed class NullableCodec : Codec + where T : class + { + private readonly Codec inner; + + internal NullableCodec(Codec inner) + { + this.inner = inner; + } + + internal override bool AcceptsNull => true; + + public override Value Encode(Encoder encoder, T? value) => encoder.Option(inner, value is not null, value!); + + public override T? Decode(Decoder decoder, Value value) + { + if (Some(decoder, value) is not { } some) + { + return null; + } + + return inner.Decode(decoder, some) ?? throw decoder.Mapping("Some of a value that decodes to null has no nullable form; use Codecs.Option"); + } + } + + internal sealed class NullableValueCodec : Codec + where T : struct + { + private readonly Codec inner; + + internal NullableValueCodec(Codec inner) + { + this.inner = inner; + } + + internal override bool AcceptsNull => true; + + public override Value Encode(Encoder encoder, T? value) => encoder.Option(inner, value.HasValue, value.GetValueOrDefault()); + + public override T? Decode(Decoder decoder, Value value) => Some(decoder, value) is { } some ? inner.Decode(decoder, some) : null; + } + + private static TupleValue Shape(Decoder decoder, Value value, int count) => + value is TupleValue tuple && tuple.Items.Count == count ? tuple : throw decoder.Mapping("tuple shape mismatch"); + + internal sealed class TupleCodec : Codec<(T1, T2)> + { + private readonly Codec first; + private readonly Codec second; + + internal TupleCodec(Codec first, Codec second) + { + this.first = first; + this.second = second; + } + + public override Value Encode(Encoder encoder, (T1, T2) value) + { + encoder.CheckCount(2, "tuple"); + return encoder.Tuple(new[] { encoder.Item(0, first, value.Item1), encoder.Item(1, second, value.Item2) }); + } + + public override (T1, T2) Decode(Decoder decoder, Value value) + { + TupleValue tuple = Shape(decoder, value, 2); + return (first.Decode(decoder, tuple.Items[0]), second.Decode(decoder, tuple.Items[1])); + } + } + + internal sealed class TupleCodec : Codec<(T1, T2, T3)> + { + private readonly Codec first; + private readonly Codec second; + private readonly Codec third; + + internal TupleCodec(Codec first, Codec second, Codec third) + { + this.first = first; + this.second = second; + this.third = third; + } + + public override Value Encode(Encoder encoder, (T1, T2, T3) value) + { + encoder.CheckCount(3, "tuple"); + return encoder.Tuple(new[] + { + encoder.Item(0, first, value.Item1), encoder.Item(1, second, value.Item2), encoder.Item(2, third, value.Item3), + }); + } + + public override (T1, T2, T3) Decode(Decoder decoder, Value value) + { + TupleValue tuple = Shape(decoder, value, 3); + return (first.Decode(decoder, tuple.Items[0]), second.Decode(decoder, tuple.Items[1]), third.Decode(decoder, tuple.Items[2])); + } + } + + internal sealed class TupleCodec : Codec<(T1, T2, T3, T4)> + { + private readonly Codec first; + private readonly Codec second; + private readonly Codec third; + private readonly Codec fourth; + + internal TupleCodec(Codec first, Codec second, Codec third, Codec fourth) + { + this.first = first; + this.second = second; + this.third = third; + this.fourth = fourth; + } + + public override Value Encode(Encoder encoder, (T1, T2, T3, T4) value) + { + encoder.CheckCount(4, "tuple"); + return encoder.Tuple(new[] + { + encoder.Item(0, first, value.Item1), + encoder.Item(1, second, value.Item2), + encoder.Item(2, third, value.Item3), + encoder.Item(3, fourth, value.Item4), + }); + } + + public override (T1, T2, T3, T4) Decode(Decoder decoder, Value value) + { + TupleValue tuple = Shape(decoder, value, 4); + return ( + first.Decode(decoder, tuple.Items[0]), + second.Decode(decoder, tuple.Items[1]), + third.Decode(decoder, tuple.Items[2]), + fourth.Decode(decoder, tuple.Items[3])); + } + } + + internal sealed class DynamicCodec : Codec + { + public override Value Encode(Encoder encoder, Dynamic value) + { + switch (value) + { + case Dynamic.Int number: + return encoder.Integer(number.Value); + case Dynamic.Real real: + return encoder.Real(real.Value); + case Dynamic.Text text: + return encoder.Text(text.Value); + case Dynamic.Bool flag: + return encoder.Bool(flag.Value); + case Dynamic.HashValue hash: + return encoder.Hash(hash.Value); + case Dynamic.Tuple tuple: + { + encoder.CheckCount(tuple.Items.Count, "tuple"); + var items = new Value[tuple.Items.Count]; + for (int i = 0; i < items.Length; i++) + { + items[i] = encoder.Item(i, this, tuple.Items[i]); + } + + return encoder.Tuple(items); + } + + case Dynamic.List list: + return encoder.List(this, list.Items, 0, list.Items.Count); + case Dynamic.Option option: + return encoder.Option(this, option.Value is not null, option.Value!); + case Dynamic.Variant variant: + { + encoder.CheckCount(variant.Fields.Count, "constructor"); + var fields = new Value[variant.Fields.Count]; + for (int i = 0; i < fields.Length; i++) + { + fields[i] = encoder.Field(i.ToString(System.Globalization.CultureInfo.InvariantCulture), this, variant.Fields[i]); + } + + return encoder.Constructor(variant.Name, fields); + } + + default: + throw encoder.Invalid("unknown Dynamic kind " + value.GetType().Name); + } + } + + /// Decodes a value Core sent, or one measured within the depth limit, so the recursion is bounded by that depth. + public override Dynamic Decode(Decoder decoder, Value value) + { + switch (value) + { + case IntValue number: + return new Dynamic.Int(number.Value); + case RealValue real: + return new Dynamic.Real(real.ToDouble()); + case TextValue text: + return new Dynamic.Text(text.Value); + case HashValue hash: + return new Dynamic.HashValue(hash.Hash); + case TupleValue tuple: + { + var items = new Dynamic[tuple.Items.Count]; + for (int i = 0; i < items.Length; i++) + { + items[i] = Decode(decoder, tuple.Items[i]); + } + + return new Dynamic.Tuple(items); + } + + case ConstructorValue constructor: + { + string label = decoder.Data.Label(constructor.Identity) ?? throw decoder.Mapping("unmapped constructor " + constructor.Identity.Hex); + switch (label) + { + case "true" or "false": + return new Dynamic.Bool(Codecs.Bool.Decode(decoder, value)); + case "cons" or "nil": + { + List items = decoder.Data.ListItems(value); + var decoded = new Dynamic[items.Count]; + for (int i = 0; i < decoded.Length; i++) + { + decoded[i] = Decode(decoder, items[i]); + } + + return new Dynamic.List(decoded); + } + + case "some" or "none": + return new Dynamic.Option(Some(decoder, value) is { } inner ? Decode(decoder, inner) : null); + default: + { + var fields = new Dynamic[constructor.Arguments.Count]; + for (int i = 0; i < fields.Length; i++) + { + fields[i] = Decode(decoder, constructor.Arguments[i]); + } + + return new Dynamic.Variant(label, fields); + } + } + } + + default: + throw decoder.Mapping("unknown value kind " + value.GetType().Name); + } + } + } +} + +/// A record (any .NET type) as one Jacquard constructor; made by and its overloads. +/// The record type. +public sealed class RecordCodec : Codec, ICase +{ + private readonly Builtins.IField[] fields; + private readonly Func make; + + internal RecordCodec(string constructor, Builtins.IField[] fields, Func make) + { + Constructor = constructor ?? throw new ArgumentNullException(nameof(constructor)); + foreach (Builtins.IField field in fields) + { + ArgumentNullException.ThrowIfNull(field, nameof(fields)); + } + + this.fields = fields; + this.make = make; + } + + /// The constructor's public name. + public string Constructor { get; } + + Type ICase.CaseType => typeof(TRecord); + + bool ICase.Decodes(Decoder decoder, Value value) => decoder.Is(value, Constructor); + + Value ICase.EncodeCase(Encoder encoder, object value) => Encode(encoder, (TRecord)value); + + object? ICase.DecodeCase(Decoder decoder, Value value) => Decode(decoder, value); + + /// + public override Value Encode(Encoder encoder, TRecord value) + { + ArgumentNullException.ThrowIfNull(encoder); + + // Counted before any field is read. + encoder.CheckCount(fields.Length, "constructor"); + var encoded = new Value[fields.Length]; + for (int i = 0; i < fields.Length; i++) + { + encoded[i] = fields[i].Encode(encoder, value); + } + + return encoder.Constructor(Constructor, encoded); + } + + /// + public override TRecord Decode(Decoder decoder, Value value) + { + ArgumentNullException.ThrowIfNull(decoder); + IReadOnlyList values = decoder.Fields(value, Constructor, fields.Length); + object?[] decoded = new object?[fields.Length]; + for (int i = 0; i < fields.Length; i++) + { + decoded[i] = fields[i].Decode(decoder, values[i]); + } + + try + { + return make(decoded); + } + catch (Exception error) when (Caller.Wraps(error)) + { + throw JacquardException.Binding(typeof(TRecord).Name + " refused the decoded fields: " + Identities.Describe(error), error); + } + } +} + +/// An enumeration as nullary constructors; made by . +/// The enumeration. +public sealed class EnumCodec : Codec, ICase + where TEnum : struct, Enum +{ + private readonly List<(TEnum Member, string Name)> members = new(); + + internal EnumCodec(IReadOnlyDictionary? explicitNames) + { + var seen = new HashSet(); + foreach (TEnum member in Enum.GetValues()) + { + if (!seen.Add(member)) + { + continue; + } + + string? name = explicitNames is null ? Enum.GetName(member) : explicitNames.TryGetValue(member, out string? given) ? given : null; + if (name is null) + { + throw new ArgumentException(typeof(TEnum).Name + "." + member + " has no constructor name", nameof(explicitNames)); + } + + members.Add((member, name)); + } + } + + Type ICase.CaseType => typeof(TEnum); + + bool ICase.Decodes(Decoder decoder, Value value) + { + foreach ((TEnum _, string name) in members) + { + if (decoder.Is(value, name)) + { + return true; + } + } + + return false; + } + + Value ICase.EncodeCase(Encoder encoder, object value) => Encode(encoder, (TEnum)value); + + object? ICase.DecodeCase(Decoder decoder, Value value) => Decode(decoder, value); + + /// + public override Value Encode(Encoder encoder, TEnum value) + { + ArgumentNullException.ThrowIfNull(encoder); + foreach ((TEnum member, string name) in members) + { + if (EqualityComparer.Default.Equals(member, value)) + { + return encoder.Constructor(name, Array.Empty()); + } + } + + throw encoder.Invalid(typeof(TEnum).Name + " has no constructor for " + value); + } + + /// + public override TEnum Decode(Decoder decoder, Value value) + { + ArgumentNullException.ThrowIfNull(decoder); + foreach ((TEnum member, string name) in members) + { + if (decoder.Is(value, name)) + { + decoder.Fields(value, name, 0); + return member; + } + } + + throw decoder.Mismatch(typeof(TEnum).Name); + } +} + +/// The constructors of one data type as a .NET type hierarchy; made by . +/// The common .NET type of the cases. +public sealed class UnionCodec : Codec, ICase, Builtins.IUnion +{ + private readonly ICase[] cases; + + /// + /// The distinct non-union cases, nested unions flattened in order, so + /// decoding asks each once. Computed on first use, when every lazy case of + /// a recursive hierarchy can be made, by one iterative walk over the whole + /// case graph; published once complete, so a race computes the same list + /// twice. + /// + private volatile ICase[]? leaves; + + internal UnionCodec(IUnionCase[] given) + { + cases = new ICase[given.Length]; + for (int i = 0; i < given.Length; i++) + { + if (given[i] is not ICase known) + { + throw new ArgumentException("a union case must be a record, enumeration, union, or lazy codec", nameof(given)); + } + + // A lazy case is checked when it is made, on first use. + if (known is not Builtins.ILazy && !typeof(TUnion).IsAssignableFrom(known.CaseType)) + { + throw new ArgumentException(known.CaseType.Name + " is not a " + typeof(TUnion).Name, nameof(given)); + } + + cases[i] = known; + } + } + + IReadOnlyList Builtins.IUnion.Cases => cases; + + Type ICase.CaseType => typeof(TUnion); + + /// Test seam: how many distinct cases this union decodes through. + internal int LeafCount => Leaves().Length; + + private ICase[] Leaves() + { + ICase[]? known = leaves; + if (known is null) + { + var flat = new List(); + var seen = new HashSet(ReferenceEqualityComparer.Instance) { this }; + var stack = new Stack>(); + stack.Push(((IEnumerable)cases).GetEnumerator()); + while (stack.Count > 0) + { + IEnumerator top = stack.Peek(); + if (!top.MoveNext()) + { + stack.Pop().Dispose(); + continue; + } + + object each = Builtins.Resolve(top.Current); + if (!seen.Add(each)) + { + continue; + } + + if (each is Builtins.IUnion nested) + { + stack.Push(nested.Cases.GetEnumerator()); + } + else if (each is ICase leaf && typeof(TUnion).IsAssignableFrom(leaf.CaseType)) + { + flat.Add(leaf); + } + else + { + throw new InvalidOperationException("a case of the " + typeof(TUnion).Name + " union is not a record or enumeration of it"); + } + } + + known = flat.ToArray(); + leaves = known; + } + + return known; + } + + bool ICase.Decodes(Decoder decoder, Value value) + { + foreach (ICase each in Leaves()) + { + if (each.Decodes(decoder, value)) + { + return true; + } + } + + return false; + } + + Value ICase.EncodeCase(Encoder encoder, object value) => Encode(encoder, (TUnion)value); + + object? ICase.DecodeCase(Decoder decoder, Value value) => Decode(decoder, value); + + /// + /// Encodes with the first flattened case, in the order decoding asks + /// them, whose type has: no call per nested + /// layer, and the cost is the number of distinct cases, not their nesting. + /// + public override Value Encode(Encoder encoder, TUnion value) + { + ArgumentNullException.ThrowIfNull(encoder); + object boxed = value!; + foreach (ICase each in Leaves()) + { + if (each.CaseType.IsInstanceOfType(boxed)) + { + return each.EncodeCase(encoder, boxed); + } + } + + throw encoder.Invalid(typeof(TUnion).Name + " has no case for " + boxed.GetType().Name); + } + + /// + public override TUnion Decode(Decoder decoder, Value value) + { + ArgumentNullException.ThrowIfNull(decoder); + foreach (ICase each in Leaves()) + { + if (each.Decodes(decoder, value)) + { + return (TUnion)each.DecodeCase(decoder, value)!; + } + } + + throw decoder.Mismatch(typeof(TUnion).Name); + } +} + +/// A codec made on first use, for recursive types; made by . +/// The .NET type. +public sealed class LazyCodec : Codec, ICase, Builtins.ILazy +{ + private readonly Lazy> target; + + internal LazyCodec(Func> make) + { + target = new Lazy>(() => make() ?? throw new InvalidOperationException("a lazy codec's factory returned null"), LazyThreadSafetyMode.ExecutionAndPublication); + } + + object Builtins.ILazy.Target => target.Value; + + internal override bool AcceptsNull => Resolved().AcceptsNull; + + Type ICase.CaseType => Case().CaseType; + + private Codec Resolved() => (Codec)Builtins.Resolve(this); + + private ICase Case() => Builtins.Resolve(this) as ICase + ?? throw new InvalidOperationException("a lazy union case stands for a codec that is not a record, enumeration, or union"); + + bool ICase.Decodes(Decoder decoder, Value value) => Case().Decodes(decoder, value); + + Value ICase.EncodeCase(Encoder encoder, object value) => Case().EncodeCase(encoder, value); + + object? ICase.DecodeCase(Decoder decoder, Value value) => Case().DecodeCase(decoder, value); + + /// + public override Value Encode(Encoder encoder, T value) => Resolved().Encode(encoder, value); + + /// + public override T Decode(Decoder decoder, Value value) => Resolved().Decode(decoder, value); +} diff --git a/dotnet/src/JacquardHost/Sdk/Codec.cs b/dotnet/src/JacquardHost/Sdk/Codec.cs new file mode 100644 index 0000000..5330181 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Codec.cs @@ -0,0 +1,328 @@ +using System; +using System.Collections.Generic; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// Maps one .NET type to Jacquard values and back. Codecs are explicit, +/// immutable, and thread-safe; they hold no program state, and encode and +/// decode against the of the program at hand. Build +/// them with the factories on , or derive from this class +/// for a custom mapping (use the 's methods so the +/// encoding is bounded as it goes, and report a value that does not decode +/// with ). +/// +/// The .NET type. +public abstract class Codec +{ + /// Encodes ; names its location for errors and bounds it by the selected limits. + /// An invalid argument. + public abstract Value Encode(Encoder encoder, T value); + + /// Decodes . + /// A binding failure when the value does not map to . + public abstract T Decode(Decoder decoder, Value value); + + /// as a function argument encoded with this codec. + public Argument Arg(T value) => Argument.Of(this, value); + + /// Whether null is a value of this codec (unit and nullable codecs); otherwise a null is refused before the codec runs. + internal virtual bool AcceptsNull => false; +} + +/// +/// A codec that can be a case of a union (): +/// a record, an enumeration, a union, or a lazy codec standing for one. +/// +public interface IUnionCase +{ +} + +/// What a union asks of each of its cases. +internal interface ICase : IUnionCase +{ + /// The .NET type this case encodes. + Type CaseType { get; } + + /// Whether is built by one of this case's constructors. + bool Decodes(Decoder decoder, Value value); + + Value EncodeCase(Encoder encoder, object value); + + object? DecodeCase(Decoder decoder, Value value); +} + +/// +/// One field of a record: its name (for error locations), its codec, and how +/// to read it from a record. Made by . +/// +/// The record type. +/// The field type. +public sealed class Field : Builtins.IField +{ + private readonly Func read; + + internal Field(string name, Codec codec, Func read) + { + Name = name; + Codec = codec; + this.read = read; + } + + /// The field's name, used in error locations. + public string Name { get; } + + /// The field's codec. + public Codec Codec { get; } + + /// Encodes the field of , reporting a reader that throws as an invalid argument with its exception kept. + internal Value Encode(Encoder encoder, TRecord record) + { + TField value; + try + { + value = read(record); + } + catch (Exception error) when (Caller.Wraps(error)) + { + throw encoder.Invalid("cannot read " + typeof(TRecord).Name + "." + Identities.Excerpt(Name) + ": " + Identities.Describe(error), error); + } + + return encoder.Field(Name, Codec, value); + } + + string Builtins.IField.Name => Name; + + Value Builtins.IField.Encode(Encoder encoder, TRecord record) => Encode(encoder, record); + + object? Builtins.IField.Decode(Decoder decoder, Value value) => Codec.Decode(decoder, value); +} + +/// +/// The codec factories: built-in types, lists, options, tuples, records, +/// enumerations, unions, lazy (recursive) codecs, raw protocol values, and +/// . Records are mapped by explicit field readers and +/// a constructor function rather than by reflection, so mapping works under +/// trimming and ahead-of-time compilation. +/// +public static class Codecs +{ + /// Int as a , Core's full 63-bit range; a outside it is refused. + public static Codec Int { get; } = new Builtins.IntCodec(); + + /// Int as an ; a result that does not fit an does not decode. + public static Codec Int32 { get; } = new Builtins.Int32Codec(); + + /// Real as the exact binary64 bits of a , including NaN payloads and signed zero. + public static Codec Real { get; } = new Builtins.RealCodec(); + + /// Text as a ; a lone surrogate is refused. + public static Codec Text { get; } = new Builtins.TextCodec(); + + /// Bool as a . + public static Codec Bool { get; } = new Builtins.BoolCodec(); + + /// Hash as a protocol . + public static Codec Hash { get; } = new Builtins.HashCodec(); + + /// Unit, the empty tuple, as . + public static Codec Unit { get; } = new Builtins.UnitCodec(); + + /// A raw protocol , passed through unchanged (within the same bounds). + public static Codec Value { get; } = new Builtins.ValueCodec(); + + /// Any value by public name, as . + public static Codec Dynamic { get; } = new Builtins.DynamicCodec(); + + /// List T as a read-only list; decoding returns a new array. + public static Codec> List(Codec item) => new Builtins.ListCodec(item ?? throw new ArgumentNullException(nameof(item))); + + /// Option T as an , which nests. + public static Codec> Option(Codec inner) => new Builtins.OptionCodec(inner ?? throw new ArgumentNullException(nameof(inner))); + + /// Option T as a nullable reference: null is None. Nested options need . + public static Codec Nullable(Codec inner) + where T : class => new Builtins.NullableCodec(inner ?? throw new ArgumentNullException(nameof(inner))); + + /// Option T as a nullable value type: null is None. + public static Codec NullableValue(Codec inner) + where T : struct => new Builtins.NullableValueCodec(inner ?? throw new ArgumentNullException(nameof(inner))); + + /// A two-item tuple. + public static Codec<(T1, T2)> Tuple(Codec first, Codec second) => + new Builtins.TupleCodec(first ?? throw new ArgumentNullException(nameof(first)), second ?? throw new ArgumentNullException(nameof(second))); + + /// A three-item tuple. + public static Codec<(T1, T2, T3)> Tuple(Codec first, Codec second, Codec third) => + new Builtins.TupleCodec( + first ?? throw new ArgumentNullException(nameof(first)), + second ?? throw new ArgumentNullException(nameof(second)), + third ?? throw new ArgumentNullException(nameof(third))); + + /// A four-item tuple. + public static Codec<(T1, T2, T3, T4)> Tuple(Codec first, Codec second, Codec third, Codec fourth) => + new Builtins.TupleCodec( + first ?? throw new ArgumentNullException(nameof(first)), + second ?? throw new ArgumentNullException(nameof(second)), + third ?? throw new ArgumentNullException(nameof(third)), + fourth ?? throw new ArgumentNullException(nameof(fourth))); + + /// A record field: its name (for error locations), its codec, and how to read it. + public static Field Field(string name, Codec codec, Func read) => + new(name ?? throw new ArgumentNullException(nameof(name)), codec ?? throw new ArgumentNullException(nameof(codec)), read ?? throw new ArgumentNullException(nameof(read))); + + /// + /// A record (any .NET type) as one Jacquard constructor with no fields, + /// such as UnknownSku. The constructor is named by public name: + /// display spelling (UnknownSku) or store spelling (unknown-sku). + /// + public static RecordCodec Record(string constructor, Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Array.Empty>(), _ => make()); + } + + /// A record as one constructor with one field; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1), a => make((T1)a[0]!)); + } + + /// A record as one constructor with two fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2), a => make((T1)a[0]!, (T2)a[1]!)); + } + + /// A record as one constructor with three fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Field f3, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2, f3), a => make((T1)a[0]!, (T2)a[1]!, (T3)a[2]!)); + } + + /// A record as one constructor with four fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Field f3, + Field f4, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2, f3, f4), a => make((T1)a[0]!, (T2)a[1]!, (T3)a[2]!, (T4)a[3]!)); + } + + /// A record as one constructor with five fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Field f3, + Field f4, + Field f5, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2, f3, f4, f5), a => make((T1)a[0]!, (T2)a[1]!, (T3)a[2]!, (T4)a[3]!, (T5)a[4]!)); + } + + /// A record as one constructor with six fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Field f3, + Field f4, + Field f5, + Field f6, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2, f3, f4, f5, f6), a => make((T1)a[0]!, (T2)a[1]!, (T3)a[2]!, (T4)a[3]!, (T5)a[4]!, (T6)a[5]!)); + } + + /// A record as one constructor with seven fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Field f3, + Field f4, + Field f5, + Field f6, + Field f7, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2, f3, f4, f5, f6, f7), a => make((T1)a[0]!, (T2)a[1]!, (T3)a[2]!, (T4)a[3]!, (T5)a[4]!, (T6)a[5]!, (T7)a[6]!)); + } + + /// A record as one constructor with eight fields, in order; each field's codec decodes before runs. + public static RecordCodec Record( + string constructor, + Field f1, + Field f2, + Field f3, + Field f4, + Field f5, + Field f6, + Field f7, + Field f8, + Func make) + { + ArgumentNullException.ThrowIfNull(make); + return new(constructor, Builtins.Fields(f1, f2, f3, f4, f5, f6, f7, f8), a => make((T1)a[0]!, (T2)a[1]!, (T3)a[2]!, (T4)a[3]!, (T5)a[4]!, (T6)a[5]!, (T7)a[6]!, (T8)a[7]!)); + } + + /// + /// An enumeration as nullary constructors named by convention: a member + /// such as PageNow is the constructor PageNow (store + /// spelling page-now). + /// + public static EnumCodec Enum() + where TEnum : struct, Enum => new(null); + + /// An enumeration with an explicit constructor name for every member. + /// A member has no name. + public static EnumCodec Enum(IReadOnlyDictionary constructors) + where TEnum : struct, Enum => new(constructors ?? throw new ArgumentNullException(nameof(constructors))); + + /// + /// The constructors of one data type as a .NET type hierarchy: each case + /// is a record, enumeration, or union codec (or a lazy one standing for + /// one) whose .NET type derives from . + /// Encoding takes the first case, in order with nested unions flattened, + /// whose type the value has; decoding takes the first whose constructor + /// built the value. + /// + /// A case is none of these. + public static UnionCodec Union(params IUnionCase[] cases) => new(cases ?? throw new ArgumentNullException(nameof(cases))); + + /// + /// A codec made on first use by , for recursive + /// types: a field of a type can refer to the type's own codec through a + /// lazy one. The codec is made once, thread-safely. + /// + public static LazyCodec Lazy(Func> make) => new(make ?? throw new ArgumentNullException(nameof(make))); + + /// A custom codec from two functions. + public static Codec Create(Func encode, Func decode) => + new Builtins.CustomCodec(encode ?? throw new ArgumentNullException(nameof(encode)), decode ?? throw new ArgumentNullException(nameof(decode))); +} diff --git a/dotnet/src/JacquardHost/Sdk/Dynamic.cs b/dotnet/src/JacquardHost/Sdk/Dynamic.cs new file mode 100644 index 0000000..043bd60 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Dynamic.cs @@ -0,0 +1,358 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Globalization; +using System.Text; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// A native value with no application type: Jacquard data by public name. +/// Decode any result into it with or +/// . Equality is structural (a +/// compares its exact bits), and both it and +/// walk a value with an explicit stack, so a deep +/// value cannot exhaust the call stack. +/// +public abstract class Dynamic : IEquatable +{ + private const int RenderLimit = 4096; + + private protected Dynamic() + { + } + + /// Unit, the empty tuple. + public static Tuple Unit { get; } = new(Array.Empty()); + + /// The child values, in order: items, fields, or an option's value. + private protected virtual IReadOnlyList Children => Array.Empty(); + + /// An Int. + public static Dynamic Of(long value) => new Int(value); + + /// A Real. + public static Dynamic Of(double value) => new Real(value); + + /// A Text. + public static Dynamic Of(string value) => new Text(value); + + /// A Bool. + public static Dynamic Of(bool value) => new Bool(value); + + /// A List. + /// More items than the v0 hard limit. + public static Dynamic ListOf(params Dynamic[] items) => new List(items); + + /// A tuple. + /// More items than the v0 hard limit. + public static Dynamic TupleOf(params Dynamic[] items) => new Tuple(items); + + /// Some(value). + public static Dynamic Some(Dynamic value) => new Option(value ?? throw new ArgumentNullException(nameof(value))); + + /// None. + public static Dynamic None() => new Option(null); + + /// A constructor value by public name; also checks that the constructor exists. + /// More fields than the v0 hard limit. + public static Dynamic VariantOf(string name, params Dynamic[] fields) => new Variant(name, fields); + + /// + /// A read-only copy of , refused with an + /// when it holds more items than any v0 + /// frame admits: counted first when the collection says its count, and + /// otherwise enumerated at most one item past the limit. + /// + private static ReadOnlyCollection Limited(IEnumerable items, string what) + { + ArgumentNullException.ThrowIfNull(items); + int limit = Limits.Hard.MaxCollectionItems; + int? known = items switch + { + IReadOnlyCollection sized => sized.Count, + ICollection sized => sized.Count, + _ => null, + }; + if (known > limit) + { + throw new ArgumentException(what + " of " + known.Value.ToString(CultureInfo.InvariantCulture) + + " items exceeds the v0 hard limit of " + limit.ToString(CultureInfo.InvariantCulture), nameof(items)); + } + + var copy = new List(known ?? 0); + foreach (Dynamic item in items) + { + if (copy.Count == limit) + { + throw new ArgumentException(what + " exceeds the v0 hard limit of " + limit.ToString(CultureInfo.InvariantCulture) + " items", nameof(items)); + } + + copy.Add(item ?? throw new ArgumentNullException(nameof(items), what + " holds null")); + } + + return copy.AsReadOnly(); + } + + /// Whether this node, apart from its children, equals . + private protected abstract bool SameNode(Dynamic other); + + /// This node's hash, apart from its children. + private protected abstract int NodeHash(); + + /// This node's text before its children, such as Variant(Job. + private protected abstract string Open(); + + /// + public bool Equals(Dynamic? other) + { + if (other is null) + { + return false; + } + + var stack = new Stack<(Dynamic Left, Dynamic Right)>(); + stack.Push((this, other)); + while (stack.Count > 0) + { + (Dynamic left, Dynamic right) = stack.Pop(); + if (ReferenceEquals(left, right)) + { + continue; + } + + if (left.GetType() != right.GetType() || !left.SameNode(right)) + { + return false; + } + + IReadOnlyList a = left.Children; + IReadOnlyList b = right.Children; + if (a.Count != b.Count) + { + return false; + } + + for (int i = 0; i < a.Count; i++) + { + stack.Push((a[i], b[i])); + } + } + + return true; + } + + /// + public override bool Equals(object? obj) => obj is Dynamic other && Equals(other); + + /// A hash of this node and its children's count; equal values hash alike. + public override int GetHashCode() => HashCode.Combine(GetType(), NodeHash(), Children.Count); + + /// The value as text, cut at 4,096 chars. + public override string ToString() + { + var output = new StringBuilder(); + var stack = new Stack(); + stack.Push(this); + while (stack.Count > 0 && output.Length <= RenderLimit) + { + object next = stack.Pop(); + if (next is string text) + { + output.Append(text); + continue; + } + + var node = (Dynamic)next; + output.Append(node.Open()); + IReadOnlyList children = node.Children; + stack.Push(")"); + for (int i = children.Count - 1; i >= 0; i--) + { + stack.Push(children[i]); + if (i > 0 || node is Variant) + { + stack.Push(", "); + } + } + } + + return output.Length > RenderLimit ? output.ToString(0, RenderLimit) + "..." : output.ToString(); + } + + /// A Jacquard Int. + /// The integer. + public sealed class Int(long Value) : Dynamic + { + /// The integer. + public long Value { get; } = Value; + + private protected override bool SameNode(Dynamic other) => other is Int value && value.Value == Value; + + private protected override int NodeHash() => Value.GetHashCode(); + + private protected override string Open() => "Int(" + Value.ToString(CultureInfo.InvariantCulture); + } + + /// A Jacquard Real; equality compares the exact bits. + /// The double. + public sealed class Real(double Value) : Dynamic + { + /// The double. + public double Value { get; } = Value; + + private protected override bool SameNode(Dynamic other) => + other is Real value && BitConverter.DoubleToInt64Bits(value.Value) == BitConverter.DoubleToInt64Bits(Value); + + private protected override int NodeHash() => BitConverter.DoubleToInt64Bits(Value).GetHashCode(); + + private protected override string Open() => "Real(" + Value.ToString("R", CultureInfo.InvariantCulture); + } + + /// A Jacquard Text. + public sealed class Text : Dynamic + { + /// A text value. + public Text(string value) + { + Value = value ?? throw new ArgumentNullException(nameof(value)); + } + + /// The string. + public string Value { get; } + + private protected override bool SameNode(Dynamic other) => other is Text value && string.Equals(value.Value, Value, StringComparison.Ordinal); + + private protected override int NodeHash() => StringComparer.Ordinal.GetHashCode(Value); + + private protected override string Open() => "Text(" + Identities.Excerpt(Value); + } + + /// A Jacquard Bool. + /// The boolean. + public sealed class Bool(bool Value) : Dynamic + { + /// The boolean. + public bool Value { get; } = Value; + + private protected override bool SameNode(Dynamic other) => other is Bool value && value.Value == Value; + + private protected override int NodeHash() => Value ? 1 : 2; + + private protected override string Open() => Value ? "Bool(true" : "Bool(false"; + } + + /// A Jacquard Hash. + public sealed class HashValue : Dynamic + { + /// A hash value. + public HashValue(Hash value) + { + Value = value ?? throw new ArgumentNullException(nameof(value)); + } + + /// The hash. + public Hash Value { get; } + + private protected override bool SameNode(Dynamic other) => other is HashValue value && value.Value.Equals(Value); + + private protected override int NodeHash() => Value.GetHashCode(); + + private protected override string Open() => "Hash(" + Value.Hex; + } + + /// A positional product; the empty tuple is unit. + public sealed class Tuple : Dynamic + { + /// A tuple of a copy of . + /// More items than the v0 hard limit. + public Tuple(IEnumerable items) + { + Items = Limited(items, "tuple"); + } + + /// The items. + public IReadOnlyList Items { get; } + + private protected override IReadOnlyList Children => Items; + + private protected override bool SameNode(Dynamic other) => true; + + private protected override int NodeHash() => 3; + + private protected override string Open() => "Tuple("; + } + + /// A Jacquard List. + public sealed class List : Dynamic + { + /// A list of a copy of . + /// More items than the v0 hard limit. + public List(IEnumerable items) + { + Items = Limited(items, "list"); + } + + /// The items. + public IReadOnlyList Items { get; } + + private protected override IReadOnlyList Children => Items; + + private protected override bool SameNode(Dynamic other) => true; + + private protected override int NodeHash() => 4; + + private protected override string Open() => "List("; + } + + /// A Jacquard Option: is null for None. + public sealed class Option : Dynamic + { + /// Some(value), or None for null. + public Option(Dynamic? value) + { + Value = value; + } + + /// The value of Some, or null for None. + public Dynamic? Value { get; } + + private protected override IReadOnlyList Children => Value is null ? Array.Empty() : new[] { Value }; + + private protected override bool SameNode(Dynamic other) => other is Option option && (option.Value is null) == (Value is null); + + private protected override int NodeHash() => Value is null ? 5 : 6; + + private protected override string Open() => Value is null ? "None(" : "Some("; + } + + /// + /// A constructor by public name (store spelling such as stock, or + /// display spelling such as Stock) and its positional fields. + /// + public sealed class Variant : Dynamic + { + /// A variant of a copy of . + /// More fields than the v0 hard limit. + public Variant(string name, IEnumerable fields) + { + Name = name ?? throw new ArgumentNullException(nameof(name)); + Fields = Limited(fields, "constructor"); + } + + /// The constructor's public name. + public string Name { get; } + + /// The positional fields. + public IReadOnlyList Fields { get; } + + private protected override IReadOnlyList Children => Fields; + + private protected override bool SameNode(Dynamic other) => other is Variant variant && string.Equals(variant.Name, Name, StringComparison.Ordinal); + + private protected override int NodeHash() => StringComparer.Ordinal.GetHashCode(Name); + + private protected override string Open() => "Variant(" + Identities.Excerpt(Name); + } +} diff --git a/dotnet/src/JacquardHost/Sdk/Encoder.cs b/dotnet/src/JacquardHost/Sdk/Encoder.cs new file mode 100644 index 0000000..e2ec488 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Encoder.cs @@ -0,0 +1,375 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// Encodes one value at a location. Every encoding is bounded by the +/// selected limits as it goes, so no value, however it is built (one long +/// text referenced many times, say), costs more than about one frame of work +/// and memory: the JSON depth Core's wire form will need, +/// max_collection_items for each list, tuple, and constructor, +/// max_value_nodes values, and max_text_bytes for each text (in +/// UTF-8 bytes) and max_frame_bytes for all of them together. A text +/// longer in chars than either is refused before it is scanned. Nesting is +/// checked before each nested encoder is made, so no value or codec nesting +/// can recurse past the depth limit. +/// +/// +/// A custom codec may return a it built itself rather +/// than through an encoder. Such a value is not unchecked: before its type is +/// checked, and so before any worker starts, a function call measures every +/// argument value again under one shared budget (nodes, JSON depth, +/// collection items, and text bytes, alone and together), whatever codec +/// built it; the invoke frame is then measured as it is rendered, and the +/// host measures a handler's result the same way before sending it. A value +/// returned by is not measured again. +/// +public sealed class Encoder +{ + private readonly int depth; + private readonly Budget budget; + + private Encoder(ProgramData data, string path, int depth, Budget budget) + { + Data = data; + Path = path; + this.depth = depth; + this.budget = budget; + } + + /// The program's data. + public ProgramData Data { get; } + + /// The location being encoded, such as triage.batch.argument[1][0].service. + public string Path { get; } + + /// What one encoding has used, shared by every location in it. + internal sealed class Budget + { + internal int Nodes { get; set; } + + internal long TextBytes { get; set; } + } + + /// A root encoder at , sharing (as one frame's arguments do) or with a fresh one. + internal static Encoder Root(ProgramData data, string path, Budget? budget = null) + { + var encoder = new Encoder(data, path, 3, budget ?? new Budget()); + encoder.CheckDepth(encoder.depth); + return encoder; + } + + /// An invalid-argument failure at this location. + public JacquardException Invalid(string message) => JacquardException.InvalidAt(Path, message); + + /// An invalid-argument failure at this location, caused by . + internal JacquardException Invalid(string message, Exception cause) => JacquardException.InvalidAt(Path, message, cause); + + private void CheckDepth(long needed) + { + if (needed > Data.Limits.MaxJsonDepth) + { + throw Invalid("value exceeds the v0 JSON depth; use smaller independent batches"); + } + } + + private Encoder Child(string segment, int extra) + { + // Checked before the child exists, so nesting stops at the depth limit. + CheckDepth((long)depth + extra); + return new Encoder(Data, Path + segment, depth + extra, budget); + } + + /// Charges one value node. + private Value Node(Value value) + { + if (budget.Nodes >= Data.Limits.MaxValueNodes) + { + throw Invalid("value exceeds max_value_nodes; use smaller independent batches"); + } + + budget.Nodes++; + return value; + } + + /// Encodes with here. + internal Value Encode(Codec codec, T value) + { + ArgumentNullException.ThrowIfNull(codec); + if (value is null && !codec.AcceptsNull) + { + throw Invalid("null has no Jacquard value; use Option.None (or a nullable codec) for None, and Unit.Value for unit"); + } + + return codec.Encode(this, value); + } + + /// Encodes a named field of a record or variant one level deeper. + /// An invalid argument. + public Value Field(string name, Codec codec, T value) + { + ArgumentNullException.ThrowIfNull(name); + return Child("." + Identities.Excerpt(name), 2).Encode(codec, value); + } + + /// Encodes a positional item of a tuple one level deeper. + /// An invalid argument. + public Value Item(int index, Codec codec, T value) => + Child("[" + index.ToString(CultureInfo.InvariantCulture) + "]", 2).Encode(codec, value); + + /// + /// A constructor value by public name, such as "Job" or "job", + /// of fields already encoded (through ). Bool, + /// List, and Option constructors are refused: use , + /// , and . + /// + /// An invalid argument: an unknown constructor, or too many fields. + public Value Constructor(string name, IReadOnlyList fields) + { + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(fields); + + // Counted before the fields are copied into the value. + CheckCount(fields.Count, "constructor"); + if (ProgramData.Builtins.Contains(Identities.DisplayToStore(name))) + { + throw Invalid("use the native form of builtin constructor " + Identities.Excerpt(name)); + } + + return Builtin(name, fields); + } + + private Value Builtin(string name, IReadOnlyList fields) + { + Hash identity = Data.ConstructorId(name) ?? throw Invalid("unknown constructor " + Identities.Excerpt(name)); + return Node(new ConstructorValue(identity, fields)); + } + + /// A tuple of already encoded items. + internal Value Tuple(IReadOnlyList items) => Node(new TupleValue(items)); + + /// Refuses more than max_collection_items items, before any is encoded. + internal void CheckCount(int count, string what) + { + if (count > Data.Limits.MaxCollectionItems) + { + throw Invalid(what + " exceeds max_collection_items"); + } + } + + internal Value Bool(bool value) => Builtin(value ? "true" : "false", Array.Empty()); + + internal Value Integer(long value) + { + if (value < Json.IntMin || value > Json.IntMax) + { + throw Invalid("integer is outside Core's 63-bit Int range"); + } + + return Node(new IntValue(value)); + } + + internal Value Real(double value) => Node(RealValue.Of(value)); + + internal Value Text(string text) + { + ChargeText(text); + return Node(new TextValue(text)); + } + + /// Charges a text by its UTF-8 bytes, refusing before it is scanned when its length in chars already exceeds a limit. + private void ChargeText(string text) + { + int limit = Data.Limits.MaxTextBytes; + + // A text's UTF-8 length is at least its length in chars, so these + // refuse before it is scanned. + if (text.Length > limit) + { + throw Invalid("text exceeds max_text_bytes"); + } + + if (budget.TextBytes + text.Length > Data.Limits.MaxFrameBytes) + { + throw Invalid("value exceeds max_frame_bytes; use smaller independent batches"); + } + + if (!JsonCodec.IsScalar(text)) + { + throw Invalid("text contains a lone surrogate, which has no Jacquard encoding"); + } + + long length = JsonCodec.Utf8Length(text); + if (length > limit) + { + throw Invalid("text exceeds max_text_bytes"); + } + + budget.TextBytes += length; + if (budget.TextBytes > Data.Limits.MaxFrameBytes) + { + throw Invalid("value exceeds max_frame_bytes; use smaller independent batches"); + } + } + + internal Value Hash(Hash hash) => Node(new HashValue(hash)); + + /// + /// A List of items of + /// from : one cons per item, nested on the + /// wire. The count is checked before any item is read. + /// + internal Value List(Codec codec, IReadOnlyList items, int start, int count) + { + CheckCount(count, "list"); + + // A linked list nests one constructor per element on the wire. + CheckDepth((long)depth + (2L * count)); + var encoded = new Value[count]; + for (int index = count - 1; index >= 0; index--) + { + encoded[index] = Child("[" + index.ToString(CultureInfo.InvariantCulture) + "]", 2 * (index + 1)).Encode(codec, items[start + index]); + } + + Value result = Builtin("nil", Array.Empty()); + for (int index = count - 1; index >= 0; index--) + { + result = Builtin("cons", new[] { encoded[index], result }); + } + + return result; + } + + /// None, or Some of one level deeper. + internal Value Option(Codec codec, bool present, T value) + { + if (!present) + { + return Builtin("none", Array.Empty()); + } + + Value inner = Child(string.Empty, 2).Encode(codec, value); + return Builtin("some", new[] { inner }); + } + + /// + /// Charges a raw boundary value passed through unchanged, walking it + /// within the same bounds; a value shared many times is counted each time + /// it occurs. The walk keeps an explicit stack, and the depth is checked + /// before each level is entered. + /// + internal Value Raw(Value value) + { + ArgumentNullException.ThrowIfNull(value); + var stack = new Stack<(Value Value, int Level)>(); + stack.Push((value, 0)); + while (stack.Count > 0) + { + (Value current, int level) = stack.Pop(); + if (current is null) + { + throw Invalid("null is not a boundary value"); + } + + if ((long)depth + (2L * level) > Data.Limits.MaxJsonDepth) + { + throw Invalid("value exceeds the v0 JSON depth; use smaller independent batches"); + } + + Node(current); + switch (current) + { + case TextValue text: + ChargeText(text.Value); + break; + case TupleValue tuple: + CheckCount(tuple.Items.Count, "tuple"); + Pending(stack.Count, tuple.Items.Count); + foreach (Value item in tuple.Items) + { + stack.Push((item, level + 1)); + } + + break; + case ConstructorValue constructor: + CheckCount(constructor.Arguments.Count, "constructor"); + Pending(stack.Count, constructor.Arguments.Count); + foreach (Value argument in constructor.Arguments) + { + stack.Push((argument, level + 1)); + } + + break; + case IntValue number when number.Value < Json.IntMin || number.Value > Json.IntMax: + throw Invalid("integer is outside Core's 63-bit Int range"); + default: + // Real and Hash are one node each. + break; + } + } + + return value; + } + + /// + /// Refuses before more values are stacked when + /// the values already charged and stacked could not all be charged, so + /// the walk's stack never holds more than max_value_nodes values. + /// + private void Pending(int stacked, int children) + { + if ((long)budget.Nodes + stacked + children > Data.Limits.MaxValueNodes) + { + throw Invalid("value exceeds max_value_nodes; use smaller independent batches"); + } + } +} + +/// Decodes boundary values with the program's data. +public sealed class Decoder +{ + internal Decoder(ProgramData data) + { + Data = data; + } + + /// The program's data. + public ProgramData Data { get; } + + /// Decodes a nested value. + /// A binding failure when the value does not map. + public T Decode(Codec codec, Value value) + { + ArgumentNullException.ThrowIfNull(codec); + return codec.Decode(this, value); + } + + /// Whether is built by the named constructor. + public bool Is(Value value, string constructor) + { + ArgumentNullException.ThrowIfNull(constructor); + return Data.ConstructorId(constructor) is { } expected && value is ConstructorValue built && built.Identity.Equals(expected); + } + + /// The fields of a value built by the named constructor with fields. + /// A mapping failure for another constructor or count. + public IReadOnlyList Fields(Value value, string constructor, int count) + { + if (Is(value, constructor) && value is ConstructorValue built && built.Arguments.Count == count) + { + return built.Arguments; + } + + throw Mapping(Identities.Excerpt(constructor) + " result has the wrong constructor or field count"); + } + + /// A mapping failure for a value that is none of a type's constructors. + public JacquardException Mismatch(string typeName) => Mapping("unknown " + Identities.Excerpt(typeName) + " case"); + + /// A mapping failure: a boundary value that does not decode into the requested type. + [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1822", Justification = "Part of the decoder a custom codec is given, beside its other failures.")] + public JacquardException Mapping(string? message) => JacquardException.Binding(message ?? "null"); +} diff --git a/dotnet/src/JacquardHost/Sdk/ProgramData.cs b/dotnet/src/JacquardHost/Sdk/ProgramData.cs new file mode 100644 index 0000000..863741b --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/ProgramData.cs @@ -0,0 +1,374 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// The prepared program's public names and identities, used to map native +/// values. It is immutable once the program is prepared, and thread-safe. +/// +public sealed class ProgramData +{ + internal static readonly ImmutableHashSet Builtins = ImmutableHashSet.Create(StringComparer.Ordinal, "true", "false", "some", "none", "cons", "nil"); + + private readonly Dictionary names; + private readonly Dictionary constructors; + private readonly Dictionary labels = new(); + private readonly Dictionary primitives = new(); + + /// Each declaration group's head identity and its member identities. + private readonly Dictionary> members = new(); + + internal ProgramData(IReadOnlyDictionary names, IReadOnlyDictionary constructors, IReadOnlyList> groups, Limits limits) + { + this.names = new Dictionary(names, StringComparer.Ordinal); + this.constructors = new Dictionary(constructors, StringComparer.Ordinal); + Limits = limits; + foreach (KeyValuePair entry in constructors.OrderBy(pair => pair.Key, StringComparer.Ordinal)) + { + labels[entry.Value] = entry.Key; + } + + foreach (string name in new[] { "int", "real", "text", "bool", "hash", "list", "option" }) + { + if (names.TryGetValue(name, out Hash? identity)) + { + primitives[identity] = name; + } + } + + foreach (IReadOnlyList group in groups) + { + if (group.Count > 0) + { + members[group[0].Hash] = new HashSet(group.Skip(1).Select(entry => entry.Hash)); + } + } + } + + /// The session limits that bound mapped values. + public Limits Limits { get; } + + /// + /// The exact identity of a public type, constructor, effect, or term name + /// in the prepared program and its discovered prelude. + /// + /// A binding failure for an unknown name. + public Hash Identity(string name) + { + ArgumentNullException.ThrowIfNull(name); + return names.TryGetValue(Identities.DisplayToStore(name), out Hash? identity) + ? identity + : throw JacquardException.Binding("unknown name " + Identities.Excerpt(name)); + } + + /// The public name of a constructor or operation identity, or null. + public string? Label(Hash identity) + { + ArgumentNullException.ThrowIfNull(identity); + return labels.TryGetValue(identity, out string? label) ? label : null; + } + + internal Hash? Name(string name) => names.TryGetValue(name, out Hash? identity) ? identity : null; + + internal Hash? ConstructorId(string name) => constructors.TryGetValue(Identities.DisplayToStore(name), out Hash? identity) ? identity : null; + + /// A after checking that the constructor exists. + /// + /// A binding failure for an unknown constructor, or for more fields than + /// the v0 hard limit of collection items. + /// + public Dynamic Make(string name, params Dynamic[] fields) + { + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(fields); + if (fields.Length > Limits.Hard.MaxCollectionItems) + { + throw JacquardException.Binding("constructor of " + fields.Length.ToString(CultureInfo.InvariantCulture) + + " fields exceeds the v0 hard limit of " + Limits.Hard.MaxCollectionItems.ToString(CultureInfo.InvariantCulture)); + } + + if (ConstructorId(name) is null) + { + throw JacquardException.Binding("unknown constructor " + Identities.Excerpt(name)); + } + + return new Dynamic.Variant(Identities.DisplayToStore(name), fields); + } + + /// Encodes a native value, bounded by the selected limits. + /// + /// An invalid argument naming the value's location; a codec that throws + /// anything else is reported so too, its exception kept as the inner + /// exception. + /// + public Value Encode(Codec codec, T value) + { + ArgumentNullException.ThrowIfNull(codec); + try + { + return Encoder.Root(this, "value").Encode(codec, value); + } + catch (Exception error) when (Caller.Wraps(error)) + { + throw CodecFailed("value", error); + } + } + + /// A caller's codec that threw while encoding at : an invalid argument quoting an excerpt, the exception kept. + internal static JacquardException CodecFailed(string path, Exception error) => + JacquardException.InvalidAt(path, "its codec failed: " + Identities.Describe(error), error); + + /// + /// Decodes a boundary value into a native value. The value is first + /// measured, within the selected max_value_nodes and nesting + /// limits, so a shared or deep value cannot make decoding unbounded; + /// every value Core sends is within them. + /// + /// + /// A binding failure when it does not decode; a codec that throws anything + /// else is reported so too, its exception kept as the inner exception. + /// + public T Decode(Codec codec, Value value) + { + ArgumentNullException.ThrowIfNull(codec); + ArgumentNullException.ThrowIfNull(value); + Measure(value); + try + { + return codec.Decode(new Decoder(this), value); + } + catch (Exception error) when (Caller.Wraps(error)) + { + throw JacquardException.Binding("value: its codec failed: " + Identities.Describe(error), error); + } + } + + /// + /// Refuses a value with more nodes than max_value_nodes, or nested + /// past the JSON depth, walking it with an explicit stack that never + /// holds more than the node limit. + /// + private void Measure(Value value) + { + int nodes = 0; + var stack = new Stack<(Value Value, int Level)>(); + stack.Push((value, 0)); + while (stack.Count > 0) + { + (Value current, int level) = stack.Pop(); + if (++nodes > Limits.MaxValueNodes) + { + throw JacquardException.Binding("value exceeds max_value_nodes"); + } + + // A value at the top of a frame's payload is at JSON depth 3, and each level adds 2. + if (3L + (2L * level) > Limits.MaxJsonDepth) + { + throw JacquardException.Binding("value exceeds the v0 JSON depth"); + } + + IReadOnlyList children = current switch + { + TupleValue tuple => tuple.Items, + ConstructorValue constructor => constructor.Arguments, + _ => Array.Empty(), + }; + if ((long)nodes + stack.Count + children.Count > Limits.MaxValueNodes) + { + throw JacquardException.Binding("value exceeds max_value_nodes"); + } + + foreach (Value child in children) + { + stack.Push((child, level + 1)); + } + } + } + + internal string? Primitive(BoundaryType expected) => + expected is NominalType nominal && primitives.TryGetValue(nominal.Identity, out string? name) ? name : null; + + internal bool IsNullary(Value value, string name) => + value is ConstructorValue constructor && constructor.Arguments.Count == 0 && string.Equals(name, Label(constructor.Identity), StringComparison.Ordinal); + + /// Walks a proper List, bounded by the collection limit. + /// A binding failure when it is not a proper List within the limit. + internal List ListItems(Value value) + { + var items = new List(); + Value cursor = value; + while (true) + { + if (cursor is ConstructorValue constructor) + { + string? label = Label(constructor.Identity); + if (constructor.Arguments.Count == 2 && label == "cons") + { + if (items.Count >= Limits.MaxCollectionItems) + { + throw JacquardException.Binding("list exceeds the selected limit"); + } + + items.Add(constructor.Arguments[0]); + cursor = constructor.Arguments[1]; + continue; + } + + if (constructor.Arguments.Count == 0 && label == "nil") + { + return items; + } + } + + throw JacquardException.Binding("expected a proper List"); + } + } + + /// + /// Checks that an encoded value has the checked boundary type, as far as + /// the prepared metadata describes it: primitives, Lists, Options, and + /// tuples exactly, and other data by its constructor's declaration group. + /// Called only on measured values, so the walk is bounded by their depth + /// and node count. + /// + [SuppressMessage("Maintainability", "CA1502", Justification = "One case per boundary type, as in the other SDKs.")] + internal void Conform(Value value, BoundaryType expected, string path) + { + if (expected is TupleType tuple) + { + if (value is TupleValue values && values.Items.Count == tuple.Items.Count) + { + for (int i = 0; i < values.Items.Count; i++) + { + Conform(values.Items[i], tuple.Items[i], path + "[" + i.ToString(CultureInfo.InvariantCulture) + "]"); + } + + return; + } + + throw Refuse(path, "a " + tuple.Items.Count.ToString(CultureInfo.InvariantCulture) + "-item tuple"); + } + + var nominal = (NominalType)expected; + BoundaryType? child = nominal.Arguments.Count == 0 ? null : nominal.Arguments[0]; + string? primitive = Primitive(expected); + if (primitive is null) + { + bool known = members.TryGetValue(nominal.Identity, out HashSet? group); + if (value is ConstructorValue constructor && (!known || group!.Contains(constructor.Identity))) + { + return; + } + + throw Refuse(path, "a " + LabelOfType(nominal.Identity) + " value"); + } + + switch (primitive) + { + case "int": + if (value is not IntValue) + { + throw Refuse(path, "an integer"); + } + + break; + case "real": + if (value is not RealValue) + { + throw Refuse(path, "a Real (double)"); + } + + break; + case "text": + if (value is not TextValue) + { + throw Refuse(path, "text"); + } + + break; + case "hash": + if (value is not HashValue) + { + throw Refuse(path, "a Hash"); + } + + break; + case "bool": + if (!IsNullary(value, "true") && !IsNullary(value, "false")) + { + throw Refuse(path, "a bool"); + } + + break; + case "list": + List items; + try + { + items = ListItems(value); + } + catch (JacquardException) + { + throw JacquardException.InvalidAt(path, "expected a List"); + } + + if (child is not null) + { + for (int i = 0; i < items.Count; i++) + { + Conform(items[i], child, path + "[" + i.ToString(CultureInfo.InvariantCulture) + "]"); + } + } + + break; + default: + if (IsNullary(value, "none")) + { + return; + } + + if (value is ConstructorValue some && some.Arguments.Count == 1 && Label(some.Identity) == "some") + { + if (child is not null) + { + Conform(some.Arguments[0], child, path); + } + + return; + } + + throw Refuse(path, "an Option"); + } + } + + private static JacquardException Refuse(string path, string what) => JacquardException.InvalidAt(path, "expected " + what); + + private string LabelOfType(Hash identity) + { + foreach (KeyValuePair entry in names.OrderBy(pair => pair.Key, StringComparer.Ordinal)) + { + if (entry.Value.Equals(identity)) + { + return entry.Key; + } + } + + return identity.Hex; + } +} + +/// Which exceptions from a caller's code the SDK wraps in a . +internal static class Caller +{ + /// + /// Every exception from caller code except the SDK's own (already a + /// ) and running out of memory, which no + /// wrapping could report. + /// + internal static bool Wraps(Exception error) => error is not JacquardException and not OutOfMemoryException; +} diff --git a/dotnet/src/JacquardHost/Sdk/Values.cs b/dotnet/src/JacquardHost/Sdk/Values.cs new file mode 100644 index 0000000..c210eda --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Values.cs @@ -0,0 +1,183 @@ +using System; +using System.Collections.Generic; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// Jacquard's unit, the empty tuple, as a .NET value. +public readonly record struct Unit +{ + /// The unit value. + public static Unit Value => default; + + /// + public override string ToString() => "()"; +} + +/// +/// A Jacquard Option: or Some of a value. +/// Unlike a nullable type, options nest: Option (Option Int) is +/// Option<Option<long>>, and Some(None) stays distinct +/// from None. +/// +/// The value's type. +[System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1000", Justification = "Option.None and Option.Some name the two cases, as Nullable and ImmutableArray do.")] +public readonly struct Option : IEquatable> +{ + private readonly T value; + + private Option(T value) + { + this.value = value; + HasValue = true; + } + + /// The empty option. + public static Option None => default; + + /// Whether this is Some. + public bool HasValue { get; } + + /// The value of Some. + /// The option is None. + public T Value => HasValue ? value : throw new InvalidOperationException("the option is None"); + + /// Some(value). + public static Option Some(T value) => new(value); + + /// Whether two options are both None, or both Some of equal values. + public static bool operator ==(Option left, Option right) => left.Equals(right); + + /// Whether two options differ. + public static bool operator !=(Option left, Option right) => !left.Equals(right); + + /// The value of Some, or for None. + public T GetValueOrDefault(T fallback) => HasValue ? value : fallback; + + /// + public bool Equals(Option other) => HasValue == other.HasValue && (!HasValue || EqualityComparer.Default.Equals(value, other.value)); + + /// + public override bool Equals(object? obj) => obj is Option other && Equals(other); + + /// + public override int GetHashCode() => HasValue ? HashCode.Combine(true, value) : 0; + + /// + public override string ToString() => HasValue ? "Some(" + value + ")" : "None"; +} + +/// Builds values with the type inferred. +public static class Option +{ + /// Some(value). + public static Option Some(T value) => Option.Some(value); +} + +/// +/// A function argument with the codec that encodes it. Plain values convert +/// implicitly: and as Int, +/// as Real, as Text, +/// , , a protocol +/// , and . Anything else +/// is passed as codec.Arg(value), as are lists and records. There is +/// deliberately no conversion from : money crosses as an +/// integer count of minor units. +/// +public abstract class Argument +{ + private protected Argument() + { + } + + /// An Int, Core's full 63-bit range; a value outside it is refused. + public static implicit operator Argument(long value) => Codecs.Int.Arg(value); + + /// An Int. + public static implicit operator Argument(int value) => Codecs.Int.Arg(value); + + /// A Real, the exact binary64 bits of the double. + public static implicit operator Argument(double value) => Codecs.Real.Arg(value); + + /// A Text; a lone surrogate is refused. + public static implicit operator Argument(string value) => Codecs.Text.Arg(value); + + /// A Bool. + public static implicit operator Argument(bool value) => Codecs.Bool.Arg(value); + + /// A Hash. + public static implicit operator Argument(Hash value) => Codecs.Hash.Arg(value); + + /// A protocol value, passed through within the same bounds. + public static implicit operator Argument(Value value) => Codecs.Value.Arg(value); + + /// A value by public name. + public static implicit operator Argument(Dynamic value) => Codecs.Dynamic.Arg(value); + + /// encoded with ; the same as codec.Arg(value). + public static Argument Of(Codec codec, T value) + { + ArgumentNullException.ThrowIfNull(codec); + return new Typed(codec, value); + } + + /// An Int. + public static Argument FromInt64(long value) => value; + + /// An Int. + public static Argument FromInt32(int value) => value; + + /// A Real. + public static Argument FromDouble(double value) => value; + + /// A Text. + public static Argument FromString(string value) => value; + + /// A Bool. + public static Argument FromBoolean(bool value) => value; + + /// A Hash. + public static Argument FromHash(Hash value) => value; + + /// A protocol value. + public static Argument FromValue(Value value) => value; + + /// A value by public name. + public static Argument FromDynamic(Dynamic value) => value; + + /// Encodes the argument at 's location. + internal abstract Value Encode(Encoder encoder); + + private sealed class Typed : Argument + { + private readonly Codec codec; + private readonly T value; + + internal Typed(Codec codec, T value) + { + this.codec = codec; + this.value = value; + } + + internal override Value Encode(Encoder encoder) => encoder.Encode(codec, value); + } +} + +/// One batch's rows as the final List argument, encoded from the caller's list without copying it. +internal sealed class Rows : Argument +{ + private readonly Codec codec; + private readonly IReadOnlyList rows; + private readonly int start; + private readonly int count; + + internal Rows(Codec codec, IReadOnlyList rows, int start, int count) + { + this.codec = codec; + this.rows = rows; + this.start = start; + this.count = count; + } + + internal override Value Encode(Encoder encoder) => encoder.List(codec, rows, start, count); +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs new file mode 100644 index 0000000..735f4bc --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs @@ -0,0 +1,738 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// Signatures, identity groups, and native values through checked types, with synthetic identities. +[TestFixture] +public sealed class DataTests +{ + internal static Hash Id(char letter) => new(new string(letter, 64)); + + internal static Hash Numbered(int n) => new(n.ToString("x64", CultureInfo.InvariantCulture)); + + internal static NominalType Nominal(Hash identity, params BoundaryType[] arguments) => new(identity, arguments); + + /// Data with synthetic identities for the builtins, an Incident record, an Urgency enumeration, and a Triage union. + internal static ProgramData Fixture(Limits limits) + { + var names = new Dictionary + { + ["int"] = Id('1'), + ["real"] = Id('2'), + ["text"] = Id('3'), + ["bool"] = Id('4'), + ["hash"] = Id('5'), + ["list"] = Id('6'), + ["option"] = Id('7'), + ["incident"] = Id('8'), + ["urgency"] = Id('9'), + ["triage"] = Numbered(10), + }; + var constructors = new Dictionary + { + ["true"] = Id('a'), + ["false"] = Id('b'), + ["some"] = Id('c'), + ["none"] = Id('d'), + ["cons"] = Id('e'), + ["nil"] = Id('f'), + ["incident"] = Id('0'), + ["page-now"] = Numbered(1), + ["watch-only"] = Numbered(2), + ["routed"] = Numbered(3), + ["invalid-incident"] = Numbered(4), + }; + var groups = new List>(); + foreach (string[] group in new[] { new[] { "incident", "incident" }, new[] { "urgency", "page-now", "watch-only" }, new[] { "triage", "routed", "invalid-incident" } }) + { + var entries = new List { new(group[0], names[group[0]]) }; + entries.AddRange(group.Skip(1).Select(member => new Identities.Entry(member, constructors[member]))); + groups.Add(entries); + } + + return new ProgramData(names, constructors, groups, limits); + } + + internal sealed record Incident(string Id, long Severity, string? Owner); + + internal enum Urgency + { + PageNow, + WatchOnly, + } + + internal abstract record Triage; + + internal sealed record Routed(string Id, Urgency Urgency) : Triage; + + internal sealed record InvalidIncident(string Id) : Triage; + + internal static readonly RecordCodec IncidentCodec = Codecs.Record( + "Incident", + Codecs.Field("id", Codecs.Text, (Incident incident) => incident.Id), + Codecs.Field("severity", Codecs.Int, (Incident incident) => incident.Severity), + Codecs.Field("owner", Codecs.Nullable(Codecs.Text), (Incident incident) => incident.Owner), + (id, severity, owner) => new Incident(id, severity, owner)); + + internal static readonly UnionCodec TriageCodec = Codecs.Union( + Codecs.Record("Routed", Codecs.Field("id", Codecs.Text, (Routed routed) => routed.Id), Codecs.Field("urgency", Codecs.Enum(), (Routed routed) => routed.Urgency), (id, urgency) => new Routed(id, urgency)), + Codecs.Record("InvalidIncident", Codecs.Field("id", Codecs.Text, (InvalidIncident invalid) => invalid.Id), id => new InvalidIncident(id))); + + private static string Refusal(Action body) => Support.Throws(body).Message; + + [Test] + public void SignaturesAndGroups() + { + var names = new Dictionary + { + ["list"] = Id('6'), ["line"] = Id('8'), ["int"] = Id('1'), ["option"] = Id('7'), ["text"] = Id('3'), ["inventory"] = Id('9'), ["dispatch"] = Id('a'), + }; + CallInterface spec = Identities.ParseSignature("(List Line, Int) ->{Inventory, Dispatch} Option Text", names); + Assert.That(spec.Parameters, Is.EqualTo(new BoundaryType[] { Nominal(Id('6'), Nominal(Id('8'))), Nominal(Id('1')) })); + Assert.That(spec.Effects, Is.EqualTo(new[] { Id('9'), Id('a') })); + Assert.That(spec.Result, Is.EqualTo(Nominal(Id('7'), Nominal(Id('3'))))); + Assert.That(Identities.ParseSignature("() ->{} ()", names).Result, Is.EqualTo(new TupleType())); + Assert.That(Identities.ParseSignature("((Int, Text)) ->{} (Int)", names).Result, Is.EqualTo(Nominal(Id('1')))); + Assert.That(Identities.ParseSignature("((Int, Text)) ->{} Int", names).Parameters, Is.EqualTo(new BoundaryType[] { new TupleType(Nominal(Id('1')), Nominal(Id('3'))) })); + foreach (string text in new[] { "forall a | e. (List a) ->{| e} Bool", "(Missing) ->{} Int", "(Int) ->{} Int extra)", "(Int) ->{} Int $" }) + { + Support.Throws(() => Identities.ParseSignature(text, names), text); + } + + string output = "0 " + Id('1').Hex + "\n0:line " + Id('1').Hex + "\n0:line " + Id('2').Hex + "\nnoise\n1:other " + Id('3').Hex + "\n"; + List> groups = Identities.HashGroups(output); + Assert.That(groups, Has.Count.EqualTo(2)); + Assert.That(groups[0][1].Hash, Is.EqualTo(Id('2'))); + Assert.That(Support.Throws(() => Identities.HashGroups("nothing here")).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + var x1 = new Identities.Entry("x", Id('1')); + Support.Throws(() => Identities.CheckGroups(new List> { new() { x1 }, new() { new("x", Id('2')) } })); + Identities.CheckGroups(new List> { new() { x1 }, new() { x1 } }); + foreach ((string display, string store) in new[] { ("WorkToday", "work-today"), ("PageNow", "page-now"), ("text", "text"), ("Inventory", "inventory") }) + { + Assert.That(Identities.DisplayToStore(display), Is.EqualTo(store)); + } + + Assert.That(Identities.Normal("lookup_stock"), Is.EqualTo("lookup-stock")); + Assert.That(Identities.DisplayToStore("\ud800X"), Is.EqualTo("\ud800X"), "a lone surrogate is not upper case"); + } + + [Test] + public void NativeValuesRoundtripAndConformToCheckedTypes() + { + ProgramData data = Fixture(Limits.Hard); + var rows = new List { new("a", 5, null), new("b", 2, "ops") }; + BoundaryType list = Nominal(Id('6'), Nominal(Id('8'))); + Value value = data.Encode(Codecs.List(IncidentCodec), rows); + data.Conform(value, list, "rows"); + Assert.That(data.Decode(Codecs.List(IncidentCodec), value), Is.EqualTo(rows)); + BoundaryType real = Nominal(Id('2')); + foreach (double number in new[] { 1.0, -0.0, double.PositiveInfinity, double.NaN, BitConverter.Int64BitsToDouble(0x7ff8000000000123L) }) + { + Value encoded = data.Encode(Codecs.Real, number); + data.Conform(encoded, real, "real"); + Assert.That(BitConverter.DoubleToInt64Bits(data.Decode(Codecs.Real, encoded)), Is.EqualTo(BitConverter.DoubleToInt64Bits(number))); + } + + BoundaryType integer = Nominal(Id('1')); + Assert.That(Refusal(() => data.Encode(Codecs.Int, 1L << 62)), Is.EqualTo("value: integer is outside Core's 63-bit Int range")); + Assert.That(Refusal(() => data.Encode(Codecs.Int, long.MinValue)), Is.EqualTo("value: integer is outside Core's 63-bit Int range")); + Assert.That(data.Encode(Codecs.Int, Json.IntMax), Is.EqualTo(new IntValue(Json.IntMax))); + Assert.That(data.Encode(Codecs.Int, Json.IntMin), Is.EqualTo(new IntValue(Json.IntMin))); + Support.Throws(() => data.Decode(Codecs.Int32, new IntValue(1L << 40))); + Assert.That(data.Decode(Codecs.Int32, new IntValue(7)), Is.EqualTo(7)); + Assert.That(Refusal(() => data.Conform(data.Encode(Codecs.Text, "x"), integer, "argument[0]")), Is.EqualTo("argument[0]: expected an integer")); + Assert.That(Refusal(() => data.Conform(data.Encode(Codecs.Int, 3L), real, "argument[0]")), Is.EqualTo("argument[0]: expected a Real (double)")); + Assert.That(Refusal(() => data.Conform(data.Encode(Codecs.Enum(), Urgency.PageNow), Nominal(Id('8')), "argument[0]")), Does.Contain("expected a incident value")); + Value texts = data.Encode(Codecs.List(Codecs.Text), new[] { "ok", "ok" }); + Assert.That(Refusal(() => data.Conform(texts, Nominal(Id('6'), integer), "argument[0]")), Is.EqualTo("argument[0][0]: expected an integer")); + var unit = new TupleType(); + data.Conform(data.Encode(Codecs.Unit, Unit.Value), unit, "unit"); + Assert.That(data.Decode(Codecs.Unit, Value.Unit), Is.EqualTo(Unit.Value)); + Support.Throws(() => data.Conform(data.Encode(Codecs.Tuple(Codecs.Int, Codecs.Int), (1L, 2L)), unit, "unit")); + + // A record with an unknown constructor is refused at its location. + Assert.That(Refusal(() => data.Encode(Codecs.Record("Unknown", Codecs.Field("field", Codecs.Int, (long x) => x), x => x), 1L)), Is.EqualTo("value: unknown constructor Unknown")); + + // A record over a builtin constructor is refused, as Dynamic is. + Assert.That(Refusal(() => data.Encode(Codecs.Record("Some", Codecs.Field("inner", Codecs.Int, (long x) => x), x => x), 1L)), Does.Contain("native form of builtin constructor Some")); + + // null is no value; Option and Unit are. + Assert.That(Refusal(() => data.Encode(Codecs.Text, null!)), Does.StartWith("value: null has no Jacquard value")); + Assert.That(Refusal(() => data.Encode(Codecs.Text, "\ud800")), Does.Contain("lone surrogate")); + } + + [Test] + public void OptionsEnumsUnionsAndDynamicValues() + { + ProgramData data = Fixture(Limits.Hard); + Triage routed = new Routed("r", Urgency.PageNow); + Value value = data.Encode(TriageCodec, routed); + Assert.That(value, Is.EqualTo(new ConstructorValue(Numbered(3), new Value[] { new TextValue("r"), new ConstructorValue(Numbered(1), Array.Empty()) }))); + Assert.That(data.Decode(TriageCodec, value), Is.EqualTo(routed)); + Value invalid = data.Encode(TriageCodec, new InvalidIncident("x")); + Assert.That(data.Decode(TriageCodec, invalid), Is.EqualTo(new InvalidIncident("x"))); + Assert.That(Refusal(() => data.Decode(Codecs.Enum(), invalid)), Does.Contain("unknown Urgency case")); + var shortRouted = new ConstructorValue(Numbered(3), new Value[] { new TextValue("r") }); + Assert.That(Refusal(() => data.Decode(TriageCodec, shortRouted)), Does.Contain("wrong constructor or field count")); + BoundaryType option = Nominal(Id('7'), Nominal(Id('7'), Nominal(Id('1')))); + Codec>> nested = Codecs.Option(Codecs.Option(Codecs.Int)); + var cases = new[] { Option>.None, Option.Some(Option.None), Option.Some(Option.Some(3L)) }; + var dynamics = new[] { Dynamic.None(), Dynamic.Some(Dynamic.None()), Dynamic.Some(Dynamic.Some(Dynamic.Of(3L))) }; + for (int i = 0; i < cases.Length; i++) + { + Value encoded = data.Encode(nested, cases[i]); + data.Conform(encoded, option, "option"); + Assert.That(data.Decode(nested, encoded), Is.EqualTo(cases[i])); + Assert.That(data.Decode(Codecs.Dynamic, encoded), Is.EqualTo(dynamics[i])); + Assert.That(data.Encode(Codecs.Dynamic, dynamics[i]), Is.EqualTo(encoded)); + } + + // A nullable value type maps null to None. + Assert.That(data.Decode(Codecs.NullableValue(Codecs.Int), data.Encode(Codecs.NullableValue(Codecs.Int), null)), Is.Null); + Assert.That(data.Decode(Codecs.NullableValue(Codecs.Int), data.Encode(Codecs.NullableValue(Codecs.Int), 4L)), Is.EqualTo(4L)); + Dynamic watch = data.Make("WatchOnly"); + Assert.That(watch, Is.EqualTo(Dynamic.VariantOf("watch-only"))); + Assert.That(data.Decode(Codecs.Enum(), data.Encode(Codecs.Dynamic, watch)), Is.EqualTo(Urgency.WatchOnly)); + Assert.That(Support.Throws(() => data.Make("Missing")).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Support.Throws(() => data.Encode(Codecs.Dynamic, Dynamic.VariantOf("Some", Dynamic.Of(1L))), "a builtin constructor by name"); + Dynamic dynamic = Dynamic.ListOf(Dynamic.TupleOf(Dynamic.Of(true), Dynamic.Of("x"), Dynamic.Of(2.5)), new Dynamic.HashValue(Id('5'))); + Assert.That(data.Decode(Codecs.Dynamic, data.Encode(Codecs.Dynamic, dynamic)), Is.EqualTo(dynamic)); + Assert.That(Refusal(() => data.Decode(Codecs.Dynamic, new ConstructorValue(Id('9'), Array.Empty()))), Does.Contain("unmapped constructor")); + Unit[] units = Enumerable.Repeat(Unit.Value, 40).ToArray(); + Assert.That(Refusal(() => data.Encode(Codecs.List(Codecs.Unit), units)), Does.Contain("JSON depth"), "a list deeper than the JSON depth"); + Assert.That(data.Identity("Urgency"), Is.EqualTo(Id('9'))); + Support.Throws(() => data.Identity("missing")); + } + + [Test] + public void ExplicitlyNamedEnumsTuplesAndUnionsAgree() + { + ProgramData data = Fixture(Limits.Hard); + EnumCodec named = Codecs.Enum(new Dictionary { [Urgency.PageNow] = "PageNow", [Urgency.WatchOnly] = "watch-only" }); + Assert.That(data.Encode(named, Urgency.WatchOnly), Is.EqualTo(data.Encode(Codecs.Enum(), Urgency.WatchOnly))); + Support.Throws(() => Codecs.Enum(new Dictionary { [Urgency.PageNow] = "PageNow" })); + Assert.That(Refusal(() => data.Encode(Codecs.Enum(), (Urgency)7)), Does.Contain("Urgency has no constructor for 7")); + + // A union's cases must derive from it. + Support.Throws(() => Codecs.Union(IncidentCodec)); + Value tuple = data.Encode(Codecs.Tuple(Codecs.Int, Codecs.Text), (1L, "x")); + Assert.That(tuple, Is.EqualTo(new TupleValue(new IntValue(1), new TextValue("x")))); + Assert.That(data.Decode(Codecs.Tuple(Codecs.Int, Codecs.Text), tuple), Is.EqualTo((1L, "x"))); + Support.Throws(() => data.Decode(Codecs.Tuple(Codecs.Int, Codecs.Text, Codecs.Text), tuple)); + + // A custom codec built from two functions. + Codec upper = Codecs.Create((encoder, text) => Codecs.Text.Encode(encoder, text.ToUpperInvariant()), (decoder, v) => Codecs.Text.Decode(decoder, v)); + Assert.That(data.Decode(upper, data.Encode(upper, "abc")), Is.EqualTo("ABC")); + } + + /// + /// There is no implicit conversion from (or from + /// to Int): money crosses as integer minor units. + /// + [Test] + public void DecimalHasNoConversion() + { + IEnumerable sources = typeof(Argument).GetMethods(BindingFlags.Public | BindingFlags.Static) + .Where(method => method.Name is "op_Implicit" or "op_Explicit") + .Select(method => method.GetParameters()[0].ParameterType); + Assert.That(sources, Does.Not.Contain(typeof(decimal))); + Assert.That(typeof(Dynamic).GetMethods().Where(method => method.Name == "Of").Select(method => method.GetParameters()[0].ParameterType), Does.Not.Contain(typeof(decimal))); + Assert.That(typeof(Codecs).GetProperties().Select(property => property.PropertyType), Does.Not.Contain(typeof(Codec))); + } + + /// A recursive type maps through a lazy codec, which is made once even when many threads ask at once. + internal sealed record Tree(string Label, IReadOnlyList Children) + { + public bool Equals(Tree? other) => other is not null && Label == other.Label && Children.SequenceEqual(other.Children); + + public override int GetHashCode() => Label.GetHashCode(StringComparison.Ordinal); + } + + [Test] + public void RecursiveTypesMapThroughALazyCodec() + { + var names = new Dictionary { ["list"] = Id('6'), ["text"] = Id('3') }; + var constructors = new Dictionary { ["cons"] = Id('e'), ["nil"] = Id('f'), ["tree"] = Id('0') }; + var data = new ProgramData(names, constructors, Array.Empty>(), Limits.Hard); + int made = 0; + RecordCodec? codec = null; + LazyCodec self = Codecs.Lazy(() => + { + Interlocked.Increment(ref made); + return codec!; + }); + codec = Codecs.Record( + "Tree", + Codecs.Field("label", Codecs.Text, (Tree tree) => tree.Label), + Codecs.Field("children", Codecs.List(self), (Tree tree) => tree.Children), + (label, children) => new Tree(label, children)); + var tree = new Tree("root", new[] { new Tree("leaf", Array.Empty()), new Tree("branch", new[] { new Tree("leaf", Array.Empty()) }) }); + Parallel.For(0, 8, _ => Assert.That(data.Decode(self, data.Encode(self, tree)), Is.EqualTo(tree))); + Assert.That(made, Is.EqualTo(1), "the lazy codec was made more than once"); + + // A chain of a million lazy codecs resolves in a loop, and one standing for itself is refused. + LazyCodec chain = Codecs.Lazy(() => Codecs.Int); + for (int i = 0; i < 1_000_000; i++) + { + LazyCodec inner = chain; + chain = Codecs.Lazy(() => inner); + } + + Assert.That(data.Decode(chain, data.Encode(chain, 5L)), Is.EqualTo(5L)); + LazyCodec? loop = null; + loop = Codecs.Lazy(() => loop!); + JacquardException refused = Support.Throws(() => data.Encode(loop, 1L)); + Assert.That(refused.Message, Does.Contain("a lazy codec stands for itself")); + Assert.That(refused.InnerException, Is.InstanceOf()); + } + + [Test] + public void ArgumentsConvertFromPlainValuesOrCarryTheirCodec() + { + ProgramData data = Fixture(Limits.Hard); + Value Encode(Argument argument) => argument.Encode(Encoder.Root(data, "f.argument[0]")); + Assert.That(Encode(35), Is.EqualTo(data.Encode(Codecs.Int, 35L))); + Assert.That(Encode(35L), Is.EqualTo(new IntValue(35))); + Assert.That(Encode(2.5), Is.EqualTo(RealValue.Of(2.5))); + Assert.That(Encode("x"), Is.EqualTo(new TextValue("x"))); + Assert.That(Encode(true), Is.EqualTo(data.Encode(Codecs.Bool, true))); + Assert.That(Encode(Id('5')), Is.EqualTo(new HashValue(Id('5')))); + Assert.That(Encode(Dynamic.Of(4L)), Is.EqualTo(new IntValue(4))); + Assert.That(Encode(Codecs.List(IncidentCodec).Arg(Array.Empty())), Is.EqualTo(data.Encode(Codecs.List(IncidentCodec), Array.Empty()))); + Assert.That(Refusal(() => Encode(long.MaxValue)), Is.EqualTo("f.argument[0]: integer is outside Core's 63-bit Int range")); + Assert.That(Refusal(() => Encode((string)null!)), Does.StartWith("f.argument[0]: null has no Jacquard value")); + } + + [Test] + public void SelectedLimitsBoundMappedValues() + { + Limits limits = Limits.Hard.With("max_collection_items", 2).With("max_text_bytes", 3); + ProgramData data = Fixture(limits); + Assert.That(Refusal(() => data.Encode(Codecs.List(Codecs.Int), new[] { 1L, 2L, 3L })), Does.Contain("max_collection_items")); + Assert.That(Refusal(() => data.Encode(Codecs.Text, "four")), Does.Contain("max_text_bytes")); + Value longList = Fixture(Limits.Hard).Encode(Codecs.List(Codecs.Int), new[] { 1L, 2L, 3L }); + Assert.That(Refusal(() => data.Decode(Codecs.List(Codecs.Int), longList)), Does.Contain("selected limit")); + + // A record's fields count against max_collection_items too, so this one needs three. + ProgramData fields = Fixture(Limits.Hard.With("max_collection_items", 3).With("max_text_bytes", 3)); + Assert.That(Refusal(() => fields.Encode(IncidentCodec, new Incident("abcd", 1, null))), Is.EqualTo("value.id: text exceeds max_text_bytes")); + Assert.That(Refusal(() => data.Encode(IncidentCodec, new Incident("abc", 1, null))), Is.EqualTo("value: constructor exceeds max_collection_items")); + } + + /// + /// A small .NET value can denote a huge one: three levels of 64 + /// references to one tuple are 64^3 integers. Encoding stops at the + /// 4097th node, so it costs no more than one frame; a raw value is + /// refused before its walk stacks more than the node limit, and a deep + /// one is refused by its depth without recursion. + /// + [Test] + public void SharedValuesAreBoundedByTheNodeBudget() + { + ProgramData data = Fixture(Limits.Hard); + var leaf = new Dynamic.Tuple(Enumerable.Repeat(Dynamic.Of(1L), 64)); + var middle = new Dynamic.Tuple(Enumerable.Repeat(leaf, 64)); + var top = new Dynamic.Tuple(Enumerable.Repeat(middle, 64)); + + // Sixty-three tuples of 65 nodes each, then the first integer of the next. + Assert.That(Refusal(() => data.Encode(Codecs.Dynamic, top)), Is.EqualTo("value[0][63][1]: value exceeds max_value_nodes; use smaller independent batches")); + var rawLeaf = new TupleValue(Enumerable.Repeat(new IntValue(1), 64)); + var rawTop = new TupleValue(Enumerable.Repeat(new TupleValue(Enumerable.Repeat(rawLeaf, 64)), 64)); + Assert.That(Refusal(() => data.Encode(Codecs.Value, rawTop)), Is.EqualTo("value: value exceeds max_value_nodes; use smaller independent batches")); + + // Decoding a supplied value is measured first, within the same bounds. + Assert.That(Refusal(() => data.Decode(Codecs.Dynamic, rawTop)), Is.EqualTo("value exceeds max_value_nodes")); + Value deep = new IntValue(1); + for (int i = 0; i < 100_000; i++) + { + deep = new TupleValue(deep); + } + + Assert.That(Refusal(() => data.Decode(Codecs.Dynamic, deep)), Is.EqualTo("value exceeds the v0 JSON depth")); + Assert.That(Refusal(() => data.Encode(Codecs.Value, deep)), Is.EqualTo("value: value exceeds the v0 JSON depth; use smaller independent batches")); + } + + /// + /// Texts are charged against the frame as they are encoded: four + /// references to one maximal text fill it, and the fifth is refused + /// before it is scanned. + /// + [Test] + public void RepeatedTextsAreChargedAgainstTheFrame() + { + ProgramData data = Fixture(Limits.Hard); + Dynamic text = Dynamic.Of(new string('x', Limits.Hard.MaxTextBytes)); + var many = new Dynamic.Tuple(Enumerable.Repeat(text, 1000)); + Assert.That(Refusal(() => data.Encode(Codecs.Dynamic, many)), Is.EqualTo("value[4]: value exceeds max_frame_bytes; use smaller independent batches")); + var rawText = new TextValue(new string('x', Limits.Hard.MaxTextBytes)); + var raw = new TupleValue(Enumerable.Repeat(rawText, 1000)); + Assert.That(Refusal(() => data.Encode(Codecs.Value, raw)), Is.EqualTo("value: value exceeds max_frame_bytes; use smaller independent batches")); + } + + /// A text too long in chars is refused before it is scanned: the lone surrogate at its end is never reached. + [Test] + public void AnOverlongTextIsRefusedBeforeItIsScanned() + { + ProgramData data = Fixture(Limits.Hard); + string text = new string('x', Limits.Hard.MaxTextBytes) + "\ud800"; + Assert.That(Refusal(() => data.Encode(Codecs.Text, text)), Is.EqualTo("value: text exceeds max_text_bytes")); + } + + /// A raw text is charged by its UTF-8 bytes: three two-byte chars exceed a four-byte limit. + [Test] + public void ARawTextIsChargedByItsUtf8Bytes() + { + ProgramData data = Fixture(Limits.Hard.With("max_text_bytes", 4)); + Assert.That(data.Encode(Codecs.Value, new TextValue("éé")), Is.EqualTo(new TextValue("éé"))); + Assert.That(Refusal(() => data.Encode(Codecs.Value, new TextValue("ééé"))), Is.EqualTo("value: text exceeds max_text_bytes")); + Assert.That(Refusal(() => data.Encode(Codecs.Value, new TextValue("\ud800"))), Is.EqualTo("value: text contains a lone surrogate, which has no Jacquard encoding")); + } + + /// A list whose count is over the limit, and whose items throw when read. + private sealed class HugeList : IReadOnlyList + { + public int Count => 1 << 30; + + public long this[int index] => throw new InvalidOperationException("element " + index.ToString(CultureInfo.InvariantCulture) + " was read"); + + public IEnumerator GetEnumerator() => throw new InvalidOperationException("the list was enumerated"); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// A list over the collection limit is refused from its count, before any element is read. + [Test] + public void AnOversizedListIsRefusedBeforeItsElementsAreRead() + { + ProgramData data = Fixture(Limits.Hard); + Assert.That(Refusal(() => data.Encode(Codecs.List(Codecs.Int), new HugeList())), Is.EqualTo("value: list exceeds max_collection_items")); + } + + private sealed class Counting : Codec + { + internal int Encodes { get; private set; } + + public override Value Encode(Encoder encoder, long value) + { + Encodes++; + return Codecs.Int.Encode(encoder, value); + } + + public override long Decode(Decoder decoder, Value value) => Codecs.Int.Decode(decoder, value); + } + + /// Tuples and records over the collection limit are refused before any item is encoded or field read. + [Test] + public void OversizedTuplesAndRecordsAreRefusedBeforeTheirItemsAreRead() + { + ProgramData data = Fixture(Limits.Hard.With("max_collection_items", 2)); + var counted = new Counting(); + Assert.That(Refusal(() => data.Encode(Codecs.Tuple(counted, counted, counted), (1L, 2L, 3L))), Is.EqualTo("value: tuple exceeds max_collection_items")); + Assert.That(counted.Encodes, Is.EqualTo(0)); + int reads = 0; + RecordCodec<(long, long, long)> record = Codecs.Record( + "Incident", + Codecs.Field("a", Codecs.Int, ((long A, long B, long C) r) => ++reads + r.A), + Codecs.Field("b", Codecs.Int, ((long A, long B, long C) r) => ++reads + r.B), + Codecs.Field("c", Codecs.Int, ((long A, long B, long C) r) => ++reads + r.C), + (a, b, c) => (a, b, c)); + Assert.That(Refusal(() => data.Encode(record, (1L, 2L, 3L))), Is.EqualTo("value: constructor exceeds max_collection_items")); + Assert.That(reads, Is.EqualTo(0)); + Fixture(Limits.Hard.With("max_collection_items", 3)).Encode(record, (1L, 2L, 3L)); + Assert.That(reads, Is.EqualTo(3)); + } + + /// A codec building a constructor by name has its fields counted before they are copied. + [Test] + public void AConstructorWithTooManyFieldsIsRefusedBeforeItIsBuilt() + { + ProgramData data = Fixture(Limits.Hard); + Value one = data.Encode(Codecs.Int, 1L); + Codec wide = Codecs.Create((encoder, count) => encoder.Constructor("routed", new RepeatList(one, (int)count)), (decoder, value) => 0); + Assert.That(Refusal(() => data.Encode(wide, 20_000_000L)), Is.EqualTo("value: constructor exceeds max_collection_items")); + Assert.That(Refusal(() => data.Encode(wide, Limits.Hard.MaxCollectionItems + 1L)), Is.EqualTo("value: constructor exceeds max_collection_items")); + } + + /// A list of one value repeated, that holds no items. + internal sealed class RepeatList : IReadOnlyList + { + private readonly T item; + + internal RepeatList(T item, int count) + { + this.item = item; + Count = count; + } + + public int Count { get; } + + public T this[int index] => item; + + public IEnumerator GetEnumerator() + { + for (int i = 0; i < Count; i++) + { + yield return item; + } + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// A Dynamic value over the hard limit is refused at construction, before its items are copied. + [Test] + public void AnOversizedDynamicIsRefusedBeforeItIsCopied() + { + int limit = Limits.Hard.MaxCollectionItems; + var many = new RepeatList(Dynamic.Of(1L), limit + 1); + Support.Throws(() => _ = new Dynamic.List(many)); + Support.Throws(() => _ = new Dynamic.Tuple(many)); + Support.Throws(() => _ = new Dynamic.Variant("routed", many)); + + // An enumeration that does not say its count is read at most one past the limit. + int read = 0; + IEnumerable endless = Enumerable.Range(0, int.MaxValue).Select(_ => + { + read++; + return Dynamic.Of(1L); + }); + Support.Throws(() => _ = new Dynamic.List(endless)); + Assert.That(read, Is.EqualTo(limit + 1)); + Assert.That(Support.Throws(() => Fixture(Limits.Hard).Make("routed", many.ToArray())).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(new Dynamic.List(new RepeatList(Dynamic.Of(1L), limit)).Items, Has.Count.EqualTo(limit)); + } + + /// A Dynamic value's equality and text walk it without recursion, so a deep caller-built value cannot exhaust the stack. + [Test] + public void DeepDynamicValuesCompareAndPrintWithoutRecursion() + { + Dynamic Deep() + { + Dynamic value = Dynamic.Of(1L); + for (int i = 0; i < 200_000; i++) + { + value = Dynamic.Some(value); + } + + return value; + } + + Assert.That(Deep().Equals(Deep()), Is.True); + Assert.That(Deep().ToString(), Has.Length.LessThanOrEqualTo(4100)); + Assert.That(Dynamic.ListOf(Dynamic.Of(1L), Dynamic.Of(2.5)).ToString(), Is.EqualTo("List(Int(1), Real(2.5))")); + Assert.That(Dynamic.VariantOf("Job", Dynamic.Of("a")).ToString(), Is.EqualTo("Variant(Job, Text(a))")); + Assert.That(Dynamic.Of(double.NaN), Is.EqualTo(Dynamic.Of(double.NaN))); + Assert.That(Dynamic.Of(0.0), Is.Not.EqualTo(Dynamic.Of(-0.0))); + + // Encoding a deep value is refused by the depth limit before it recurses past it. + Assert.That(Refusal(() => Fixture(Limits.Hard).Encode(Codecs.Dynamic, Deep())), Does.Contain("JSON depth")); + } + + /// + /// A constructor name longer than any the SDK maps is neither expanded + /// nor copied whole into a message: it names nothing, and the refusal + /// quotes its first 64 chars and its length. + /// + [Test] + public void AHugeConstructorNameIsRefusedWithoutBeingExpanded() + { + string huge = new('X', 1_000_000); + Assert.That(ReferenceEquals(huge, Identities.DisplayToStore(huge)), Is.True, "the name was converted"); + Assert.That(Refusal(() => Fixture(Limits.Hard).Encode(Codecs.Dynamic, Dynamic.VariantOf(huge))), Is.EqualTo("value: unknown constructor " + new string('X', 64) + "... (1000000 chars)")); + Assert.That(Identities.DisplayToStore("JobTicket"), Is.EqualTo("job-ticket")); + } + + /// + /// Messages quote caller text as an excerpt, and no message is kept + /// past 8,192 chars: a result codec naming a huge constructor, codecs, + /// record readers, and constructors that throw huge messages each give a + /// short refusal that keeps the caller's exception. + /// + [Test] + public void MessagesQuoteCallerTextAsAnExcerpt() + { + string huge = new('X', 1_000_000); + string excerpt = new string('X', 64) + "... (1000000 chars)"; + ProgramData data = Fixture(Limits.Hard); + Value unit = data.Encode(Codecs.Unit, Unit.Value); + JacquardException decode = Support.Throws(() => data.Decode(Codecs.Record(huge, () => 1L), unit)); + Assert.That(decode.Message, Has.Length.LessThan(200)); + Assert.That(decode.Message, Does.Contain("... (1000000 chars)")); + Codec throwing = Codecs.Create((encoder, value) => throw new InvalidOperationException(huge), (decoder, value) => throw new InvalidOperationException(huge)); + JacquardException encode = Support.Throws(() => data.Encode(throwing, 1L)); + Assert.That(encode.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(encode.Message, Is.EqualTo("value: its codec failed: System.InvalidOperationException: " + excerpt)); + Assert.That(encode.InnerException, Is.InstanceOf(), "the codec's exception was not kept"); + JacquardException decoded = Support.Throws(() => data.Decode(throwing, data.Encode(Codecs.Int, 1L))); + Assert.That(decoded.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(decoded.Message, Is.EqualTo("value: its codec failed: System.InvalidOperationException: " + excerpt)); + Assert.That(decoded.InnerException, Is.InstanceOf()); + + // A record constructor that refuses its fields, and a reader that throws. + RecordCodec refusing = Codecs.Record( + "Incident", + Codecs.Field("id", Codecs.Text, (Incident incident) => incident.Id), + Codecs.Field("severity", Codecs.Int, (Incident incident) => throw new ArgumentException(huge)), + Codecs.Field("owner", Codecs.Nullable(Codecs.Text), (Incident incident) => incident.Owner), + (Func)((id, severity, owner) => throw new ArgumentException(huge))); + Value incident = data.Encode(IncidentCodec, new Incident("ok", 1, null)); + JacquardException refused = Support.Throws(() => data.Decode(refusing, incident)); + Assert.That(refused.Message, Is.EqualTo("Incident refused the decoded fields: System.ArgumentException: " + excerpt)); + Assert.That(refused.InnerException, Is.InstanceOf(), "the constructor's refusal was not kept"); + JacquardException reader = Support.Throws(() => data.Encode(refusing, new Incident("ok", 1, null))); + Assert.That(reader.Message, Is.EqualTo("value: cannot read Incident.severity: System.ArgumentException: " + excerpt)); + Assert.That(reader.InnerException, Is.InstanceOf()); + + // A caller's field name in a path. + Codec named = Codecs.Create((encoder, value) => encoder.Field(huge, Codecs.Text, null!), (decoder, value) => 0); + Assert.That(Refusal(() => data.Encode(named, 1L)), Does.StartWith("value." + excerpt + ": null has no Jacquard value")); + string bounded = JacquardException.Bounded(huge); + Assert.That(bounded, Has.Length.EqualTo(JacquardException.MessageLimit), "the cap is exact, its suffix included"); + Assert.That(bounded, Does.EndWith("X... (1000000 chars)")); + + // A caller's message through the public helpers is bounded before it is joined to a path. + Codec invalidating = Codecs.Create((encoder, value) => throw encoder.Invalid(huge), (decoder, value) => throw decoder.Mapping(huge)); + string invalid = Refusal(() => data.Encode(invalidating, 1L)); + Assert.That(invalid, Has.Length.LessThanOrEqualTo(JacquardException.MessageLimit)); + Assert.That(invalid, Does.StartWith("value: XXXX")); + Assert.That(invalid, Does.EndWith("X... (1000007 chars)"), "joined to its path once, the whole length noted"); + string mapping = Refusal(() => data.Decode(invalidating, data.Encode(Codecs.Int, 1L))); + Assert.That(mapping, Has.Length.EqualTo(JacquardException.MessageLimit)); + Assert.That(mapping, Does.EndWith("X... (1000000 chars)")); + + // A note larger than any string is not the SDK's, so it is not trusted. + string forged = new string('Y', JacquardException.MessageLimit - 31) + "... (9223372036854775807 chars)"; + Assert.That(forged, Has.Length.EqualTo(JacquardException.MessageLimit)); + Assert.That(JacquardException.Joined("p: ", forged), Does.EndWith("... (" + (forged.Length + 3).ToString(CultureInfo.InvariantCulture) + " chars)")); + Assert.That(JacquardException.Joined("p: ", null), Is.EqualTo("p: null")); + + // A cut that dropped a high surrogate at the boundary is still recognised as cut. + string emoji = new string('X', JacquardException.MessageLimit - 20) + "😀" + huge; + string cut = JacquardException.Bounded(emoji); + Assert.That(cut, Has.Length.EqualTo(JacquardException.MessageLimit - 1), "the cut did not land on the surrogate"); + Assert.That(JacquardException.Joined("batch 1: ", cut), Does.EndWith("... (" + (emoji.Length + 9).ToString(CultureInfo.InvariantCulture) + " chars)")); + string again = JacquardException.Joined("batch 1: ", mapping); + Assert.That(again, Has.Length.EqualTo(JacquardException.MessageLimit)); + Assert.That(again, Does.StartWith("batch 1: XXX").And.EndWith("X... (1000009 chars)")); + + // A caller's exception whose Message throws is described by its type alone. + Assert.That(Identities.Describe(new ThrowingMessage()), Is.EqualTo(typeof(ThrowingMessage).FullName)); + } + + private sealed class ThrowingMessage : Exception + { + public override string Message => throw new InvalidOperationException("no message"); + } + + internal abstract record Shape; + + internal sealed record Circle : Shape; + + internal sealed record Square : Shape; + + internal sealed record One; + + /// An explicit union whose first case is a narrower union: encoding asks the flattened cases in decoding's order, so both agree. + [Test] + public void EncodingAndDecodingAgreeOnNestedUnions() + { + ProgramData data = Fixture(Limits.Hard); + UnionCodec circle = Codecs.Union(Codecs.Record("routed", () => new Circle())); + UnionCodec outer = Codecs.Union(circle, Codecs.Record("invalid-incident", () => new Square())); + Assert.That(data.Decode(outer, data.Encode(outer, new Square())), Is.EqualTo(new Square())); + Assert.That(data.Decode(outer, data.Encode(outer, new Circle())), Is.EqualTo(new Circle())); + } + + /// + /// Nested unions are flattened on first use by one iterative walk, so + /// decoding asks each distinct case once, and 100,000 layers of a + /// two-case union over one record neither overflow the stack nor take + /// long: one distinct case, and a round trip. + /// + [Test] + public void NestedUnionsFlattenWithoutRecursion() + { + ProgramData data = Fixture(Limits.Hard); + UnionCodec codec = Codecs.Union(Codecs.Record("routed", () => new One())); + for (int i = 0; i < 100_000; i++) + { + codec = Codecs.Union(codec, codec); + } + + Assert.That(codec.LeafCount, Is.EqualTo(1)); + Value notRouted = data.Encode(Codecs.Int, 1L); + Assert.That(Support.Throws(() => data.Decode(codec, notRouted)).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(data.Decode(codec, data.Encode(codec, new One())), Is.EqualTo(new One())); + } + + /// A recursive union, reached through a lazy case, flattens to its distinct cases and round-trips, also from many threads at once. + internal abstract record Expr; + + internal sealed record Leaf(long Value) : Expr; + + internal sealed record Pair(Expr Left, Expr Right) : Expr; + + [Test] + public void ARecursiveUnionThroughALazyCaseRoundTrips() + { + var constructors = new Dictionary { ["leaf"] = Id('1'), ["pair"] = Id('2') }; + var data = new ProgramData(new Dictionary(), constructors, Array.Empty>(), Limits.Hard); + UnionCodec? expr = null; + LazyCodec self = Codecs.Lazy(() => expr!); + expr = Codecs.Union( + Codecs.Record("Leaf", Codecs.Field("value", Codecs.Int, (Leaf leaf) => leaf.Value), value => new Leaf(value)), + Codecs.Record("Pair", Codecs.Field("left", self, (Pair pair) => pair.Left), Codecs.Field("right", self, (Pair pair) => pair.Right), (left, right) => new Pair(left, right)), + self); + Expr tree = new Pair(new Leaf(1), new Pair(new Leaf(2), new Leaf(3))); + Parallel.For(0, 8, _ => Assert.That(data.Decode(expr, data.Encode(expr, tree)), Is.EqualTo(tree))); + Assert.That(expr.LeafCount, Is.EqualTo(2)); + } + + private static string Nested(int depth) => + "(" + string.Concat(Enumerable.Repeat("Option (", depth - 2)) + "Option Int" + new string(')', depth - 2) + ") ->{} Int"; + + /// + /// A signature's types may nest 64 levels, the protocol's value depth, + /// and no deeper. Nesting is refused before the parser recurses past it, + /// so 5,000 parentheses, and 100,000, are refused with a binding failure + /// rather than a stack overflow. + /// + [Test] + public void SignatureNestingIsBounded() + { + var names = new Dictionary { ["int"] = Id('1'), ["option"] = Id('7') }; + Assert.That(Identities.SignatureDepth, Is.EqualTo(64)); + BoundaryType type = Identities.ParseSignature(Nested(64), names).Parameters[0]; + int depth = 1; + while (type is NominalType { Arguments.Count: > 0 } nominal) + { + type = nominal.Arguments[0]; + depth++; + } + + Assert.That(depth, Is.EqualTo(64)); + const string deep = "signature nests types deeper than 64 levels, the protocol's value-depth limit (max_json_depth)"; + foreach (string text in new[] { Nested(65), Nested(2500), "(" + new string('(', 5000) + "Int" + new string(')', 5000) + ") ->{} Int" }) + { + JacquardException error = Support.Throws(() => Identities.ParseSignature(text, names)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(error.Message, Is.EqualTo(deep)); + } + + JacquardException huge = Support.Throws(() => + Identities.ParseSignature("(" + string.Concat(Enumerable.Repeat("Option (", 100_000)) + "Int" + new string(')', 100_000) + ") ->{} Int", names)); + Assert.That(huge.Message, Is.EqualTo("signature is longer than " + Identities.SignatureTokens.ToString(CultureInfo.InvariantCulture) + " tokens")); + string wide = "(" + string.Concat(Enumerable.Repeat("Int, ", Identities.SignatureNodes)) + "Int) ->{} Int"; + Assert.That(Support.Throws(() => Identities.ParseSignature(wide, names)).Message, Is.EqualTo("signature names more than 4096 types")); + } +} From b668519e21d673b23e7e7a0393820b29d15a36cc Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 07:04:33 -0400 Subject: [PATCH 03/52] Add .NET SDK preparation, services, and invocations Runtime.Prepare checks, hashes, and installs sources with Core's public commands into a verified, owner-only cache keyed on the source bytes, the Core executable's bytes, every prelude file, and the protocol, with one version check and prelude discovery per key in the process. Loading derives every binding again from Core's cached raw output and refuses a changed entry. Source counts, path lengths, source bytes, fingerprinting, tree walks, and the failure cleanup are bounded and observe the token. Operations binds typed handlers with the HandlerContext and its token, PreparedProgram binds checked functions with complete services, and Function calls, invokes, and maps explicit batches (each validated first, none retried), with Invocation keeping the raw evidence. --- dotnet/src/JacquardHost/Sdk/Core.cs | 507 +++++++++ dotnet/src/JacquardHost/Sdk/Function.cs | 284 +++++ dotnet/src/JacquardHost/Sdk/Invocation.cs | 125 ++ dotnet/src/JacquardHost/Sdk/Operations.cs | 378 +++++++ dotnet/src/JacquardHost/Sdk/Options.cs | 33 + .../src/JacquardHost/Sdk/PreparedProgram.cs | 301 +++++ dotnet/src/JacquardHost/Sdk/Runtime.cs | 1001 +++++++++++++++++ .../JacquardHost.Tests/Sdk/BatchesTests.cs | 257 +++++ .../test/JacquardHost.Tests/Sdk/FakeCore.cs | 120 ++ .../JacquardHost.Tests/Sdk/OperationsTests.cs | 122 ++ .../JacquardHost.Tests/Sdk/RuntimeTests.cs | 755 +++++++++++++ 11 files changed, 3883 insertions(+) create mode 100644 dotnet/src/JacquardHost/Sdk/Core.cs create mode 100644 dotnet/src/JacquardHost/Sdk/Function.cs create mode 100644 dotnet/src/JacquardHost/Sdk/Invocation.cs create mode 100644 dotnet/src/JacquardHost/Sdk/Operations.cs create mode 100644 dotnet/src/JacquardHost/Sdk/Options.cs create mode 100644 dotnet/src/JacquardHost/Sdk/PreparedProgram.cs create mode 100644 dotnet/src/JacquardHost/Sdk/Runtime.cs create mode 100644 dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs create mode 100644 dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs create mode 100644 dotnet/test/JacquardHost.Tests/Sdk/OperationsTests.cs create mode 100644 dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs diff --git a/dotnet/src/JacquardHost/Sdk/Core.cs b/dotnet/src/JacquardHost/Sdk/Core.cs new file mode 100644 index 0000000..fcaa70f --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Core.cs @@ -0,0 +1,507 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using JacquardLang.Host.Carrier; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// Bounded runs of Core's public commands, digests of prepared files, and the cache's trust boundary. +internal static class Core +{ + /// Each Core command must finish within this time. + internal static readonly TimeSpan CommandTimeout = TimeSpan.FromSeconds(30); + + internal const int StdoutLimit = 8 << 20; + internal const int StderrLimit = 4096; + + /// Why a preparation stopped early. + internal const string Aborted = "preparation was cancelled"; + + /// The most bytes of the Core executable that are fingerprinted. + internal const long ExecutableLimit = 512L << 20; + + /// The most bytes reads under one root. + internal const long TreeLimit = 256L << 20; + + /// The most entries (files and directories, empty ones included) visits under one root. + internal const int TreeEntries = 100_000; + + /// The longest path, in chars, the SDK passes to the file system (Linux's PATH_MAX). + internal const int PathLimit = 4096; + + private const int Chunk = 1 << 16; + + /// Test seam: the entry cap and the failure cleanup apply. + internal static volatile int TreeEntryCap = TreeEntries; + + /// Test seam: runs after each chunk reads. Null outside tests. + internal static volatile Action? DigestChunkHook; + + /// Test seam: runs after digests each file, with its path. Null outside tests. + internal static volatile Action? TreeFileHook; + + /// What does about who may write what it walks. + internal enum Access + { + /// Nothing: a Core installation's prelude. + Any, + + /// Removes every group and other permission, as each new cache entry's files are made owner-only. + Restrict, + + /// Refuses the root or any entry this user does not own or that its group or others may write. + OwnerOnly, + } + + /// + /// Runs one Core subcommand with the prelude set, returning its stdout. + /// The command is bounded by , its output by the + /// limits above, and it stops at once when the token is cancelled. + /// + /// A preparation failure. + internal static string Run(string binary, string prelude, TimeSpan timeout, CancellationToken cancellationToken, params string[] arguments) + { + string name = arguments.Length == 0 ? string.Empty : arguments[0]; + var command = new List { binary }; + command.AddRange(arguments); + CoreCommand.Output output; + try + { + output = CoreCommand.Run( + command, + new Dictionary(StringComparer.Ordinal) { ["JACQUARD_PRELUDE"] = prelude }, + timeout, + StdoutLimit, + StderrLimit, + cancellationToken); + } + catch (CarrierException error) + { + string detail = error.IsTimeout + ? "no exit within " + (timeout.TotalMilliseconds % 1000 == 0 + ? ((long)timeout.TotalSeconds).ToString(CultureInfo.InvariantCulture) + " seconds" + : ((long)timeout.TotalMilliseconds).ToString(CultureInfo.InvariantCulture) + " ms") + : error.IsAborted ? Aborted : error.Message; + throw JacquardException.Preparation("Core " + name + " failed: " + detail); + } + + if (output.ExitStatus != 0) + { + string detail = Encoding.UTF8.GetString(output.Stderr).Trim(); + throw JacquardException.Preparation("Core " + name + " failed: " + + (detail.Length == 0 ? "exit status " + output.ExitStatus.ToString(CultureInfo.InvariantCulture) : detail)); + } + + if (output.StdoutOverflow) + { + throw JacquardException.Preparation("Core " + name + " output exceeds 8 MiB"); + } + + return Utf8(output.Stdout) ?? throw JacquardException.Preparation("Core " + name + " output is not UTF-8"); + } + + private static readonly UTF8Encoding StrictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + + /// The bytes as strict UTF-8 text, or null when they are not. + internal static string? Utf8(byte[] bytes) + { + try + { + return StrictUtf8.GetString(bytes); + } + catch (DecoderFallbackException) + { + return null; + } + } + + /// The SHA-256 digest of bytes, in lowercase hexadecimal. + internal static string Digest(ReadOnlySpan data) => Convert.ToHexString(SHA256.HashData(data)).ToLowerInvariant(); + + /// + /// The SHA-256 digest of the Core executable's bytes, read in pieces. A + /// file larger than is refused from its size + /// before it is read, and the limit is enforced again while it is read. + /// Before each piece it observes the token. + /// + /// A preparation failure. + internal static string FileDigest(string path, CancellationToken cancellationToken) + { + JacquardException over = JacquardException.Preparation("Core executable " + path + " exceeds " + + (ExecutableLimit >> 20).ToString(CultureInfo.InvariantCulture) + " MiB"); + Unix.Status status = StatOrFail(path, follow: true, "cannot read "); + if (status.Kind != Unix.Kind.Regular) + { + throw JacquardException.Preparation("cannot read " + path + ": not a regular file"); + } + + if (status.Size > ExecutableLimit) + { + throw over; + } + + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + byte[] buffer = new byte[Chunk]; + long read = 0; + try + { + using FileStream stream = OpenRead(path); + while (true) + { + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Aborted); + } + + int n = stream.Read(buffer, 0, buffer.Length); + if (n <= 0) + { + break; + } + + read += n; + if (read > ExecutableLimit) + { + throw over; + } + + hash.AppendData(buffer, 0, n); + DigestChunkHook?.Invoke(); + } + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw JacquardException.Preparation("cannot read " + path + ": " + error.Message); + } + + return Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); + } + + /// Options that write a file, created owner-only (read and write for its owner) where the file system has POSIX permissions. + internal static FileStreamOptions OwnerOnly(FileMode mode) + { + var options = new FileStreamOptions { Mode = mode, Access = FileAccess.Write }; + if (!OperatingSystem.IsWindows()) + { + options.UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite; + } + + return options; + } + + private static FileStream OpenRead(string path) => + new(path, new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Share = FileShare.ReadWrite, BufferSize = 0 }); + + /// The status of , or a preparation failure starting with . + internal static Unix.Status StatOrFail(string path, bool follow, string context) + { + Unix.Status status; + try + { + status = Unix.Stat(path, follow); + } + catch (IOException error) + { + throw JacquardException.Preparation(context + path + ": " + error.Message); + } + + if (status.Kind == Unix.Kind.Missing) + { + throw JacquardException.Preparation(context + path + ": no such file or directory"); + } + + return status; + } + + /// + /// Digests every regular file under , keyed by + /// /-separated relative path, refusing other entry kinds (symbolic + /// links, FIFOs, and devices included, so no read can block on one), more + /// than entries, and more than + /// bytes in all, which is checked against each + /// file's size before it is read and again while it is read. The walk + /// streams each directory and keeps an explicit stack, so nothing beyond + /// the entry cap is held and no directory depth can exhaust the call + /// stack; before each entry and each piece of a file it observes the + /// token. says whether entries are also checked + /// for owner-only access, or made owner-only. + /// + /// A preparation failure. + internal static SortedDictionary Tree(string root, Access access, CancellationToken cancellationToken) + { + var files = new SortedDictionary(Json.CodePointOrder); + Check(root, access); + long total = 0; + int entries = 0; + int cap = TreeEntryCap; + var stack = new Stack>(); + try + { + stack.Push(Directory.EnumerateFileSystemEntries(root).GetEnumerator()); + while (stack.Count > 0) + { + IEnumerator top = stack.Peek(); + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Aborted); + } + + if (!top.MoveNext()) + { + stack.Pop().Dispose(); + continue; + } + + string path = top.Current; + if (++entries > cap) + { + throw JacquardException.Preparation("more than " + cap.ToString(CultureInfo.InvariantCulture) + " entries under " + root); + } + + string relative = Path.GetRelativePath(root, path).Replace(Path.DirectorySeparatorChar, '/'); + Unix.Status status = StatOrFail(path, follow: false, "cannot read "); + if (status.Kind == Unix.Kind.Directory) + { + Check(path, access); + stack.Push(Directory.EnumerateFileSystemEntries(path).GetEnumerator()); + } + else if (status.Kind == Unix.Kind.Regular) + { + Check(path, access); + JacquardException over = JacquardException.Preparation("files under " + root + " exceed " + + (TreeLimit >> 20).ToString(CultureInfo.InvariantCulture) + " MiB"); + if (status.Size > TreeLimit - total) + { + throw over; + } + + files[relative] = BoundedDigest(path, TreeLimit - total, ref total, over, cancellationToken); + TreeFileHook?.Invoke(path); + } + else + { + throw JacquardException.Preparation("unsupported filesystem entry in prepared artifacts: " + relative); + } + } + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw JacquardException.Preparation("cannot read " + root + ": " + error.Message); + } + finally + { + while (stack.Count > 0) + { + stack.Pop().Dispose(); + } + } + + return files; + } + + /// The digest of at most bytes of a file, adding what it read to . + private static string BoundedDigest(string path, long room, ref long total, JacquardException over, CancellationToken cancellationToken) + { + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + byte[] buffer = new byte[Chunk]; + long read = 0; + using (FileStream stream = OpenRead(path)) + { + int n; + while ((n = stream.Read(buffer, 0, buffer.Length)) > 0) + { + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Aborted); + } + + read += n; + if (read > room) + { + throw over; + } + + hash.AppendData(buffer, 0, n); + } + } + + total += read; + return Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); + } + + /// Canonical JSON for a string map: keys in code-point order. + internal static string Canonical(IEnumerable> map) + { + var fields = new List>(); + foreach (KeyValuePair pair in map) + { + fields.Add(new(pair.Key, Json.Of(pair.Value))); + } + + return JsonCodec.Stringify(Json.Object(fields), 4); + } + + /// The digest of a string map's canonical JSON. + internal static string MapDigest(IEnumerable> map) => Digest(Encoding.UTF8.GetBytes(Canonical(map))); + + /// + /// , refused when it is empty, holds a NUL (no path + /// does), or is longer than any path, as given or made absolute, before + /// the file system or any message sees it whole. + /// + /// A preparation failure quoting an excerpt. + internal static string CheckedPath(string? path, string what) + { + if (path is null) + { + throw new ArgumentNullException(nameof(path), what + " is null"); + } + + if (path.Length == 0 || path.Contains('\0', StringComparison.Ordinal)) + { + throw JacquardException.Preparation(what + " " + Identities.Excerpt(path) + " is not a path"); + } + + // As given first, so a huge path is not copied again; then as the + // file system will see it, since a relative path grows. + if (path.Length > PathLimit || AbsoluteLength(path) > PathLimit) + { + throw JacquardException.Preparation(what + " " + Identities.Excerpt(path) + " is longer than any path"); + } + + return path; + } + + /// The length of made absolute, or one past the limit when it cannot be. + internal static int AbsoluteLength(string path) + { + try + { + return Path.GetFullPath(path).Length; + } + catch (Exception error) when (error is ArgumentException or NotSupportedException or PathTooLongException or System.Security.SecurityException) + { + return PathLimit + 1; + } + } + + /// Finds an executable the way a shell would, then resolves a final symbolic link. + /// A preparation failure for a missing executable. + internal static string Executable(string binary) + { + JacquardException Missing() => JacquardException.Preparation("cannot find Core executable " + Identities.Excerpt(binary) + + "; configure JACQUARD_BIN using the setup guide"); + + // No path is this long (PATH_MAX is 4096 on Linux), and none holds a + // NUL, so such a name is refused before it is used. + if (binary.Length > PathLimit || binary.Contains('\0', StringComparison.Ordinal)) + { + throw Missing(); + } + + string? candidate = null; + if (binary.Contains('/', StringComparison.Ordinal)) + { + candidate = binary; + } + else + { + // Like Go's exec.LookPath, relative entries (including empty ones, + // which mean the current directory) are not searched. + foreach (string entry in (Environment.GetEnvironmentVariable("PATH") ?? string.Empty).Split(':')) + { + if (Path.IsPathRooted(entry) && IsExecutable(Path.Combine(entry, binary))) + { + candidate = Path.Combine(entry, binary); + break; + } + } + } + + if (candidate is null || !IsExecutable(candidate)) + { + throw Missing(); + } + + try + { + FileSystemInfo? target = new FileInfo(candidate).ResolveLinkTarget(returnFinalTarget: true); + return Path.GetFullPath(target?.FullName ?? candidate); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException or ArgumentException) + { + throw JacquardException.Preparation("cannot resolve Core executable " + Identities.Excerpt(binary) + ": " + error.Message); + } + } + + private static bool IsExecutable(string path) + { + if (path.Length > PathLimit) + { + return false; + } + + try + { + Unix.Status status = Unix.Stat(path, follow: true); + return status.Kind == Unix.Kind.Regular && (status.Mode & 0x49) != 0; + } + catch (IOException) + { + return false; + } + } + + /// + /// Applies to , which is + /// not followed if it is a link: refuses it unless this user owns it and + /// neither its group nor others may write it, or removes every group and + /// other permission. Nothing inside a cache entry can resist someone who + /// may rewrite it, so the guarantee is that only the cache's owner can. + /// + /// A preparation failure. + internal static void Check(string path, Access access) + { + if (access == Access.Any || !Unix.Supported) + { + return; + } + + Unix.Status status = StatOrFail(path, follow: false, "cannot read "); + if (access == Access.Restrict) + { + if ((status.Mode & 0x3F) != 0) + { + try + { + if (!OperatingSystem.IsWindows()) + { + File.SetUnixFileMode(path, (UnixFileMode)(status.Mode & 0x1C0)); + } + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw JacquardException.Preparation("cannot make " + path + " owner-only: " + error.Message); + } + } + + return; + } + + if (status.Owner != Unix.EffectiveUser()) + { + throw JacquardException.Preparation("cache path " + path + " is not owned by the current user; only its owner may write the cache"); + } + + if ((status.Mode & 0x12) != 0) + { + throw JacquardException.Preparation("cache path " + path + " is group- or world-writable; only its owner may write the cache" + + " (make it owner-only, for example chmod " + (status.Kind == Unix.Kind.Directory ? "700 " : "600 ") + path + ")"); + } + } +} diff --git a/dotnet/src/JacquardHost/Sdk/Function.cs b/dotnet/src/JacquardHost/Sdk/Function.cs new file mode 100644 index 0000000..7d8c461 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Function.cs @@ -0,0 +1,284 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// A bound checked function. It is immutable and thread-safe; each call runs +/// one fresh, serial worker. Arguments are s: plain +/// values convert implicitly, and anything else is passed as +/// codec.Arg(value). Before any worker starts they are checked +/// against the function's checked signature, the selected limits, and the +/// frame size. +/// +public sealed class Function +{ + private readonly Hash target; + private readonly OperationRegistry registry; + private readonly Host host; + + internal Function(string name, CallInterface signature, Hash target, OperationRegistry registry, Host host, ProgramData data) + { + Name = name; + Signature = signature; + this.target = target; + this.registry = registry; + this.host = host; + Data = data; + } + + /// Test seam: runs after the arguments are measured, just before their types are checked. Null outside tests. + internal static volatile Action? ConformHook; + + /// The public name. + public string Name { get; } + + /// The checked signature. + public CallInterface Signature { get; } + + internal ProgramData Data { get; } + + /// + /// Refuses a call with the wrong number of arguments from the count + /// alone, before anything is copied or encoded, so the work below is + /// bounded by the checked signature, not by what the caller passed. + /// + private void Arity(long received) + { + int expected = Signature.Parameters.Count; + if (received != expected) + { + throw JacquardException.Invalid(Name + ": expected " + expected.ToString(CultureInfo.InvariantCulture) + " arguments, received " + + received.ToString(CultureInfo.InvariantCulture)); + } + } + + private string ArgumentPath(int index) => Name + ".argument[" + index.ToString(CultureInfo.InvariantCulture) + "]"; + + /// Encodes , one frame's worth, with one shared budget. + private List Encode(IReadOnlyList args) + { + Arity(args.Count); + var budget = new Encoder.Budget(); + var values = new List(args.Count); + for (int i = 0; i < args.Count; i++) + { + string path = ArgumentPath(i); + Encoder encoder = Encoder.Root(Data, path, budget); + if (args[i] is not { } argument) + { + throw encoder.Invalid("null has no Jacquard value; pass codec.Arg(value)"); + } + + try + { + values.Add(argument.Encode(encoder)); + } + catch (Exception error) when (Caller.Wraps(error)) + { + // A caller's codec failed; no worker starts, as for any refusal. + throw ProgramData.CodecFailed(path, error); + } + } + + return values; + } + + /// + /// Validates encoded arguments against the signature and the frame + /// limits, before any worker starts. Every value is first measured under + /// one shared budget (nodes, depth, text bytes, collection items), + /// whatever codec built it, so the type check below walks at most one + /// frame's worth of values. + /// + private List Check(List values) + { + Arity(values.Count); + var measured = new Encoder.Budget(); + for (int i = 0; i < values.Count; i++) + { + Encoder.Root(Data, ArgumentPath(i), measured).Raw(values[i]); + } + + ConformHook?.Invoke(); + for (int i = 0; i < values.Count; i++) + { + Data.Conform(values[i], Signature.Parameters[i], ArgumentPath(i)); + } + + Limits limits = host.Limits; + try + { + JsonObject frame = Protocol.Codec.InvokeFrame(target, Signature, values, registry.Entries(), limits); + JsonCodec.FrameBytes(frame, limits.MaxFrameBytes, limits.MaxJsonDepth); + } + catch (ProtocolException error) + { + throw JacquardException.Invalid(Name + ": " + error.Message + "; reduce input size or use smaller independent batches"); + } + + return values; + } + + private Invocation Run(List values, CancellationToken cancellationToken) => + new(host.Invoke(target, Signature, values, registry, cancellationToken), this); + + /// + /// Validates the arguments locally, then runs one worker. The exception + /// is only a local refusal; Core and carrier failures are in the + /// . Nothing is retried. + /// + /// An invalid argument; no worker started. + public Invocation Invoke(params Argument[] args) => Invoke((IReadOnlyList)args, default); + + /// + /// ; cancelling + /// kills the worker. + /// + /// An invalid argument; no worker started. + public Invocation Invoke(IReadOnlyList args, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(args); + return Run(Check(Encode(args)), cancellationToken); + } + + /// Invokes the function and decodes its result. + /// + /// An invalid argument, a failed invocation, or a result that does not + /// decode (which keeps the successful raw result). + /// + public TResult Call(Codec result, params Argument[] args) => Call(result, (IReadOnlyList)args, default); + + /// ; cancelling the token kills the worker. + /// An invalid argument, a failed invocation, or a result that does not decode. + public TResult Call(Codec result, IReadOnlyList args, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(result); + return Invoke(args, cancellationToken).Decode(result); + } + + /// + /// Runs a pure function whose final parameter and result are Lists over + /// explicit batches of , with + /// as the other arguments, and concatenates the + /// decoded results. Every batch is validated before the first runs, one + /// at a time: validation returns nothing, so each batch's encoding is + /// unreachable once it is checked, and the batch is encoded and checked + /// again just before it runs. Before each batch, and once more after the + /// last, the validation observes the token. Validation has no deadline of + /// its own, as in the other SDKs. Nothing is retried. + /// + /// + /// A refusal before any worker started ( + /// is -1): an invalid batch, or a cancellation, which is reported as the + /// host's aborted invocation without a worker; or the failure of one + /// batch, with the decoded values of the batches before it in + /// . + /// + public IReadOnlyList MapBatches( + Codec item, + Codec row, + IReadOnlyList rows, + int batchSize, + IReadOnlyList? leading = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(item); + ArgumentNullException.ThrowIfNull(row); + ArgumentNullException.ThrowIfNull(rows); + leading ??= Array.Empty(); + if (batchSize < 1) + { + throw JacquardException.Invalid("batch size must be positive"); + } + + Hash? list = Data.Name("list"); + IReadOnlyList parameters = Signature.Parameters; + if (Signature.Effects.Count != 0 || parameters.Count == 0 || !IsList(parameters[^1], list) || !IsList(Signature.Result, list)) + { + throw JacquardException.Binding("MapBatches needs a pure function with a final List parameter and a List result"); + } + + // From the count alone, before any leading argument is copied. + Arity(leading.Count + 1L); + List empty = Batch(leading, row, rows, 0, 0); + int count = (int)((rows.Count + (long)batchSize - 1) / batchSize); + for (int index = 0; index < count; index++) + { + RefuseIfStopped(empty, cancellationToken); + Validate(leading, row, rows, index, batchSize); + } + + // A cancellation during the last batch's validation starts no worker. + RefuseIfStopped(empty, cancellationToken); + var completed = new List(); + Codec> results = Codecs.List(item); + for (int index = 0; index < count; index++) + { + try + { + (int start, int size) = Range(rows.Count, index, batchSize); + completed.AddRange(Run(Batch(leading, row, rows, start, size), cancellationToken).Decode(results)); + } + catch (JacquardException error) + { + var done = new object?[completed.Count]; + for (int i = 0; i < done.Length; i++) + { + done[i] = completed[i]; + } + + throw JacquardException.Batch(index, done, error); + } + } + + return completed; + } + + /// Throws the host's aborted invocation, which starts no worker, once the token is cancelled. + private void RefuseIfStopped(List empty, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + { + // The host observes the cancellation before it spawns a worker. + Invocation stopped = Run(empty, cancellationToken); + if (!stopped.Ok) + { + throw JacquardException.Invocation(stopped.Raw); + } + } + } + + /// Encodes and checks one batch, keeping nothing: the encoding is unreachable once this returns. + private void Validate(IReadOnlyList leading, Codec row, IReadOnlyList rows, int index, int batchSize) + { + (int start, int size) = Range(rows.Count, index, batchSize); + Batch(leading, row, rows, start, size); + } + + private static (int Start, int Size) Range(int total, int index, int batchSize) + { + long start = (long)index * batchSize; + return ((int)start, (int)Math.Min(total - start, batchSize)); + } + + private List Batch(IReadOnlyList leading, Codec row, IReadOnlyList rows, int start, int size) + { + var args = new Argument[leading.Count + 1]; + for (int i = 0; i < leading.Count; i++) + { + args[i] = leading[i]; + } + + args[^1] = new Rows(row, rows, start, size); + return Check(Encode(args)); + } + + private static bool IsList(BoundaryType type, Hash? list) => list is not null && type is NominalType nominal && nominal.Identity.Equals(list); + + /// + public override string ToString() => "Function[" + Name + ", " + target.Hex + "]"; +} diff --git a/dotnet/src/JacquardHost/Sdk/Invocation.cs b/dotnet/src/JacquardHost/Sdk/Invocation.cs new file mode 100644 index 0000000..028c63b --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Invocation.cs @@ -0,0 +1,125 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// One completed invocation with its raw evidence. +public sealed class Invocation +{ + private readonly Function function; + + internal Invocation(InvocationResult raw, Function function) + { + Raw = raw; + this.function = function; + } + + /// The host's result: Core's frames, the host's frames, and evidence. + public InvocationResult Raw { get; } + + /// Whether Core reported success. + public bool Ok => Raw.Ok; + + /// The primary diagnostic or host code. + public string Code => Raw.Code; + + /// A summary for people. + public string Message => Raw.Message; + + /// The classified failure, or null on success. + public JacquardException? Error => Ok ? null : JacquardException.Invocation(Raw); + + /// Decodes the result. + /// + /// The invocation's failure, or a binding failure keeping the successful + /// raw result when it does not decode; when the caller's codec or record + /// constructor threw, that exception is the inner exception. + /// + public T Decode(Codec codec) + { + ArgumentNullException.ThrowIfNull(codec); + if (Error is { } error) + { + throw error; + } + + Value value = Raw.Value ?? throw JacquardException.Binding(function.Name + ": Core completed without a value", Raw, null); + try + { + return function.Data.Decode(codec, value); + } + catch (JacquardException mapping) + { + // Already bounded, and the SDK's own words. + throw JacquardException.Binding( + JacquardException.Joined(function.Name + ": Core completed, but result mapping failed: ", mapping.Message), + Raw, + mapping.InnerException ?? mapping); + } + } + + /// Decodes the result into . + /// As for . + public Dynamic ToDynamic() => Decode(Codecs.Dynamic); + + /// Lists the outcome and each effect request by operation name. + public string Render() + { + var lines = new List { function.Name + ": " + (Ok ? "ok" : Message) }; + foreach (EffectRequest request in Raw.Requests) + { + lines.Add(" " + request.Ordinal.ToString(CultureInfo.InvariantCulture) + ". " + (function.Data.Label(request.Operation) ?? request.Operation.Hex)); + } + + return string.Join("\n", lines); + } + + /// + /// Writes the invocation's frames and host evidence. The record contains + /// argument and result values; it is written only when called. A new file + /// is created owner-only; an existing file keeps its permissions. + /// + /// The file cannot be written, or the path is longer than any path. + /// The file may not be written. + public void Save(string path) + { + ArgumentNullException.ThrowIfNull(path); + if (path.Length == 0 || path.Contains('\0', StringComparison.Ordinal) || path.Length > Core.PathLimit || Core.AbsoluteLength(path) > Core.PathLimit) + { + throw new IOException(Identities.Excerpt(path) + " is not a path or is longer than any path"); + } + + JsonObject host = Json.Object( + ("evidence", Raw.HostEvidence), + ("exit_status", Raw.ExitStatus is { } status ? Json.Of(status) : Json.Null), + ("signal", Raw.Signal is { } signal ? Json.Of(signal) : Json.Null), + ("stderr_base64", Json.Of(Convert.ToBase64String(Raw.Stderr.ToArray())))); + JsonObject record = Json.Object( + ("core_frames", Json.Of(Raw.CoreFrames)), + ("function", Json.Of(function.Name)), + ("host", host), + ("host_frames", Json.Of(Raw.HostFrames)), + ("schema", Json.Of("jacquard-host-inspection-v1"))); + string text; + try + { + text = JsonCodec.Stringify(record, 128); + } + catch (ProtocolException error) + { + throw new IOException(error.Message, error); + } + + byte[] bytes = Encoding.UTF8.GetBytes(text + "\n"); + using var stream = new FileStream(path, Core.OwnerOnly(FileMode.Create)); + stream.Write(bytes); + } + + /// + public override string ToString() => "Invocation[" + function.Name + ", " + Raw + "]"; +} diff --git a/dotnet/src/JacquardHost/Sdk/Operations.cs b/dotnet/src/JacquardHost/Sdk/Operations.cs new file mode 100644 index 0000000..d300add --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Operations.cs @@ -0,0 +1,378 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// One effect's service: operation names mapped to .NET handlers. Names are +/// store spellings such as "lookup-stock", or display spellings; +/// _ is accepted for -. Each handler declares a codec per +/// parameter and one for its result, and may take the +/// first, whose +/// is cancelled when the invocation stops or ends: +/// +/// var inventory = new Operations() +/// .On("lookup-stock", Codecs.Text, Codecs.Int, sku => sku == "sensor" ? 20L : 0L) +/// .On("reserve", Codecs.Text, Codecs.Int, Codecs.Unit, (context, sku, units) => +/// { +/// context.CancellationToken.ThrowIfCancellationRequested(); +/// return Unit.Value; +/// }); +/// +/// +/// +/// A request whose arguments do not decode into the parameters, or whose +/// argument count differs, is reported as an outside failure and the handler +/// does not run. Nor does it run once the invocation is cancelled or its +/// lifetime passed while the arguments were decoded. A +/// is reported as that failure; any other +/// exception is reported as completion unknown, without its message. A +/// result with no encoding within the selected limits is reported as an +/// outside failure, after the handler returned. The service is thread-safe. +/// +public sealed class Operations +{ + private readonly SortedDictionary handlers = new(StringComparer.Ordinal); + private readonly object sync = new(); + + /// + /// Test seam: runs on the handler thread after the arguments are decoded + /// and before the stop check that precedes the handler. Null outside tests. + /// + internal static volatile Action? AfterDecodeHook; + + /// A typed handler with its codecs applied. + internal delegate Value Handler(ProgramData data, HandlerContext context, IReadOnlyList arguments); + + /// One parameter's codec, decoding to a boxed value. + private interface ICodecBox + { + object? Decode(Decoder decoder, Value value); + } + + /// How many operations this service binds, without copying them. + internal int Count + { + get + { + lock (sync) + { + return handlers.Count; + } + } + } + + /// + /// The handlers by store-spelled operation name, or null when there are + /// more than : the count is checked before + /// anything is copied, under the same lock as the copy. + /// + internal IReadOnlyDictionary? Snapshot(int limit) + { + lock (sync) + { + return handlers.Count > limit ? null : new ReadOnlyDictionary(new Dictionary(handlers, StringComparer.Ordinal)); + } + } + + private sealed class CodecBox : ICodecBox + { + private readonly Codec codec; + + internal CodecBox(Codec codec) + { + this.codec = codec; + } + + public object? Decode(Decoder decoder, Value value) => codec.Decode(decoder, value); + } + + private static CodecBox Box(Codec codec) => new CodecBox(codec ?? throw new ArgumentNullException(nameof(codec))); + + private Operations Bind(string operation, ICodecBox[] parameters, Codec result, Func invoke) + { + ArgumentNullException.ThrowIfNull(operation); + ArgumentNullException.ThrowIfNull(result); + Handler handler = (data, context, arguments) => + { + if (arguments.Count != parameters.Length) + { + throw HandlerException.OutsideFailed("handler parameter count differs from the request"); + } + + var decoder = new Decoder(data); + object?[] values = new object?[parameters.Length]; + for (int i = 0; i < values.Length; i++) + { + try + { + values[i] = parameters[i].Decode(decoder, arguments[i]); + } + catch (Exception error) when (error is not OutOfMemoryException) + { + throw HandlerException.OutsideFailed("handler argument cannot be mapped"); + } + } + + AfterDecodeHook?.Invoke(); + + // Decoding takes time; no outside action begins once the call stopped. + if (context.IsCancelled) + { + throw HandlerException.OutsideFailed("handler was not started"); + } + + TResult value = invoke(context, values); + try + { + return Encoder.Root(data, "result").Encode(result, value); + } + catch (Exception error) when (error is not OutOfMemoryException) + { + throw HandlerException.OutsideFailed("handler result cannot be represented"); + } + }; + string key = Identities.Normal(operation); + lock (sync) + { + handlers[key] = handler; + } + + return this; + } + + /// + /// Serves with a handler of 0 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, Array.Empty(), result, (context, values) => handler()); + } + + /// + /// Serves with a handler that takes the first, of 0 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, Array.Empty(), result, (context, values) => handler(context)); + } + + /// + /// Serves with a handler of 1 decoded parameter, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first) }, result, (context, values) => handler((TA)values[0]!)); + } + + /// + /// Serves with a handler that takes the first, of 1 decoded parameter, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first) }, result, (context, values) => handler(context, (TA)values[0]!)); + } + + /// + /// Serves with a handler of 2 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second) }, result, (context, values) => handler((TA)values[0]!, (TB)values[1]!)); + } + + /// + /// Serves with a handler that takes the first, of 2 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second) }, result, (context, values) => handler(context, (TA)values[0]!, (TB)values[1]!)); + } + + /// + /// Serves with a handler of 3 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third) }, result, (context, values) => handler((TA)values[0]!, (TB)values[1]!, (TC)values[2]!)); + } + + /// + /// Serves with a handler that takes the first, of 3 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third) }, result, (context, values) => handler(context, (TA)values[0]!, (TB)values[1]!, (TC)values[2]!)); + } + + /// + /// Serves with a handler of 4 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Decodes parameter 3. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec fourth, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third), Box(fourth) }, result, (context, values) => handler((TA)values[0]!, (TB)values[1]!, (TC)values[2]!, (TD)values[3]!)); + } + + /// + /// Serves with a handler that takes the first, of 4 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Decodes parameter 3. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec fourth, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third), Box(fourth) }, result, (context, values) => handler(context, (TA)values[0]!, (TB)values[1]!, (TC)values[2]!, (TD)values[3]!)); + } + + /// + /// Serves with a handler of 5 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Decodes parameter 3. + /// Decodes parameter 4. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec fourth, Codec fifth, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third), Box(fourth), Box(fifth) }, result, (context, values) => handler((TA)values[0]!, (TB)values[1]!, (TC)values[2]!, (TD)values[3]!, (TE)values[4]!)); + } + + /// + /// Serves with a handler that takes the first, of 5 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Decodes parameter 3. + /// Decodes parameter 4. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec fourth, Codec fifth, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third), Box(fourth), Box(fifth) }, result, (context, values) => handler(context, (TA)values[0]!, (TB)values[1]!, (TC)values[2]!, (TD)values[3]!, (TE)values[4]!)); + } + + /// + /// Serves with a handler of 6 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Decodes parameter 3. + /// Decodes parameter 4. + /// Decodes parameter 5. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec fourth, Codec fifth, Codec sixth, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third), Box(fourth), Box(fifth), Box(sixth) }, result, (context, values) => handler((TA)values[0]!, (TB)values[1]!, (TC)values[2]!, (TD)values[3]!, (TE)values[4]!, (TF)values[5]!)); + } + + /// + /// Serves with a handler that takes the first, of 6 decoded parameters, replacing any earlier handler for it. + /// + /// The operation's name. + /// Decodes parameter 0. + /// Decodes parameter 1. + /// Decodes parameter 2. + /// Decodes parameter 3. + /// Decodes parameter 4. + /// Decodes parameter 5. + /// Encodes the handler's result ( for unit). + /// The handler. + /// This service. + public Operations On(string operation, Codec first, Codec second, Codec third, Codec fourth, Codec fifth, Codec sixth, Codec result, Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + return Bind(operation, new ICodecBox[] { Box(first), Box(second), Box(third), Box(fourth), Box(fifth), Box(sixth) }, result, (context, values) => handler(context, (TA)values[0]!, (TB)values[1]!, (TC)values[2]!, (TD)values[3]!, (TE)values[4]!, (TF)values[5]!)); + } + + /// + public override string ToString() + { + lock (sync) + { + return "Operations[" + string.Join(", ", handlers.Keys) + "]"; + } + } +} diff --git a/dotnet/src/JacquardHost/Sdk/Options.cs b/dotnet/src/JacquardHost/Sdk/Options.cs new file mode 100644 index 0000000..7b4e75d --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Options.cs @@ -0,0 +1,33 @@ +using System; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// Options for Core discovery and invocation bounds. A null or empty value means unset. +public sealed record RuntimeOptions +{ + /// Every option unset. + public static RuntimeOptions Defaults { get; } = new(); + + /// The Core executable; defaults to $JACQUARD_BIN, then jacquard on the PATH. + public string? Binary { get; init; } + + /// Core's prelude directory; defaults to $JACQUARD_PRELUDE. + public string? Prelude { get; init; } + + /// Bounds each invocation's worker lifetime; null means the host's default (30 seconds). + public TimeSpan? Deadline { get; init; } + + /// The session limits; null selects the hard limits. + public Limits? Limits { get; init; } +} + +/// Options for one preparation. +public sealed record PrepareOptions +{ + /// The default cache directory. + public static PrepareOptions Defaults { get; } = new(); + + /// Holds verified prepared programs; null means . + public string? CacheDir { get; init; } +} diff --git a/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs b/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs new file mode 100644 index 0000000..bb11311 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs @@ -0,0 +1,301 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using JacquardLang.Host.Carrier; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// A prepared, verified program. Bind its checked functions with +/// . +/// changes the services of functions bound afterwards; +/// bound functions are immutable and thread-safe. +/// +public sealed class PreparedProgram +{ + private readonly object sync = new(); + private readonly SortedDictionary interfaces = new(StringComparer.Ordinal); + private readonly SortedDictionary effects = new(StringComparer.Ordinal); + private readonly SortedDictionary services = new(StringComparer.Ordinal); + + /// Why each checked function without a binding has none. + private readonly SortedDictionary unsupported = new(StringComparer.Ordinal); + + private Host host; + + [SuppressMessage("Design", "CA1031", Justification = "No exception from reading one signature may fail the preparation; the function is left unbound with the reason.")] + internal PreparedProgram(Runtime runtime, string directory, Runtime.Bindings bindings, string executable) + { + Runtime = runtime; + Directory = directory; + var names = new Dictionary(StringComparer.Ordinal); + foreach (Identities.Entry entry in bindings.Names) + { + names[entry.Name] = entry.Hash; + } + + var constructors = new Dictionary(StringComparer.Ordinal); + foreach (Identities.Entry entry in bindings.Constructors) + { + constructors[entry.Name] = entry.Hash; + } + + var environment = new Dictionary(StringComparer.Ordinal); + foreach (DictionaryEntry variable in Environment.GetEnvironmentVariables()) + { + if (variable.Key is string key && variable.Value is string value) + { + environment[key] = value; + } + } + + environment["JACQUARD_PRELUDE"] = runtime.Prelude; + var carrier = new WorkerOptions { Command = new[] { executable }, Environment = environment, Deadline = runtime.Deadline }; + try + { + host = new Host(Path.Combine(directory, "store"), new HostOptions { Carrier = carrier, Limits = runtime.Limits }); + } + catch (ArgumentException error) + { + throw JacquardException.Binding(error.Message); + } + + var used = new HashSet(); + foreach (KeyValuePair signature in bindings.Signatures) + { + try + { + CallInterface parsed = Identities.ParseSignature(signature.Value, names); + used.UnionWith(parsed.Effects); + interfaces[signature.Key] = parsed; + } + catch (JacquardException reason) + { + // Polymorphic, too deep, and other unsupported signatures + // have no binding; the other functions keep theirs. + unsupported[signature.Key] = reason.Message; + } + catch (Exception unexpected) when (unexpected is not OutOfMemoryException) + { + // The parser's bounds make this unreachable; nothing escapes preparation either way. + unsupported[signature.Key] = "signature cannot be read (" + Identities.Describe(unexpected) + ")"; + } + } + + foreach (IReadOnlyList group in bindings.Groups) + { + foreach (Identities.Entry entry in group) + { + if (used.Contains(entry.Hash)) + { + effects[entry.Name] = new EffectGroup(entry.Hash, group.Where(other => !other.Hash.Equals(entry.Hash)).ToList()); + } + } + } + + Data = new ProgramData(names, constructors, bindings.Groups, runtime.Limits); + } + + private sealed record EffectGroup(Hash Identity, IReadOnlyList Operations); + + /// The runtime that prepared this program. + public Runtime Runtime { get; } + + /// The verified cache entry. + public string Directory { get; } + + /// The program's names and identities. + public ProgramData Data { get; } + + /// The checked functions that have a binding, by public name. + public IReadOnlyList Functions + { + get + { + lock (sync) + { + return interfaces.Keys.ToList(); + } + } + } + + /// Test seam: replaces the worker command for functions bound afterwards. + internal void SetWorkerCommand(IReadOnlyList command, IReadOnlyDictionary? environment = null) + { + lock (sync) + { + host = new Host(host.Store, new HostOptions + { + Carrier = new WorkerOptions { Command = command, Environment = environment, Deadline = Runtime.Deadline }, + Limits = host.Limits, + }); + } + } + + /// + /// Configures an effect's service, by effect name such as + /// "Inventory", for every function bound afterwards. + /// + /// A binding failure for an effect no checked function uses. + public PreparedProgram Provide(string effect, Operations operations) + { + ArgumentNullException.ThrowIfNull(effect); + ArgumentNullException.ThrowIfNull(operations); + lock (sync) + { + string key = Identities.Normal(effect); + if (!effects.ContainsKey(key)) + { + throw JacquardException.Binding("unknown effect " + Identities.Excerpt(effect) + (unsupported.Count == 0 ? string.Empty + : "; the effects of functions with unsupported signatures (" + string.Join(", ", unsupported.Keys.Select(Identities.Excerpt)) + ") are not bound")); + } + + services[key] = operations; + return this; + } + } + + /// + /// Binds a checked public function with services for its effects, by + /// effect name, overriding . Every operation of each + /// checked root effect must have a handler, and every name a service + /// binds must be an operation of the program. + /// + /// + /// A binding failure for an unknown function or effect, a missing + /// handler, or a name that is no operation. + /// + public Function Function(string name, IReadOnlyDictionary? bindings = null) + { + ArgumentNullException.ThrowIfNull(name); + bindings ??= new Dictionary(); + lock (sync) + { + if (!interfaces.TryGetValue(name, out CallInterface? signature)) + { + string? reason = unsupported.TryGetValue(name, out string? why) ? why : null; + throw JacquardException.Binding(Identities.Excerpt(name) + ": absent or unsupported closed v0 signature" + (reason is null ? string.Empty : ": " + reason)); + } + + Hash target = Data.Name(name) ?? throw JacquardException.Binding(Identities.Excerpt(name) + ": no public identity"); + var selected = new SortedDictionary(StringComparer.Ordinal); + foreach (KeyValuePair effect in effects) + { + if (signature.Effects.Contains(effect.Value.Identity)) + { + selected[effect.Key] = effect.Value; + } + } + + if (selected.Count != signature.Effects.Count) + { + throw JacquardException.Binding(name + ": effect metadata is unavailable; use the low-level host package"); + } + + // Each binding must name one of the function's effects, so more + // bindings than effects are refused from the count, before any is read. + if (bindings.Count > selected.Count) + { + throw JacquardException.Binding(name + ": " + bindings.Count.ToString(CultureInfo.InvariantCulture) + " bindings given, but it uses " + + selected.Count.ToString(CultureInfo.InvariantCulture) + " effect" + (selected.Count == 1 ? string.Empty : "s") + + (selected.Count == 0 ? string.Empty : ": " + string.Join(", ", selected.Keys))); + } + + var chosen = new SortedDictionary(services, StringComparer.Ordinal); + int read = 0; + foreach (KeyValuePair binding in bindings) + { + // A dictionary whose count understated its entries is not read past the bound. + if (++read > selected.Count) + { + throw JacquardException.Binding(name + ": more bindings than effects"); + } + + ArgumentNullException.ThrowIfNull(binding.Key, nameof(bindings)); + string key = Identities.Normal(binding.Key); + if (!selected.ContainsKey(key)) + { + throw JacquardException.Binding(name + " does not use " + Identities.Excerpt(binding.Key)); + } + + chosen[key] = binding.Value ?? throw new ArgumentNullException(nameof(bindings), "operations"); + } + + var operationNames = new HashSet(StringComparer.Ordinal); + foreach (EffectGroup effect in effects.Values) + { + foreach (Identities.Entry operation in effect.Operations) + { + operationNames.Add(operation.Name); + } + } + + var registry = new OperationRegistry(); + var missing = new List(); + foreach (KeyValuePair effect in selected) + { + IReadOnlyDictionary handlers = new Dictionary(); + if (chosen.TryGetValue(effect.Key, out Operations? service)) + { + // Every name a service binds must be an operation of the + // program, so a service binding more is refused from its + // count, before any is copied. + handlers = service.Snapshot(operationNames.Count) + ?? throw JacquardException.Binding(name + ": the service for " + effect.Key + " binds more operations than the program's " + + operationNames.Count.ToString(CultureInfo.InvariantCulture)); + } + + // One service may serve several effects, but every name it + // binds must be some effect's operation. + foreach (string key in handlers.Keys.OrderBy(key => key, StringComparer.Ordinal)) + { + if (!operationNames.Contains(key)) + { + throw JacquardException.Binding(name + ": " + effect.Key + " has no operation " + Identities.Excerpt(key)); + } + } + + foreach (Identities.Entry operation in effect.Value.Operations) + { + if (!handlers.TryGetValue(operation.Name, out Operations.Handler? handler)) + { + missing.Add(effect.Key + "." + operation.Name); + continue; + } + + ProgramData data = Data; + try + { + registry.Bind(effect.Value.Identity, operation.Hash, (context, arguments) => handler(data, context, arguments)); + } + catch (ArgumentException error) + { + throw JacquardException.Binding(name + ": " + error.Message); + } + } + } + + if (missing.Count != 0) + { + throw JacquardException.Binding(name + ": missing handlers: " + string.Join(", ", missing) + + ". Bind every operation in each declared effect before calling"); + } + + return new Function(name, signature, target, registry, host, Data); + } + } + + /// + public override string ToString() + { + lock (sync) + { + return "PreparedProgram[" + Directory + ", functions=" + string.Join(", ", interfaces.Keys) + "]"; + } + } +} diff --git a/dotnet/src/JacquardHost/Sdk/Runtime.cs b/dotnet/src/JacquardHost/Sdk/Runtime.cs new file mode 100644 index 0000000..c39a713 --- /dev/null +++ b/dotnet/src/JacquardHost/Sdk/Runtime.cs @@ -0,0 +1,1001 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using JacquardLang.Host.Carrier; +using JacquardLang.Host.Protocol; + +namespace JacquardLang.Host.Sdk; + +/// +/// Prepares programs against one Core installation. A runtime is immutable +/// and thread-safe. The --version check and prelude discovery run once +/// per Core executable and prelude contents in this process, across runtimes +/// and threads. +/// +[SuppressMessage("Naming", "CA1724", Justification = "Runtime names the same concept in every SDK; it lives in JacquardLang.Host.Sdk.")] +public sealed class Runtime +{ + /// Prepared programs live here unless is set. + public const string DefaultCacheDir = ".scratch/jacquard-host-dotnet"; + + /// The most source bytes one preparation reads, separators included. + public const long SourceLimit = 16L << 20; + + /// The most source files one preparation takes; a longer list is refused from its count before any element is read. + public const int SourceFiles = 1024; + + internal const string ProgramSchema = "jacquard-dotnet-program-v1"; + + /// Core's raw outputs, kept as verified files of each cache entry, from which every binding is derived. + internal const string CheckOutput = "core/check.out"; + + internal const string HashOutput = "core/hash.out"; + + /// The most bytes of program.json a load reads. + private const int MetadataLimit = 32 << 20; + + private const int PollMinMs = 1; + private const int PollMaxMs = 10; + + private static readonly string[] PreludeFiles = { "01-prim.jqd", "02-data.jqd", "19-audit.jqd" }; + + /// Version checks and prelude discoveries, by key, shared by every runtime in this process. + private static readonly ConcurrentDictionary Flights = new(StringComparer.Ordinal); + + private static long staging; + + private int commands; + + /// + /// Validates options; Core does not run yet. + /// + /// A binding failure for a missing prelude or invalid options. + public Runtime(RuntimeOptions? options = null) + { + options ??= RuntimeOptions.Defaults; + string? bin = options.Binary; + if (string.IsNullOrEmpty(bin)) + { + bin = Environment.GetEnvironmentVariable("JACQUARD_BIN"); + } + + string? prelude = options.Prelude; + if (string.IsNullOrEmpty(prelude)) + { + prelude = Environment.GetEnvironmentVariable("JACQUARD_PRELUDE"); + } + + if (string.IsNullOrEmpty(prelude)) + { + throw JacquardException.Binding("set JACQUARD_PRELUDE or RuntimeOptions.Prelude; Core and prelude must match the compatibility pin"); + } + + if (prelude.Length > Core.PathLimit || prelude.Contains('\0', StringComparison.Ordinal) || Core.AbsoluteLength(prelude) > Core.PathLimit) + { + throw JacquardException.Binding("prelude " + Identities.Excerpt(prelude) + " is not a path or is longer than any path"); + } + + if (options.Deadline is { } deadline && deadline <= TimeSpan.Zero) + { + throw JacquardException.Binding("deadline must be positive"); + } + + Limits selected = options.Limits ?? Limits.Hard; + try + { + selected.Validate(); + } + catch (ProtocolException error) + { + throw JacquardException.Binding("invalid runtime limits: " + error.Message); + } + + Binary = string.IsNullOrEmpty(bin) ? "jacquard" : bin; + Prelude = Path.GetFullPath(prelude); + Deadline = options.Deadline; + Limits = selected; + } + + /// The Core executable as configured, before it is resolved. + public string Binary { get; } + + /// The absolute prelude directory. + public string Prelude { get; } + + internal TimeSpan? Deadline { get; } + + internal Limits Limits { get; } + + /// Test seam: how long each Core command may run. + internal TimeSpan CommandTimeout { get; set; } = Core.CommandTimeout; + + /// Test seam: how many Core commands this runtime has run. + internal int CoreCommands => Volatile.Read(ref commands); + + /// Test seam: runs on a preparing thread when it starts to wait for another thread's version check or prelude discovery. Null outside tests. + internal static volatile Action? FlightWaitHook; + + /// Test seam: runs after each source file is read, with its path. Null outside tests. + internal static volatile Action? SourceReadHook; + + /// Test seam: runs when a cache entry has been loaded, just before its program is returned. Null outside tests. + internal static volatile Action? LoadedHook; + + /// Test seam: runs with the vacated staging path once an entry is published. Null outside tests. + internal static volatile Action? PublishedHook; + + /// Test seam: runs after the failure cleanup removes each entry, with its path. Null outside tests. + internal static volatile Action? CleanupHook; + + private string Command(string executable, CancellationToken cancellationToken, params string[] arguments) + { + Interlocked.Increment(ref commands); + return Core.Run(executable, Prelude, CommandTimeout, cancellationToken, arguments); + } + + /// One discovery: empty until a run succeeds; a failed run leaves it empty for the next caller. + private sealed class Flight + { + internal IReadOnlyList? Outputs { get; set; } + + internal bool Running { get; set; } + } + + /// + /// Runs once per key. Concurrent callers wait for + /// one run, each observing its own token in sleeps of 1 to 10 ms; the run + /// itself is bounded by the command timeout. A successful result stays + /// cached, and a failed run leaves the next caller to try again. + /// + private static IReadOnlyList Once(string key, Func> work, CancellationToken cancellationToken) + { + Flight flight = Flights.GetOrAdd(key, static _ => new Flight()); + int idle = PollMinMs; + bool waiting = false; + while (true) + { + lock (flight) + { + if (flight.Outputs is { } outputs) + { + return outputs; + } + + if (!flight.Running) + { + flight.Running = true; + break; + } + } + + if (!waiting) + { + FlightWaitHook?.Invoke(); + } + + waiting = true; + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + // poll(2) as a sleep: it neither waits on a .NET handle nor consumes a Thread.Interrupt. + Posix.Poll(Span.Empty, idle); + idle = Math.Min(idle * 2, PollMaxMs); + } + + try + { + IReadOnlyList outputs = work(); + lock (flight) + { + flight.Outputs = outputs; + } + + return outputs; + } + finally + { + lock (flight) + { + flight.Running = false; + } + } + } + + /// Runs --version once per executable digest; accepts output whose last field is the pinned version. + private void CheckVersion(string executable, string binaryHash, CancellationToken cancellationToken) => + Once("version:" + binaryHash, () => + { + string version = Command(executable, cancellationToken, "--version").Trim(); + string[] fields = version.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + if (fields.Length == 0 || !string.Equals(fields[^1], Compat.CoreVersion, StringComparison.Ordinal)) + { + throw JacquardException.Preparation("expected pinned Core version " + Compat.CoreVersion + ", found \"" + + Identities.Excerpt(version) + "\"; the exact release is also required"); + } + + return Array.Empty(); + }, cancellationToken); + + /// Core's hash output for each prelude file, which must parse. + private IReadOnlyList PreludeOutputs(string executable, string key, CancellationToken cancellationToken) => + Once("prelude:" + key, () => + { + var outputs = new List(); + foreach (string name in PreludeFiles) + { + string output = Command(executable, cancellationToken, "hash", Path.Combine(Prelude, name)); + Identities.HashGroups(output); + outputs.Add(output); + } + + return outputs.AsReadOnly(); + }, cancellationToken); + + /// The bindings of a program: what program.json records beside its identity and files. + internal sealed record Bindings( + IReadOnlyList Names, + IReadOnlyList Constructors, + SortedDictionary Signatures, + IReadOnlyList> Groups) + { + /// Whether two bindings record the same names, constructors, signatures, and groups in the same order. + internal bool Same(Bindings other) => + Names.SequenceEqual(other.Names) + && Constructors.SequenceEqual(other.Constructors) + && Signatures.Count == other.Signatures.Count + && Signatures.SequenceEqual(other.Signatures) + && Groups.Count == other.Groups.Count + && Groups.Zip(other.Groups).All(pair => pair.First.SequenceEqual(pair.Second)); + } + + /// + /// Derives every binding from Core's raw outputs: check --print-sigs + /// for the signatures, and hash of each prelude file and of the + /// source for the names, constructors, and declaration groups. + /// + internal static Bindings Derive(string check, IReadOnlyList preludeHashes, string hash) + { + var signatures = new SortedDictionary(Json.CodePointOrder); + foreach (string line in check.Split('\n')) + { + string trimmed = line.Trim(); + int split = trimmed.IndexOf(" : ", StringComparison.Ordinal); + if (split >= 0) + { + string name = trimmed[..split]; + if (!name.Contains(' ', StringComparison.Ordinal) && !name.Contains('\t', StringComparison.Ordinal)) + { + signatures[name] = trimmed[(split + 3)..]; + } + } + } + + var groups = new List>(); + foreach (string output in preludeHashes) + { + groups.AddRange(Identities.HashGroups(output)); + } + + groups.AddRange(Identities.HashGroups(hash)); + Identities.CheckGroups(groups); + var names = new List(); + var constructorOrder = new List(); + var constructors = new Dictionary(StringComparer.Ordinal); + var seen = new HashSet(StringComparer.Ordinal); + foreach (List group in groups) + { + foreach (Identities.Entry entry in group) + { + if (seen.Add(entry.Name)) + { + names.Add(entry); + } + + if (!constructors.ContainsKey(entry.Name)) + { + constructorOrder.Add(entry.Name); + } + + constructors[entry.Name] = entry.Hash; + } + } + + var constructorList = constructorOrder.Select(name => new Identities.Entry(name, constructors[name])).ToList(); + return new Bindings(names, constructorList, signatures, groups.Select(group => (IReadOnlyList)group).ToList()); + } + + /// + /// Checks, hashes, and installs first-party declaration sources + /// (.jac files, or .jqd files in dependency order) into a + /// verified cache keyed by the source bytes, the Core executable's bytes, + /// every prelude file, and the protocol. Each Core command is bounded by + /// a 30-second timeout and bounded output. Cancelling + /// stops the running command and + /// fails the preparation. + /// + /// The source files; more than is refused from the count alone. + /// The cache directory; null selects the defaults. + /// Stops the preparation. + /// The prepared, verified program. + /// A preparation failure. + /// or one of its elements is null. + public PreparedProgram Prepare(IReadOnlyList sources, PrepareOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(sources); + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + if (!Unix.Supported) + { + throw JacquardException.Preparation("the .NET SDK prepares programs on Linux and macOS only"); + } + + PreparedProgram program; + try + { + program = PrepareChecked(sources, options ?? PrepareOptions.Defaults, cancellationToken); + } + catch (JacquardException error) + { + // Checked here, not in a filter: a filter runs before the + // preparation's cleanup, which may observe the stop. Report the + // stop, not a misleading read or cache failure it caused. + if (cancellationToken.IsCancellationRequested && !error.Message.EndsWith(Core.Aborted, StringComparison.Ordinal)) + { + throw JacquardException.Preparation(Core.Aborted); + } + + throw; + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw JacquardException.Preparation("cannot prepare: " + error.Message); + } + + // A stop during the last of the work is reported, not a program. + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + return program; + } + + private PreparedProgram PrepareChecked(IReadOnlyList sources, PrepareOptions options, CancellationToken cancellationToken) + { + JacquardException Usage() => JacquardException.Preparation("pass .jac files or .jqd files in dependency order"); + + // The count comes from Count alone, before any element is read. + int count = sources.Count; + if (count == 0) + { + throw Usage(); + } + + if (count > SourceFiles) + { + throw JacquardException.Preparation("more than " + SourceFiles.ToString(CultureInfo.InvariantCulture) + " sources"); + } + + string? syntax = null; + + // Each element is read once, so the files read below are the ones checked here. + var checkedSources = new List(count); + for (int i = 0; i < count; i++) + { + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + string source = Core.CheckedPath(sources[i], "source"); + checkedSources.Add(source); + string kind = Extension(source); + syntax ??= kind; + if (!(kind is ".jac" or ".jqd") || !string.Equals(syntax, kind, StringComparison.Ordinal)) + { + throw Usage(); + } + } + + string configured = Path.GetFullPath(Core.CheckedPath(options.CacheDir ?? DefaultCacheDir, "cache directory")); + string executable = Core.Executable(Binary); + string binaryHash = Core.FileDigest(executable, cancellationToken); + CheckVersion(executable, binaryHash, cancellationToken); + string preludeHash; + try + { + preludeHash = Core.MapDigest(Core.Tree(Prelude, Core.Access.Any, cancellationToken)); + } + catch (JacquardException error) + { + throw JacquardException.Preparation("cannot read prelude " + Prelude + ": " + error.Message); + } + + byte[] bytes = ReadSources(checkedSources, cancellationToken); + var identity = new SortedDictionary(Json.CodePointOrder) + { + ["binary"] = binaryHash, + ["prelude"] = preludeHash, + ["protocol"] = Compat.Protocol, + ["schema"] = ProgramSchema, + ["source"] = Core.Digest(bytes), + ["syntax"] = syntax!, + }; + string cache; + try + { + if (OperatingSystem.IsWindows()) + { + Directory.CreateDirectory(configured); + } + else + { + Directory.CreateDirectory(configured, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + + // A symbolic link at the cache root is followed once, here; the + // directory it names is checked, and nothing inside is followed. + cache = new DirectoryInfo(configured).ResolveLinkTarget(returnFinalTarget: true)?.FullName ?? configured; + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException or ArgumentException) + { + throw JacquardException.Preparation("cannot create " + configured + ": " + error.Message); + } + + string directory = Path.Combine(cache, Core.MapDigest(identity)); + + // Only the cache's owner may write it; nothing inside it can resist + // someone else who may. + Core.Check(cache, Core.Access.OwnerOnly); + if (Unix.Stat(directory, follow: false).Kind != Unix.Kind.Missing) + { + return Load(directory, identity, executable, cancellationToken); + } + + string stage = StagingDirectory(cache); + bool published = false; + try + { + string snapshot = Path.Combine(stage, "source" + syntax); + Write(snapshot, bytes, "cannot write the source snapshot"); + string check = Command(executable, cancellationToken, "check", snapshot, "--print-sigs"); + IReadOnlyList preludeHashes = PreludeOutputs(executable, binaryHash + preludeHash, cancellationToken); + string hash = Command(executable, cancellationToken, "hash", snapshot); + Bindings bindings = Derive(check, preludeHashes, hash); + string store = Path.Combine(stage, "store"); + Command(executable, cancellationToken, "run", snapshot, "--store", store); + CreateDirectory(Path.Combine(stage, "core")); + Write(Path.Combine(stage, CheckOutput), Encoding.UTF8.GetBytes(check), "cannot write Core's output"); + Write(Path.Combine(stage, HashOutput), Encoding.UTF8.GetBytes(hash), "cannot write Core's output"); + for (int i = 0; i < PreludeFiles.Length; i++) + { + Write(Path.Combine(stage, PreludeOutput(PreludeFiles[i])), Encoding.UTF8.GetBytes(preludeHashes[i]), "cannot write Core's output"); + } + + // Core made the store with its own permissions; the entry is owner-only. + var metadata = new Metadata(identity, bindings, Core.Tree(stage, Core.Access.Restrict, cancellationToken)); + Write(Path.Combine(stage, "program.json"), Encoding.UTF8.GetBytes(metadata.ToJson()), "cannot write program.json"); + try + { + Directory.Move(stage, directory); + + // The staging path is vacated; whatever appears there later is not this call's. + published = true; + PublishedHook?.Invoke(stage); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + // A concurrent preparation published the same artifact first. + if (Unix.Stat(directory, follow: false).Kind == Unix.Kind.Missing) + { + throw JacquardException.Preparation("cannot publish " + directory + ": " + error.Message); + } + } + } + finally + { + if (!published) + { + RemoveTree(stage, cancellationToken); + } + } + + return Load(directory, identity, executable, cancellationToken); + } + + private static string PreludeOutput(string file) => "core/prelude-" + file + ".out"; + + private static string Extension(string source) + { + string name = Path.GetFileName(source); + int dot = name.LastIndexOf('.'); + return dot <= 0 ? string.Empty : name[dot..]; + } + + /// + /// The sources joined by newlines. Each must be a regular file (a + /// symbolic link to one is followed); the total is checked against + /// from the files' sizes before each is read, + /// and no read goes past it. Before each file the token is observed. + /// + private static byte[] ReadSources(List sources, CancellationToken cancellationToken) + { + JacquardException Over() => JacquardException.Preparation("sources exceed " + (SourceLimit >> 20).ToString(CultureInfo.InvariantCulture) + " MiB in total"); + using var content = new MemoryStream(); + long total = 0; + for (int i = 0; i < sources.Count; i++) + { + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + string source = sources[i]; + Unix.Status status = Core.StatOrFail(source, follow: true, "cannot read "); + if (status.Kind != Unix.Kind.Regular) + { + throw JacquardException.Preparation("cannot read " + source + ": not a regular file"); + } + + total += (i > 0 ? 1 : 0) + status.Size; + if (total > SourceLimit) + { + throw Over(); + } + + if (i > 0) + { + content.WriteByte((byte)'\n'); + } + + int room = (int)(SourceLimit - content.Length); + byte[] buffer = new byte[Math.Min(room + 1L, status.Size + 1L)]; + int read = 0; + try + { + using var stream = new FileStream(source, new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Share = FileShare.ReadWrite, BufferSize = 0 }); + while (true) + { + if (read == buffer.Length) + { + // The file grew past its size: read one more byte to see whether it is over. + if (buffer.Length > room) + { + break; + } + + Array.Resize(ref buffer, (int)Math.Min(room + 1L, buffer.Length * 2L)); + } + + int n = stream.Read(buffer, read, buffer.Length - read); + if (n <= 0) + { + break; + } + + read += n; + } + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw JacquardException.Preparation("cannot read " + source + ": " + error.Message); + } + + if (read > room) + { + throw Over(); + } + + content.Write(buffer, 0, read); + SourceReadHook?.Invoke(source); + } + + return content.ToArray(); + } + + private static void CreateDirectory(string path) + { + try + { + if (!Unix.MakeDirectory(path, 0x1C0)) + { + throw JacquardException.Preparation("cannot create " + path + ": it already exists"); + } + } + catch (IOException error) + { + throw JacquardException.Preparation("cannot create " + path + ": " + error.Message); + } + } + + /// Writes a new owner-only file. + private static void Write(string path, byte[] bytes, string context) + { + try + { + using var stream = new FileStream(path, Core.OwnerOnly(FileMode.CreateNew)); + stream.Write(bytes); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw JacquardException.Preparation(context + ": " + error.Message); + } + } + + /// A new owner-only staging directory, created exclusively, so a directory the call did not make is never used or removed. + private static string StagingDirectory(string cache) + { + long stamp = Stopwatch.GetTimestamp() & 0xffffffffL; + for (int attempt = 0; attempt < 100; attempt++) + { + string path = Path.Combine(cache, string.Format( + CultureInfo.InvariantCulture, + "prepare-{0}-{1:x8}-{2}", + Environment.ProcessId, + stamp, + Interlocked.Increment(ref staging))); + try + { + if (Unix.MakeDirectory(path, 0x1C0)) + { + return path; + } + } + catch (IOException error) + { + throw JacquardException.Preparation("cannot create " + path + ": " + error.Message); + } + } + + throw JacquardException.Preparation("cannot create a staging directory"); + } + + /// + /// Removes a staging directory unless it was published (and so no longer + /// exists), in a streamed post-order walk that holds one directory + /// enumerator per level on an explicit stack. It removes at most the + /// tree's entry cap, and ends at the first cancellation, checked before + /// each entry. What it leaves is an unpublished prepare-* + /// directory, which is never loaded as a cache entry: entries are named by + /// their identity digest. + /// + [SuppressMessage("Design", "CA1031", Justification = "Cleanup is best effort; what it cannot remove is left for the user and never becomes a cache entry.")] + internal static void RemoveTree(string root, CancellationToken cancellationToken) + { + int cap = Core.TreeEntryCap; + int removed = 0; + var stack = new Stack<(string Directory, IEnumerator Entries)>(); + try + { + if (Unix.Stat(root, follow: false).Kind != Unix.Kind.Directory) + { + return; + } + + stack.Push((root, Directory.EnumerateFileSystemEntries(root).GetEnumerator())); + while (stack.Count > 0) + { + if (cancellationToken.IsCancellationRequested || removed >= cap) + { + return; + } + + (string directory, IEnumerator entries) = stack.Peek(); + if (!entries.MoveNext()) + { + stack.Pop().Entries.Dispose(); + Directory.Delete(directory, recursive: false); + removed++; + CleanupHook?.Invoke(directory); + continue; + } + + string path = entries.Current; + if (Unix.Stat(path, follow: false).Kind == Unix.Kind.Directory) + { + stack.Push((path, Directory.EnumerateFileSystemEntries(path).GetEnumerator())); + continue; + } + + File.Delete(path); + removed++; + CleanupHook?.Invoke(path); + } + } + catch (Exception) + { + // Left for the user to remove; it never becomes a cache entry. + } + finally + { + while (stack.Count > 0) + { + stack.Pop().Entries.Dispose(); + } + } + } + + /// + /// Loads a cache entry, refusing an accidental or partial change: the + /// entry and everything in it must be owned by this user and writable by + /// no one else, every file but program.json must match the digests + /// it records, and every binding it records must equal what is derived + /// again from Core's raw outputs, which are among those verified files. A + /// coordinated rewrite of several files is accepted: only the cache's + /// owner can make one, and the owner is trusted. + /// + private PreparedProgram Load(string directory, SortedDictionary identity, string executable, CancellationToken cancellationToken) + { + JacquardException Fail(string detail) => JacquardException.Preparation("prepared cache " + directory + + " is incomplete or changed (" + detail + "); remove that cache entry and prepare again"); + + SortedDictionary observed; + try + { + // Regular files only, bounded, so no read below can block. + observed = Core.Tree(directory, Core.Access.OwnerOnly, cancellationToken); + } + catch (JacquardException error) + { + if (error.Message == Core.Aborted) + { + throw; + } + + throw Fail(error.Message); + } + + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + if (!observed.ContainsKey("program.json")) + { + throw Fail("no program.json"); + } + + Metadata metadata = Metadata.FromJson(ReadVerified(directory, "program.json", null, MetadataLimit, Fail)) ?? throw Fail("unreadable program.json"); + observed.Remove("program.json"); + if (!metadata.Identity.SequenceEqual(identity) || !metadata.Files.SequenceEqual(observed)) + { + throw Fail("cached artifact digests do not match"); + } + + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + Bindings derived; + try + { + var preludeHashes = new List(); + foreach (string file in PreludeFiles) + { + preludeHashes.Add(Text(ReadVerified(directory, PreludeOutput(file), observed, Core.StdoutLimit, Fail), Fail)); + } + + derived = Derive( + Text(ReadVerified(directory, CheckOutput, observed, Core.StdoutLimit, Fail), Fail), + preludeHashes, + Text(ReadVerified(directory, HashOutput, observed, Core.StdoutLimit, Fail), Fail)); + } + catch (JacquardException error) when (!error.Message.StartsWith("prepared cache ", StringComparison.Ordinal)) + { + throw Fail("Core's cached output does not parse: " + error.Message); + } + + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Core.Aborted); + } + + if (!derived.Same(metadata.Bindings)) + { + throw Fail("program.json bindings differ from Core's cached output"); + } + + var program = new PreparedProgram(this, directory, metadata.Bindings, executable); + LoadedHook?.Invoke(); + return program; + } + + /// + /// The bytes of a regular file in a cache entry, not followed if it is a + /// link, at most ; when + /// is given, they must have the digest it records. + /// + private static byte[] ReadVerified(string directory, string name, SortedDictionary? digests, int limit, Func fail) + { + if (digests is not null && !digests.ContainsKey(name)) + { + throw fail("no " + name); + } + + string path = Path.Combine(directory, name); + byte[] bytes; + try + { + Unix.Status status = Unix.Stat(path, follow: false); + if (status.Kind != Unix.Kind.Regular) + { + throw fail(name + " is not a regular file"); + } + + if (status.Size > limit) + { + throw fail(name + " is too long"); + } + + using var stream = new FileStream(path, new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Share = FileShare.ReadWrite, BufferSize = 0 }); + byte[] buffer = new byte[status.Size + 1]; + int read = 0; + int n; + while (read < buffer.Length && (n = stream.Read(buffer, read, buffer.Length - read)) > 0) + { + read += n; + } + + if (read > status.Size) + { + throw fail(name + " changed while it was read"); + } + + bytes = buffer.AsSpan(0, read).ToArray(); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw fail(name + ": " + error.Message); + } + + if (digests is not null && !string.Equals(Core.Digest(bytes), digests[name], StringComparison.Ordinal)) + { + throw fail("cached artifact digests do not match"); + } + + return bytes; + } + + private static string Text(byte[] bytes, Func fail) => Core.Utf8(bytes) ?? throw fail("Core's cached output is not UTF-8"); + + /// The prepared metadata in program.json. + internal sealed record Metadata(SortedDictionary Identity, Bindings Bindings, SortedDictionary Files) + { + private static JsonObject Strings(IEnumerable> map) => + Json.Object(map.Select(pair => new KeyValuePair(pair.Key, Json.Of(pair.Value)))); + + private static JsonArray Pairs(IEnumerable entries) => + Json.Of(entries.Select(entry => (Json)Json.Of(new Json[] { Json.Of(entry.Name), Json.Of(entry.Hash.Hex) }))); + + internal string ToJson() + { + JsonArray groups = Json.Of(Bindings.Groups.Select(group => (Json)Json.Of(group.Select(entry => + (Json)Json.Object(("hash", Json.Of(entry.Hash.Hex)), ("name", Json.Of(entry.Name))))))); + JsonObject json = Json.Object( + ("constructors", Pairs(Bindings.Constructors)), + ("files", Strings(Files)), + ("groups", groups), + ("identity", Strings(Identity)), + ("names", Pairs(Bindings.Names)), + ("signatures", Strings(Bindings.Signatures))); + return JsonCodec.Stringify(json, 8); + } + + private static SortedDictionary? Map(JsonObject json, string key) + { + if (json.Get(key) is not JsonObject fields) + { + return null; + } + + var output = new SortedDictionary(Json.CodePointOrder); + foreach (KeyValuePair field in fields) + { + if (field.Value.AsString() is not { } text) + { + return null; + } + + output[field.Key] = text; + } + + return output; + } + + private static Identities.Entry? Entry(string? name, Json? hash) + { + string? hex = hash?.AsString(); + return name is null || hex is null || !Protocol.Hash.LowerHex(hex, 64) ? null : new Identities.Entry(name, new Hash(hex)); + } + + private static List? Pairs(JsonObject json, string key) + { + if (json.Get(key)?.AsArray() is not { } items) + { + return null; + } + + var output = new List(items.Count); + foreach (Json item in items) + { + if (item.AsArray() is not { Count: 2 } pair || Entry(pair[0].AsString(), pair[1]) is not { } entry) + { + return null; + } + + output.Add(entry); + } + + return output; + } + + /// The metadata, or null when it is not what writes. + internal static Metadata? FromJson(byte[] text) + { + JsonObject json; + try + { + json = JsonCodec.DecodeObject(text, 8); + } + catch (ProtocolException) + { + return null; + } + + if (json.Get("groups")?.AsArray() is not { } groupItems) + { + return null; + } + + var groups = new List>(groupItems.Count); + foreach (Json group in groupItems) + { + if (group.AsArray() is not { } entries) + { + return null; + } + + var parsed = new List(entries.Count); + foreach (Json entry in entries) + { + if (entry is not JsonObject fields || Entry(fields.Get("name")?.AsString(), fields.Get("hash")) is not { } value) + { + return null; + } + + parsed.Add(value); + } + + groups.Add(parsed); + } + + SortedDictionary? identity = Map(json, "identity"); + List? names = Pairs(json, "names"); + List? constructors = Pairs(json, "constructors"); + SortedDictionary? signatures = Map(json, "signatures"); + SortedDictionary? files = Map(json, "files"); + if (identity is null || names is null || constructors is null || signatures is null || files is null) + { + return null; + } + + return new Metadata(identity, new Bindings(names, constructors, signatures, groups), files); + } + } + + /// + public override string ToString() => "Runtime[binary=" + Identities.Excerpt(Binary) + ", prelude=" + Prelude + "]"; +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs new file mode 100644 index 0000000..634e08e --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs @@ -0,0 +1,257 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; +using JacquardLang.Host.Carrier; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// +/// Function preflight with synthetic identities and no Core: arguments are +/// counted before they are encoded, a caller's codec that throws is an +/// invalid argument, and in MapBatches a cancellation ends validation +/// between batches and after the last, there is no validation deadline, and +/// no batch's encoding is kept once it is checked. Rows go through a codec +/// that counts its encodings, so each bound is an exact count. The worker +/// command does not exist, so a call that gets past validation fails at +/// batch 0. +/// +[TestFixture] +public sealed class BatchesTests +{ + private string root = string.Empty; + + [SetUp] + public void Prepare() => root = SdkSupport.Scratch("batches"); + + [TearDown] + public void CleanUp() + { + Function.ConformHook = null; + SdkSupport.RemoveScratch(root); + } + + private Host MissingWorker() => + new(Path.Combine(root, "store"), new HostOptions { Carrier = new WorkerOptions { Command = new[] { Path.Combine(root, "no-such-jacquard") } } }); + + /// ints : (List Int) ->{} List Int over the data fixture. + private Function Ints() + { + ProgramData data = DataTests.Fixture(Limits.Hard); + NominalType list = DataTests.Nominal(data.Name("list")!, DataTests.Nominal(data.Name("int")!)); + var signature = new CallInterface(new BoundaryType[] { list }, list, Array.Empty()); + return new Function("ints", signature, DataTests.Numbered(99), new OperationRegistry(), MissingWorker(), data); + } + + /// Encodes Ints, counting each encoding and running the action returns for the count. + private sealed class Counting : Codec + { + private readonly Func each; + + internal Counting(Func each) + { + this.each = each; + } + + internal int Encodes { get; private set; } + + public override Value Encode(Encoder encoder, long value) + { + each(++Encodes)?.Invoke(); + return Codecs.Int.Encode(encoder, value); + } + + public override long Decode(Decoder decoder, Value value) => Codecs.Int.Decode(decoder, value); + } + + private static DataTests.RepeatList Rows(int count) => new(1L, count); + + /// + /// The argument count is checked from the count alone: a million + /// arguments to a one-parameter function, or a million leading arguments + /// to a batch, are refused before one is copied or encoded. + /// + [Test] + public void TheWrongArgumentCountIsRefusedBeforeAnyArgumentIsEncoded() + { + var codec = new Counting(_ => null); + Argument[] many = Enumerable.Repeat(codec.Arg(1L), 1_000_000).ToArray(); + JacquardException call = Support.Throws(() => Ints().Invoke(many)); + Assert.That(call.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(call.Message, Is.EqualTo("ints: expected 1 arguments, received 1000000")); + JacquardException batches = Support.Throws(() => Ints().MapBatches(Codecs.Int, codec, Rows(10), 5, many)); + Assert.That(batches.Message, Is.EqualTo("ints: expected 1 arguments, received 1000001")); + Assert.That(codec.Encodes, Is.EqualTo(0), "arguments were encoded before the count was checked"); + } + + /// A caller's codec that throws is a local refusal, as any other: no worker starts, and its exception is kept. + [Test] + public void ACodecThatThrowsIsAnInvalidArgument() + { + var broken = new Counting(_ => () => throw new InvalidOperationException("broken codec")); + JacquardException error = Support.Throws(() => Ints().Invoke(Codecs.List(broken).Arg(new[] { 1L }))); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(error.Message, Is.EqualTo("ints.argument[0]: its codec failed: System.InvalidOperationException: broken codec")); + Assert.That(error.InnerException, Is.InstanceOf()); + JacquardException nothing = Support.Throws(() => Ints().Invoke(new Argument[] { null! })); + Assert.That(nothing.Message, Does.StartWith("ints.argument[0]: null has no Jacquard value")); + } + + [Test] + public void AnAlreadyCancelledCallEncodesNoBatch() + { + var codec = new Counting(_ => null); + using var cancel = new CancellationTokenSource(); + cancel.Cancel(); + JacquardException error = Support.Throws(() => Ints().MapBatches(Codecs.Int, codec, Rows(10_000), 10, cancellationToken: cancel.Token)); + Assert.That(codec.Encodes, Is.EqualTo(0), "rows were encoded after the stop"); + Assert.That(error.BatchIndex, Is.EqualTo(-1)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Invocation), error.Message); + Assert.That(Support.Reason(error.Result!), Is.EqualTo("aborted")); + Assert.That(error.Failure, Is.EqualTo(Failure.WorkerLost)); + Assert.That(error.Completed, Is.Empty); + } + + [Test] + public void ACancelDuringValidationStopsAtTheNextBatch() + { + using var cancel = new CancellationTokenSource(); + + // Cancelled while batch 2 (rows 21 to 30) is being encoded. + var codec = new Counting(n => n == 25 ? cancel.Cancel : null); + JacquardException error = Support.Throws(() => Ints().MapBatches(Codecs.Int, codec, Rows(10_000), 10, cancellationToken: cancel.Token)); + Assert.That(codec.Encodes, Is.EqualTo(30)); + Assert.That(error.BatchIndex, Is.EqualTo(-1)); + Assert.That(Support.Reason(error.Result!), Is.EqualTo("aborted")); + } + + /// + /// A cancellation while the last batch is validated is observed before + /// any worker starts: the refusal is the host's aborted result at batch + /// index -1, and no batch is encoded again to run. + /// + [Test] + public void AStopDuringTheLastBatchsValidationStartsNoWorker() + { + using var cancel = new CancellationTokenSource(); + var codec = new Counting(n => n == 30 ? cancel.Cancel : null); + JacquardException error = Support.Throws(() => Ints().MapBatches(Codecs.Int, codec, Rows(30), 10, cancellationToken: cancel.Token)); + Assert.That(codec.Encodes, Is.EqualTo(30), "a batch was encoded again after the stop"); + Assert.That(error.BatchIndex, Is.EqualTo(-1)); + Assert.That(Support.Reason(error.Result!), Is.EqualTo("aborted")); + } + + /// + /// Validation has no deadline of its own (as in the other SDKs), so it + /// does not depend on the machine's speed: every batch is validated, and + /// the call goes on to the worker (here a missing executable, reported + /// at batch 0), encoding batch 0 again to run it. + /// + [Test] + public void EveryBatchIsValidatedAndTheFirstEncodedAgainToRun() + { + var codec = new Counting(_ => null); + JacquardException error = Support.Throws(() => Ints().MapBatches(Codecs.Int, codec, Rows(30), 10)); + Assert.That(error.BatchIndex, Is.EqualTo(0), error.Message); + Assert.That(Support.Reason(error.Result!), Is.EqualTo("worker_spawn_failed")); + Assert.That(codec.Encodes, Is.EqualTo(40)); + } + + /// + /// Retention is structurally impossible: the validation step returns + /// nothing, so no caller can keep a batch's encoding, and no field of a + /// function can hold one. + /// + [Test] + public void ValidationKeepsNoEncoding() + { + MethodInfo validate = typeof(Function).GetMethod("Validate", BindingFlags.NonPublic | BindingFlags.Instance)!; + Assert.That(validate.ReturnType, Is.EqualTo(typeof(void))); + foreach (FieldInfo field in typeof(Function).GetFields(BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public)) + { + Assert.That( + field.FieldType == typeof(string) || (!typeof(IEnumerable).IsAssignableFrom(field.FieldType) && !typeof(Value).IsAssignableFrom(field.FieldType)), + Is.True, + "a field could hold encodings: " + field); + } + } + + /// Invokes a one-parameter function of with a constant-time codec returning , failing the call if the type check is reached. + private JacquardException RefusedBeforeTheTypeCheck(ProgramData data, BoundaryType type, Value value) + { + var function = new Function("deep", new CallInterface(new[] { type }, type, Array.Empty()), DataTests.Numbered(98), new OperationRegistry(), MissingWorker(), data); + bool conformed = false; + Function.ConformHook = () => + { + conformed = true; + throw new InvalidOperationException("the type check ran"); + }; + Codec constant = Codecs.Create((encoder, ignored) => value, (decoder, ignored) => 0); + JacquardException error = Support.Throws(() => function.Invoke(constant.Arg(0))); + Assert.That(conformed, Is.False, "the type check ran before the value was measured"); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument), error.Message); + return error; + } + + /// + /// A constant-time custom codec returns a small shared graph standing for + /// 2^28 leaves, and it is measured before its type is checked, under the + /// shared budgets, so the type check never runs. As 28 levels of a + /// two-item tuple it is refused by the node budget; as 28 levels of a + /// two-item list it is refused by the JSON depth bound first. + /// + [Test] + public void ASharedGraphFromACustomCodecIsMeasuredBeforeItsTypeIsChecked() + { + ProgramData data = DataTests.Fixture(Limits.Hard); + BoundaryType tupleType = DataTests.Nominal(data.Name("int")!); + Value tuple = new IntValue(1); + for (int i = 0; i < 28; i++) + { + tupleType = new TupleType(tupleType, tupleType); + tuple = new TupleValue(tuple, tuple); + } + + Assert.That(RefusedBeforeTheTypeCheck(data, tupleType, tuple).Message, Does.EndWith("value exceeds max_value_nodes; use smaller independent batches")); + Hash list = data.Name("list")!; + Hash cons = data.ConstructorId("cons")!; + var nil = new ConstructorValue(data.ConstructorId("nil")!, Array.Empty()); + BoundaryType listType = DataTests.Nominal(data.Name("int")!); + Value lists = new IntValue(1); + for (int i = 0; i < 28; i++) + { + listType = DataTests.Nominal(list, listType); + lists = new ConstructorValue(cons, new[] { lists, new ConstructorValue(cons, new[] { lists, nil }) }); + } + + Assert.That(RefusedBeforeTheTypeCheck(data, listType, lists).Message, Does.EndWith("value exceeds the v0 JSON depth; use smaller independent batches")); + } + + [Test] + public void CompletedResultsAreKeptReadOnly() + { + JacquardException error = JacquardException.Batch(1, new object?[] { Unit.Value, null }, JacquardException.Invalid("later")); + Assert.That(error.BatchIndex, Is.EqualTo(1)); + Assert.That(error.Completed, Is.EqualTo(new object?[] { Unit.Value, null })); + Assert.That(error.Message, Is.EqualTo("batch 1: later")); + Assert.That(((IList)error.Completed).IsReadOnly, Is.True); + Support.Throws(() => ((IList)error.Completed).Add(1L)); + } + + /// MapBatches needs a pure function with a final List parameter and a List result; a batch size must be positive. + [Test] + public void MapBatchesNeedsAListFunction() + { + ProgramData data = DataTests.Fixture(Limits.Hard); + NominalType integer = DataTests.Nominal(data.Name("int")!); + var scalar = new Function("one", new CallInterface(new BoundaryType[] { integer }, integer, Array.Empty()), DataTests.Numbered(97), new OperationRegistry(), MissingWorker(), data); + Assert.That(Support.Throws(() => scalar.MapBatches(Codecs.Int, Codecs.Int, Rows(3), 1)).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(Support.Throws(() => Ints().MapBatches(Codecs.Int, Codecs.Int, Rows(3), 0)).Message, Is.EqualTo("batch size must be positive")); + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs b/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs new file mode 100644 index 0000000..20dfb8e --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs @@ -0,0 +1,120 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// +/// A fake Core: a shell script that logs its first argument, answers +/// --version, and runs a case for each other command. Every process a +/// case leaves behind records its pid in a *.pid or pids file; +/// kills them. +/// +internal sealed class FakeCore : IDisposable +{ + internal static readonly string First = new('a', 64); + internal static readonly string Second = new('b', 64); + + private FakeCore(string root, Runtime runtime) + { + Root = root; + Runtime = runtime; + } + + internal string Root { get; } + + internal Runtime Runtime { get; } + + internal string Binary => Path.Combine(Root, "jacquard"); + + internal PrepareOptions Options => new() { CacheDir = Path.Combine(Root, "cache") }; + + /// + /// A fake whose --version prints (unless + /// replaces it), and whose other commands + /// run (shell case arms). $ROOT + /// names the fake's directory. + /// + internal static FakeCore Create(string root, string version, string? versionCase, string cases) + { + Directory.CreateDirectory(Path.Combine(root, "prelude")); + File.WriteAllText(Path.Combine(root, "prelude", "01-prim.jqd"), "prelude\n"); + File.WriteAllText(Path.Combine(root, "rules.jac"), "rules\n"); + string script = "#!/bin/sh\nROOT='" + root + "'\necho \"$1\" >> \"$ROOT/log\"\ncase \"$1\" in\n--version) " + + (versionCase ?? "echo 'jacquard " + version + "'") + " ;;\n" + cases + "\nesac\n"; + string binary = Path.Combine(root, "jacquard"); + File.WriteAllText(binary, script); + File.SetUnixFileMode(binary, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + var runtime = new Runtime(new RuntimeOptions { Binary = binary, Prelude = Path.Combine(root, "prelude") }); + return new FakeCore(root, runtime); + } + + /// A fake whose check fails with boom. + internal static FakeCore Failing(string root, string version) => Create(root, version, null, "check) echo 'boom' >&2; exit 3 ;;"); + + /// A fake whose check and hash succeed and whose run fails. + internal static FakeCore Hashing(string root) => + Create(root, Compat.CoreVersion, null, "check) ;;\nhash) echo '0:x " + new string('0', 64) + "' ;;\nrun) exit 4 ;;"); + + /// + /// A fake that prepares every source as two functions, first and + /// second, with identities and + /// , over a one-type prelude. + /// + internal static FakeCore Programs(string root) => Create(root, Compat.CoreVersion, null, + "check) echo 'first : () ->{} Int'; echo 'second : () ->{} Int' ;;\n" + + "hash) case \"$2\" in */prelude/*) echo '0:int " + new string('1', 64) + "' ;; *) echo '0:first " + First + "'; echo '1:second " + + Second + "' ;; esac ;;\n" + + "run) mkdir -p \"$4\" && echo objects > \"$4/objects\" ;;"); + + internal string Source(string name) + { + string source = Path.Combine(Root, name); + if (!File.Exists(source)) + { + File.WriteAllText(source, name + "\n"); + } + + return source; + } + + /// Prepares , which must fail. + internal JacquardException PrepareFails(string name = "rules.jac", CancellationToken cancellationToken = default) => + Support.Throws(() => Runtime.Prepare(new[] { Source(name) }, Options, cancellationToken))!; + + internal long Calls(string command) + { + string log = Path.Combine(Root, "log"); + return File.Exists(log) ? File.ReadAllLines(log).Count(line => line == command) : 0; + } + + internal bool StagingLeft() + { + string cache = Path.Combine(Root, "cache"); + return Directory.Exists(cache) && Directory.EnumerateDirectories(cache).Any(entry => Path.GetFileName(entry).StartsWith("prepare-", StringComparison.Ordinal)); + } + + /// The cache's entries, each a staging directory or a published entry, sorted. + internal List CacheEntries() => + Directory.EnumerateFileSystemEntries(Path.Combine(Root, "cache")).Select(Path.GetFileName).OrderBy(name => name, StringComparer.Ordinal).ToList()!; + + /// Waits for to hold a complete pid line and returns it. + internal int Pid(string name) => SdkSupport.PidFile(Path.Combine(Root, name)); + + public void Dispose() + { + foreach (string file in Directory.EnumerateFiles(Root).Where(path => path.EndsWith("pids", StringComparison.Ordinal) || path.EndsWith(".pid", StringComparison.Ordinal))) + { + foreach (string pid in File.ReadAllText(file).Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)) + { + Support.Kill(int.Parse(pid, CultureInfo.InvariantCulture)); + } + } + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/OperationsTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/OperationsTests.cs new file mode 100644 index 0000000..9898705 --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/OperationsTests.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// .NET handlers adapted to operations: decoding, results, failures, the context and its token, and the stop check before a handler starts. +[TestFixture] +public sealed class OperationsTests +{ + private readonly ProgramData data = DataTests.Fixture(Limits.Hard); + + [TearDown] + public void ClearHook() => Operations.AfterDecodeHook = null; + + private static HandlerContext Context(Func? stopping = null, CancellationToken token = default) => new(stopping ?? (() => false), () => false, token); + + private static Value Text(string text) => new TextValue(text); + + private Value Call(Operations service, string name, HandlerContext context, params Value[] arguments) => + service.Snapshot(100)![name](data, context, arguments); + + private HandlerException Failure(Operations service, string name, params Value[] arguments) => + Support.Throws(() => Call(service, name, Context(), arguments)); + + [Test] + public void HandlersAdaptToOperations() + { + int calls = 0; + Operations service = new Operations() + .On("read-document", Codecs.Text, Codecs.Text, (context, key) => + { + calls++; + if (context.IsCancelled) + { + throw HandlerException.CompletionUnknown("cancelled"); + } + + return key == "example" ? "text" : throw HandlerException.Refused("unknown document"); + }) + .On("forget", Codecs.Text, Codecs.Unit, (string key) => throw new IOException("private detail")) + .On("lookup_stock", Codecs.Text, Codecs.Int, sku => (long)sku.Length) + .On("pair", Codecs.Tuple(Codecs.Int, Codecs.Int), () => (3L, 4L)) + .On("route", DataTests.IncidentCodec, DataTests.TriageCodec, incident => new DataTests.Routed(incident.Id, DataTests.Urgency.WatchOnly)) + .On("huge", Codecs.Int, () => long.MaxValue) + .On("nothing", Codecs.Text, () => null!) + .On("six", Codecs.Int, Codecs.Int, Codecs.Int, Codecs.Int, Codecs.Int, Codecs.Int, Codecs.Int, (context, a, b, c, d, e, f) => a + b + c + d + e + f); + Assert.That(Call(service, "read-document", Context(), Text("example")), Is.EqualTo(Text("text"))); + Assert.That(calls, Is.EqualTo(1)); + Assert.That(Failure(service, "read-document", Text("other")).Category, Is.EqualTo(FailureCategory.RefusedAuthority)); + HandlerException arity = Failure(service, "read-document"); + Assert.That(arity.Category, Is.EqualTo(FailureCategory.OutsideFailure)); + Assert.That(arity.Message, Is.EqualTo("handler parameter count differs from the request")); + Assert.That(Failure(service, "read-document", new IntValue(1)).Message, Is.EqualTo("handler argument cannot be mapped")); + Assert.That(calls, Is.EqualTo(2), "a handler ran with unmappable arguments"); + + // Any other exception reaches the host, which reports completion unknown without its message. + Support.Throws(() => Call(service, "forget", Context(), Text("x"))); + Assert.That(Call(service, "lookup-stock", Context(), Text("abc")), Is.EqualTo(new IntValue(3))); + Assert.That(Call(service, "pair", Context()), Is.EqualTo(new TupleValue(new IntValue(3), new IntValue(4)))); + Value incident = data.Encode(DataTests.IncidentCodec, new DataTests.Incident("i", 1, null)); + Assert.That(data.Decode(DataTests.TriageCodec, Call(service, "route", Context(), incident)), Is.EqualTo(new DataTests.Routed("i", DataTests.Urgency.WatchOnly))); + foreach (string unrepresentable in new[] { "huge", "nothing" }) + { + HandlerException result = Failure(service, unrepresentable); + Assert.That(result.Category, Is.EqualTo(FailureCategory.OutsideFailure)); + Assert.That(result.Message, Is.EqualTo("handler result cannot be represented")); + } + + Value[] six = { new IntValue(1), new IntValue(2), new IntValue(3), new IntValue(4), new IntValue(5), new IntValue(6) }; + Assert.That(Call(service, "six", Context(), six), Is.EqualTo(new IntValue(21))); + Operations replaced = new Operations().On("op", Codecs.Int, () => 1L).On("op", Codecs.Int, () => 2L); + Assert.That(Call(replaced, "op", Context()), Is.EqualTo(new IntValue(2))); + Assert.That(replaced.Count, Is.EqualTo(1)); + + // A service over a limit is refused from its count, before anything is copied. + Assert.That(service.Snapshot(3), Is.Null); + } + + /// The handler sees the invocation's context and its token. + [Test] + public void HandlersSeeTheContextAndItsToken() + { + using var stop = new CancellationTokenSource(); + CancellationToken seen = default; + Operations service = new Operations().On("watch", Codecs.Bool, context => + { + seen = context.CancellationToken; + return context.CancellationToken.IsCancellationRequested; + }); + Assert.That(Call(service, "watch", Context(token: stop.Token)), Is.EqualTo(data.Encode(Codecs.Bool, false))); + Assert.That(seen, Is.EqualTo(stop.Token)); + } + + /// + /// Decoding a request takes time. A stop that lands after the arguments + /// are decoded, here through the test seam, must keep the handler from + /// starting; the handler is never called. + /// + [Test] + public void NoHandlerStartsOnceTheCallStoppedDuringDecoding() + { + bool stopped = false; + int calls = 0; + Operations service = new Operations().On("act", Codecs.Text, Codecs.Unit, key => + { + calls++; + return Unit.Value; + }); + Operations.AfterDecodeHook = () => stopped = true; + HandlerException refused = Support.Throws(() => Call(service, "act", Context(() => stopped), Text("x"))); + Assert.That(refused.Message, Is.EqualTo("handler was not started")); + Assert.That(calls, Is.EqualTo(0), "the handler started after the call stopped"); + Operations.AfterDecodeHook = null; + Assert.That(Call(service, "act", Context(), Text("x")), Is.InstanceOf()); + Assert.That(calls, Is.EqualTo(1)); + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs new file mode 100644 index 0000000..08ccd56 --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs @@ -0,0 +1,755 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// +/// Runtime options, and preparation against a fake Core: one version check +/// and prelude discovery per key across threads and runtimes, a child +/// holding a command's output killed, every Core command bounded by its +/// timeout, its output limit, and the token, bounded regular-file reads and +/// walks, a verified owner-only cache, and a bounded failure cleanup that +/// removes only what the preparation made. +/// +[TestFixture] +[Platform("Linux")] +public sealed class RuntimeTests +{ + private const string SlowCheck = "check) sleep 300 & echo $! > \"$ROOT/child.pid\"; wait ;;"; + + private string scratch = string.Empty; + + [SetUp] + public void Prepare() => scratch = SdkSupport.Scratch("runtime"); + + [TearDown] + public void CleanUp() + { + Runtime.FlightWaitHook = null; + Runtime.SourceReadHook = null; + Runtime.LoadedHook = null; + Runtime.PublishedHook = null; + Runtime.CleanupHook = null; + Core.DigestChunkHook = null; + Core.TreeFileHook = null; + Core.TreeEntryCap = Core.TreeEntries; + SdkSupport.RemoveScratch(scratch); + } + + private string At(string name) => Path.Combine(scratch, name); + + private static string Excerpt(char c, int length) => new string(c, 64) + "... (" + length.ToString(CultureInfo.InvariantCulture) + " chars)"; + + [Test] + public void RuntimeOptionsAreValidatedBeforeCoreRuns() + { + var baseOptions = new RuntimeOptions { Prelude = scratch }; + Assert.That(Support.Throws(() => _ = new Runtime(baseOptions with { Deadline = TimeSpan.Zero })).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That( + Support.Throws(() => _ = new Runtime(baseOptions with { Limits = Limits.Hard.With("max_frame_bytes", 0) })).Message, + Does.StartWith("invalid runtime limits")); + var runtime = new Runtime(baseOptions with { Binary = "/no/such/jacquard" }); + foreach (string[] sources in new[] { new[] { "rules.txt" }, Array.Empty(), new[] { "a.jac", "b.jqd" }, new[] { ".jac" } }) + { + JacquardException error = Support.Throws(() => runtime.Prepare(sources)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Preparation), string.Join(",", sources)); + Assert.That(error.Message, Is.EqualTo("pass .jac files or .jqd files in dependency order")); + } + + Assert.That(Support.Throws(() => runtime.Prepare(new[] { "rules.jac" })).Message, Does.Contain("cannot find Core executable /no/such/jacquard")); + Assert.That(runtime.CoreCommands, Is.EqualTo(0)); + + // A name longer than any path is refused with an excerpt, never used or copied whole. + var huge = new Runtime(baseOptions with { Binary = "/" + new string('x', 1_000_000) }); + Assert.That(Support.Throws(() => huge.Prepare(new[] { "rules.jac" })).Message, Does.StartWith("cannot find Core executable /" + new string('x', 63) + "... (1000001 chars)")); + string longer = Support.Throws(() => new Runtime(baseOptions with { Binary = "/no/such/" + new string('y', 1000) }).Prepare(new[] { "rules.jac" })).Message; + Assert.That(longer, Has.Length.LessThan(200)); + Assert.That(longer, Does.Contain("... (1009 chars)")); + + // Paths longer than any path are refused with an excerpt before the file system sees them. + string source = Support.Throws(() => runtime.Prepare(new[] { "/" + new string('s', 1_000_000) + ".jac" })).Message; + Assert.That(source, Does.StartWith("source /sss").And.EndWith("(1000005 chars) is longer than any path")); + string cache = Support.Throws(() => runtime.Prepare(new[] { "rules.jac" }, new PrepareOptions { CacheDir = "/" + new string('c', 1_000_000) })).Message; + Assert.That(cache, Does.StartWith("cache directory /ccc")); + Assert.That(cache, Has.Length.LessThan(200)); + + // A relative path is measured as the file system will see it: 4,080 chars grow past the limit. + string relative = Support.Throws(() => runtime.Prepare(new[] { "rules.jac" }, new PrepareOptions { CacheDir = new string('c', 4080) })).Message; + Assert.That(relative, Does.StartWith("cache directory ccc").And.EndWith("is longer than any path")); + Assert.That(Directory.Exists(new string('c', 200)), Is.False, "a directory was made"); + JacquardException prelude = Support.Throws(() => _ = new Runtime(baseOptions with { Prelude = "/" + new string('p', 1_000_000) })); + Assert.That(prelude.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(prelude.Message, Has.Length.LessThan(200)); + foreach (string nul in new[] { "jac\0quard", "/tmp/jac\0quard" }) + { + JacquardException error = Support.Throws(() => new Runtime(baseOptions with { Binary = nul }).Prepare(new[] { "rules.jac" })); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Preparation)); + Assert.That(error.Message, Does.StartWith("cannot find Core executable")); + } + + Assert.That(Support.Throws(() => runtime.Prepare(new[] { "ru\0les.jac" })).Message, Does.StartWith("source ru").And.EndWith("is not a path")); + Support.Throws(() => runtime.Prepare(new string[] { null! })); + } + + [Test] + public void EmptyRuntimeOptionsMeanUnset() + { + var runtime = new Runtime(new RuntimeOptions { Binary = string.Empty, Prelude = scratch }); + string? bin = Environment.GetEnvironmentVariable("JACQUARD_BIN"); + Assert.That(runtime.Binary, Is.EqualTo(string.IsNullOrEmpty(bin) ? "jacquard" : bin)); + Assert.That(runtime.Prelude, Is.EqualTo(Path.GetFullPath(scratch))); + } + + [Test] + public void VersionIsCheckedOnceAcrossThreadsAndRuntimes() + { + using FakeCore fake = FakeCore.Failing(At("once"), Compat.CoreVersion); + JacquardException[] errors = Enumerable.Range(0, 4).AsParallel().WithDegreeOfParallelism(4).Select(_ => fake.PrepareFails()).ToArray(); + foreach (JacquardException error in errors) + { + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Preparation)); + Assert.That(error.Message, Is.EqualTo("Core check failed: boom")); + } + + Assert.That(fake.Calls("--version"), Is.EqualTo(1)); + Assert.That(fake.Calls("check"), Is.EqualTo(4)); + Assert.That(fake.StagingLeft(), Is.False, "a staging directory was left behind"); + + // Another runtime for the same Core shares the check. + var other = new Runtime(new RuntimeOptions { Binary = fake.Binary, Prelude = Path.Combine(fake.Root, "prelude") }); + Support.Throws(() => other.Prepare(new[] { fake.Source("rules.jac") }, fake.Options)); + Assert.That(fake.Calls("--version"), Is.EqualTo(1)); + Assert.That(other.CoreCommands, Is.EqualTo(1)); + } + + /// + /// A Core command whose child keeps its output open does not hold the + /// preparation: the command's group is killed when it exits, so the child + /// dies and preparation goes on (and here fails at run). + /// + [Test] + public void ACoreCommandsChildHoldingItsOutputIsKilledAtTheExit() + { + Assume.That(CoreCommand.Setsid, Is.Not.Null, "no setsid at /usr/bin or /bin"); + using FakeCore fake = FakeCore.Create(At("holder"), Compat.CoreVersion, null, + "check) sleep 300 & echo $! > \"$ROOT/holder.pid\" ;;\nhash) echo '0:x " + new string('0', 64) + "' ;;\nrun) exit 4 ;;"); + Assert.That(fake.PrepareFails().Message, Is.EqualTo("Core run failed: exit status 4")); + SdkSupport.Dies("the command's child", fake.Pid("holder.pid")); + Assert.That(fake.StagingLeft(), Is.False); + } + + [Test] + public void AVersionMismatchIsRefusedAndCheckedAgain() + { + using FakeCore fake = FakeCore.Failing(At("mismatch"), "9.9.9"); + for (int i = 0; i < 2; i++) + { + string message = fake.PrepareFails().Message; + Assert.That(message, Does.Contain("expected pinned Core version").And.Contain("jacquard 9.9.9")); + } + + Assert.That(fake.Calls("--version"), Is.EqualTo(2)); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + [Test] + public void PreludeDiscoveryRunsOncePerCoreAndPrelude() + { + using FakeCore fake = FakeCore.Hashing(At("prelude")); + foreach (string source in new[] { "rules.jac", "other.jac" }) + { + Assert.That(fake.PrepareFails(source).Message, Is.EqualTo("Core run failed: exit status 4")); + } + + // Three prelude files once, then one local hash per source. + Assert.That(fake.Calls("hash"), Is.EqualTo(5)); + Assert.That(fake.Calls("--version"), Is.EqualTo(1)); + Assert.That(fake.Calls("run"), Is.EqualTo(2)); + Assert.That(fake.StagingLeft(), Is.False); + } + + [Test] + public void CoreOutputBeyondItsLimitIsRefused() + { + using FakeCore fake = FakeCore.Create(At("overflow"), Compat.CoreVersion, null, "check) head -c 9000000 /dev/zero ;;"); + Assert.That(fake.PrepareFails().Message, Is.EqualTo("Core check output exceeds 8 MiB")); + } + + [Test] + public void ACoreCommandThatOutlivesItsTimeoutIsKilledWithItsChild() + { + using FakeCore fake = FakeCore.Create(At("timeout"), Compat.CoreVersion, null, SlowCheck); + fake.Runtime.CommandTimeout = TimeSpan.FromMilliseconds(500); + Assert.That(fake.PrepareFails().Message, Is.EqualTo("Core check failed: no exit within 500 ms")); + SdkSupport.Dies("the timed-out command's child", fake.Pid("child.pid")); + Assert.That(fake.StagingLeft(), Is.False); + } + + [Test] + public void CancellationStopsTheRunningCoreCommand() + { + using FakeCore fake = FakeCore.Create(At("cancel"), Compat.CoreVersion, null, SlowCheck); + using var cancel = new CancellationTokenSource(); + Thread canceller = Support.After(TimeSpan.Zero, () => + { + fake.Pid("child.pid"); + cancel.Cancel(); + }); + Assert.That(fake.PrepareFails("rules.jac", cancel.Token).Message, Is.EqualTo("Core check failed: preparation was cancelled")); + SdkSupport.Dies("the cancelled command's child", fake.Pid("child.pid")); + canceller.Join(); + + // The cleanup observes the stop too, so the unpublished staging directory stays. + Assert.That(fake.StagingLeft(), Is.True); + } + + /// + /// One preparation runs a slow --version; another waits for that + /// check. The waiter observes its own cancellation while the first still + /// runs, and never runs the command. + /// + [Test] + public void APreparationWaitingForAnotherObservesItsCancellation() + { + string slowVersion = "sleep 300 & echo $! > \"$ROOT/version.pid\"; wait; echo 'jacquard " + Compat.CoreVersion + "'"; + using FakeCore fake = FakeCore.Create(At("waiter"), Compat.CoreVersion, slowVersion, "check) exit 3 ;;"); + using var first = new CancellationTokenSource(); + Task owner = Task.Factory.StartNew(() => fake.PrepareFails("rules.jac", first.Token), CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); + int sleeper = fake.Pid("version.pid"); + + // The seam cancels each waiter only once it waits, so the wait itself must notice. + using var waiter = new CancellationTokenSource(); + Runtime.FlightWaitHook = waiter.Cancel; + Assert.That(fake.PrepareFails("rules.jac", waiter.Token).Message, Is.EqualTo("preparation was cancelled")); + Runtime.FlightWaitHook = null; + Assert.That(Support.Alive(sleeper), Is.True, "the first preparation's command stopped"); + Assert.That(fake.Calls("--version"), Is.EqualTo(1)); + first.Cancel(); + Assert.That(owner.Wait(TimeSpan.FromMinutes(1)), Is.True); + Assert.That(owner.Result.Message, Is.EqualTo("Core --version failed: preparation was cancelled")); + SdkSupport.Dies("the first preparation's command", sleeper); + } + + /// + /// Prepares on another thread, which must finish within a minute. If it + /// is blocked opening , a writer is opened to + /// release it and the test fails. + /// + private static JacquardException PreparesWithoutBlocking(FakeCore fake, string[] sources, string fifo) + { + Task preparing = Task.Factory.StartNew( + () => + { + try + { + fake.Runtime.Prepare(sources, fake.Options); + return null; + } + catch (JacquardException error) + { + return error; + } + }, + CancellationToken.None, + TaskCreationOptions.LongRunning, + TaskScheduler.Default); + if (!preparing.Wait(TimeSpan.FromMinutes(1))) + { + using (var writer = new FileStream(fifo, FileMode.Open, FileAccess.Write)) + { + writer.WriteByte((byte)'\n'); + } + + preparing.Wait(TimeSpan.FromMinutes(1)); + Assert.Fail("the preparation blocked on " + fifo); + } + + Assert.That(preparing.Result, Is.Not.Null, "the preparation succeeded"); + return preparing.Result!; + } + + [Test] + public void ASourceThatIsNotARegularFileIsRefusedWithoutBlocking() + { + using FakeCore fake = FakeCore.Failing(At("kinds"), Compat.CoreVersion); + string fifo = Path.Combine(fake.Root, "fifo.jac"); + SdkSupport.MakeFifo(fifo); + Assert.That(PreparesWithoutBlocking(fake, new[] { fifo }, fifo).Message, Is.EqualTo("cannot read " + fifo + ": not a regular file")); + string directory = Directory.CreateDirectory(Path.Combine(fake.Root, "directory.jac")).FullName; + Assert.That(Support.Throws(() => fake.Runtime.Prepare(new[] { directory }, fake.Options)).Message, Is.EqualTo("cannot read " + directory + ": not a regular file")); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + + // A symbolic link to a regular file is read. + string link = Path.Combine(fake.Root, "link.jac"); + File.CreateSymbolicLink(link, fake.Source("rules.jac")); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(new[] { link }, fake.Options)).Message, Is.EqualTo("Core check failed: boom")); + } + + [Test] + public void SourcesOverTheCeilingAreRefusedBeforeTheyAreRead() + { + using FakeCore fake = FakeCore.Failing(At("ceiling"), Compat.CoreVersion); + var read = new List(); + Runtime.SourceReadHook = path => + { + lock (read) + { + read.Add(path); + } + }; + + // Sparse and unreadable, so a refusal that is not "cannot read" was made from the size alone. + string one = SdkSupport.Sparse(Path.Combine(fake.Root, "one.jac"), Runtime.SourceLimit + 1); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(new[] { one }, fake.Options)).Message, Is.EqualTo("sources exceed 16 MiB in total")); + string half = SdkSupport.Sparse(Path.Combine(fake.Root, "half.jac"), Runtime.SourceLimit / 2, UnixFileMode.UserRead); + string other = SdkSupport.Sparse(Path.Combine(fake.Root, "other.jac"), Runtime.SourceLimit / 2); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(new[] { half, other }, fake.Options)).Message, Is.EqualTo("sources exceed 16 MiB in total")); + Assert.That(read, Is.EqualTo(new[] { half }), "a source past the ceiling was read"); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + [Test] + public void ACancellationIsObservedBetweenSourceFiles() + { + using FakeCore fake = FakeCore.Failing(At("between"), Compat.CoreVersion); + string[] sources = { fake.Source("a.jac"), fake.Source("b.jac") }; + var read = new List(); + using var cancel = new CancellationTokenSource(); + Runtime.SourceReadHook = path => + { + read.Add(path); + cancel.Cancel(); + }; + Assert.That(Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options, cancel.Token)).Message, Is.EqualTo(Core.Aborted)); + Assert.That(read, Is.EqualTo(new[] { sources[0] })); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + [Test] + public void ThePreludeWalkReadsOnlyBoundedRegularFiles() + { + using FakeCore fake = FakeCore.Failing(At("walk"), Compat.CoreVersion); + string big = SdkSupport.Sparse(Path.Combine(fake.Root, "prelude", "zz-big.jqd"), Core.TreeLimit + 1); + Assert.That(fake.PrepareFails().Message, Does.StartWith("cannot read prelude").And.EndWith("exceed 256 MiB")); + File.Delete(big); + string fifo = Path.Combine(fake.Root, "prelude", "zz-fifo.jqd"); + SdkSupport.MakeFifo(fifo); + Assert.That(PreparesWithoutBlocking(fake, new[] { fake.Source("rules.jac") }, fifo).Message, Does.EndWith("unsupported filesystem entry in prepared artifacts: zz-fifo.jqd")); + File.Delete(fifo); + File.WriteAllText(Path.Combine(fake.Root, "prelude", "02-more.jqd"), "more\n"); + var digested = new List(); + using var cancel = new CancellationTokenSource(); + Core.TreeFileHook = path => + { + digested.Add(path); + cancel.Cancel(); + }; + Assert.That(fake.PrepareFails("rules.jac", cancel.Token).Message, Does.EndWith(Core.Aborted)); + + // The walk streams each directory, so either file may come first. + Assert.That(digested, Has.Count.EqualTo(1)); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + /// A walk thousands of directories deep keeps an explicit stack, so it is refused by the entry cap or read, never by the call stack. + [Test] + public void ADeepPreludeTreeIsWalkedWithoutRecursion() + { + using FakeCore fake = FakeCore.Failing(At("deep"), Compat.CoreVersion); + string directory = Path.Combine(fake.Root, "prelude"); + for (int i = 0; i < 1500; i++) + { + directory = Path.Combine(directory, "d"); + } + + Directory.CreateDirectory(directory); + File.WriteAllText(Path.Combine(directory, "leaf.jqd"), "leaf\n"); + Assert.That(fake.PrepareFails().Message, Is.EqualTo("Core check failed: boom")); + } + + [Test] + public void AChangedCacheEntryIsRefusedIncludingItsMetadata() + { + using FakeCore fake = FakeCore.Programs(At("verified")); + string[] sources = { fake.Source("rules.jac") }; + PreparedProgram program = fake.Runtime.Prepare(sources, fake.Options); + Assert.That(program.Data.Identity("first").Hex, Is.EqualTo(FakeCore.First)); + Assert.That(File.Exists(Path.Combine(program.Directory, Runtime.HashOutput)), Is.True); + int commands = fake.Runtime.CoreCommands; + Assert.That(fake.Runtime.Prepare(sources, fake.Options).Directory, Is.EqualTo(program.Directory)); + Assert.That(fake.Runtime.CoreCommands, Is.EqualTo(commands), "a warm preparation ran Core"); + string metadata = Path.Combine(program.Directory, "program.json"); + string original = File.ReadAllText(metadata); + var changes = new[] + { + ("[\"first\",\"" + FakeCore.First + "\"]", "[\"first\",\"" + FakeCore.Second + "\"]"), + ("{\"hash\":\"" + FakeCore.First + "\",\"name\":\"first\"}", "{\"hash\":\"" + FakeCore.Second + "\",\"name\":\"first\"}"), + ("\"first\":\"() ->{} Int\"", "\"first\":\"() ->{} Text\""), + }; + foreach ((string from, string to) in changes) + { + Replace(metadata, from, to); + JacquardException error = Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options), "a changed program.json was accepted: " + to); + Assert.That(error.Message, Does.Contain("program.json bindings differ from Core's cached output")); + File.WriteAllText(metadata, original); + } + + Assert.That(fake.Runtime.Prepare(sources, fake.Options).Directory, Is.EqualTo(program.Directory)); + string hash = Path.Combine(program.Directory, Runtime.HashOutput); + string output = File.ReadAllText(hash); + Replace(hash, "0:first " + FakeCore.First, "0:first " + FakeCore.Second); + string changed = File.ReadAllText(hash); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options)).Message, Does.Contain("cached artifact digests do not match")); + Replace(metadata, Core.Digest(Encoding.UTF8.GetBytes(output)), Core.Digest(Encoding.UTF8.GetBytes(changed))); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options)).Message, Does.Contain("program.json bindings differ")); + + // A file replaced by a symbolic link is refused, not followed. + File.Delete(metadata); + File.CreateSymbolicLink(metadata, hash); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options)).Message, Does.Contain("unsupported filesystem entry in prepared artifacts: program.json")); + Assert.That(fake.Runtime.CoreCommands, Is.EqualTo(commands)); + } + + private static void Replace(string file, string from, string to) + { + string text = File.ReadAllText(file); + Assert.That(text, Does.Contain(from), from + " is not in " + file); + File.WriteAllText(file, text.Replace(from, to, StringComparison.Ordinal)); + } + + /// + /// A new cache entry is owner-only throughout (Core's store included), + /// and the cache root, the entry, or any file or directory in it that its + /// group may write is refused on load; restoring the permission makes the + /// entry load again. + /// + [Test] + public void AGroupWritableCacheEntryIsRefused() + { + using FakeCore fake = FakeCore.Programs(At("owner")); + string[] sources = { fake.Source("rules.jac") }; + PreparedProgram program = fake.Runtime.Prepare(sources, fake.Options); + string entry = program.Directory; + var targets = Directory.EnumerateFileSystemEntries(entry, "*", SearchOption.AllDirectories).Append(entry).ToList(); + Assert.That(targets, Does.Contain(Path.Combine(entry, "store", "objects"))); + foreach (string path in targets) + { + UnixFileMode mode = File.GetUnixFileMode(path); + Assert.That(mode & ~(UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute), Is.EqualTo(UnixFileMode.None), path + " is not owner-only: " + mode); + } + + string[] checkedPaths = + { + Path.GetFullPath(fake.Options.CacheDir!), entry, Path.Combine(entry, "program.json"), Path.Combine(entry, Runtime.HashOutput), Path.Combine(entry, "core"), + Path.Combine(entry, "store"), Path.Combine(entry, "store", "objects"), + }; + foreach (string path in checkedPaths) + { + UnixFileMode original = File.GetUnixFileMode(path); + File.SetUnixFileMode(path, original | UnixFileMode.GroupWrite); + JacquardException error = Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options), "a group-writable " + path + " was accepted"); + Assert.That(error.Message, Does.Contain(path + " is group- or world-writable")); + File.SetUnixFileMode(path, original); + } + + Assert.That(fake.Runtime.Prepare(sources, fake.Options).Directory, Is.EqualTo(entry)); + } + + /// + /// A cache root this user does not own is refused before anything is + /// written to it. The root directory stands in for a foreign owner; a + /// foreign-owned entry inside an owned root would need another account, + /// so it is not tested. + /// + [Test] + public void AForeignOwnedCacheRootIsRefused() + { + Assume.That(Unix.EffectiveUser(), Is.Not.EqualTo(0u), "running as root"); + using FakeCore fake = FakeCore.Programs(At("foreign")); + JacquardException error = Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = "/" })); + Assert.That(error.Message, Does.Contain("cache path / is not owned by the current user")); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + private static Runtime RuntimeFor(FakeCore fake, string binary) => new(new RuntimeOptions { Binary = binary, Prelude = Path.Combine(fake.Root, "prelude") }); + + /// An executable over the 512 MiB fingerprint ceiling is refused from its size: it is sparse and unreadable, so a read would fail with "cannot read" instead. + [Test] + public void AnExecutableOverTheFingerprintCeilingIsRefusedBeforeItIsRead() + { + using FakeCore fake = FakeCore.Failing(At("huge"), Compat.CoreVersion); + string huge = SdkSupport.Sparse(Path.Combine(fake.Root, "huge-jacquard"), Core.ExecutableLimit + 1, UnixFileMode.UserWrite | UnixFileMode.UserExecute); + string message = Support.Throws(() => RuntimeFor(fake, huge).Prepare(new[] { fake.Source("rules.jac") }, fake.Options)).Message; + Assert.That(message, Does.StartWith("Core executable ").And.EndWith(" exceeds 512 MiB")); + } + + /// A cancellation during fingerprinting stops it after the piece being read (a test seam counts the pieces of a 16 MiB executable). + [Test] + public void AStopDuringFingerprintingIsObserved() + { + using FakeCore fake = FakeCore.Failing(At("digest"), Compat.CoreVersion); + string big = SdkSupport.Sparse(Path.Combine(fake.Root, "big-jacquard"), 16L << 20, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + int pieces = 0; + using var cancel = new CancellationTokenSource(); + Core.DigestChunkHook = () => + { + pieces++; + cancel.Cancel(); + }; + Assert.That(Support.Throws(() => RuntimeFor(fake, big).Prepare(new[] { fake.Source("rules.jac") }, fake.Options, cancel.Token)).Message, Is.EqualTo(Core.Aborted)); + Assert.That(pieces, Is.EqualTo(1)); + Assert.That(fake.Calls("--version"), Is.EqualTo(0)); + } + + /// + /// A cancellation after the first of two empty prelude files stops the + /// walk before the second, whichever comes first: an empty file has no + /// piece to read, so only the check before each entry can see the stop. + /// + [Test] + public void AStopBetweenEmptyFilesIsObserved() + { + using FakeCore fake = FakeCore.Failing(At("empty"), Compat.CoreVersion); + File.WriteAllText(Path.Combine(fake.Root, "prelude", "01-prim.jqd"), string.Empty); + File.WriteAllText(Path.Combine(fake.Root, "prelude", "02-empty.jqd"), string.Empty); + var digested = new List(); + using var cancel = new CancellationTokenSource(); + Core.TreeFileHook = path => + { + digested.Add(path); + cancel.Cancel(); + }; + Assert.That(fake.PrepareFails("rules.jac", cancel.Token).Message, Does.EndWith(Core.Aborted)); + Assert.That(digested, Has.Count.EqualTo(1)); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + /// A tree over the entry cap is refused, zero-byte files counted (a test seam lowers the cap to 3). + [Test] + public void ATreeOverTheEntryCapIsRefused() + { + Assert.That(Core.TreeEntries, Is.EqualTo(100_000)); + using FakeCore fake = FakeCore.Failing(At("entries"), Compat.CoreVersion); + foreach (string name in new[] { "a.jqd", "b.jqd", "c.jqd" }) + { + File.WriteAllText(Path.Combine(fake.Root, "prelude", name), string.Empty); + } + + Core.TreeEntryCap = 3; + string prelude = Path.Combine(fake.Root, "prelude"); + Assert.That(fake.PrepareFails().Message, Does.StartWith("cannot read prelude").And.EndWith("more than 3 entries under " + prelude)); + Assert.That(fake.Calls("check"), Is.EqualTo(0)); + } + + /// A source list that counts the elements read, and fails a read past limit. + private sealed class CountingList : IReadOnlyList + { + private readonly string element; + private readonly int limit; + private readonly Action onRead; + + internal CountingList(int count, string element, int limit, Action onRead) + { + Count = count; + this.element = element; + this.limit = limit; + this.onRead = onRead; + } + + internal List Indices { get; } = new(); + + public int Count { get; } + + public string this[int index] + { + get + { + if (Indices.Count >= limit) + { + throw new InvalidOperationException("read past the source cap"); + } + + Indices.Add(index); + onRead(index); + return element; + } + } + + public IEnumerator GetEnumerator() => throw new InvalidOperationException("the list was enumerated"); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// A source list over 1,024 is refused from its count, before any element is read. A list of exactly 1,024 passes that check. + [Test] + public void AHugeSourceListIsRefusedBeforeAnyElementIsRead() + { + Assert.That(Runtime.SourceFiles, Is.EqualTo(1024)); + using FakeCore fake = FakeCore.Programs(At("count")); + string source = fake.Source("rules.jac"); + var huge = new CountingList(int.MaxValue, source, 2048, _ => { }); + JacquardException error = Support.Throws(() => fake.Runtime.Prepare(huge, fake.Options)); + Assert.That(error.Message, Is.EqualTo("more than 1024 sources")); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Preparation)); + Assert.That(huge.Indices, Is.Empty); + var over = new CountingList(1025, source, 2048, _ => { }); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(over, fake.Options)).Message, Is.EqualTo("more than 1024 sources")); + Assert.That(over.Indices, Is.Empty); + var most = new CountingList(1024, source, 2048, _ => { }); + fake.Runtime.Prepare(most, fake.Options); + Assert.That(most.Indices, Has.Count.EqualTo(1024)); + } + + /// A cancellation while the list is validated stops it before the next element is read. + [Test] + public void AStopDuringSourceValidationIsObserved() + { + using FakeCore fake = FakeCore.Programs(At("validate")); + string source = fake.Source("rules.jac"); + using var cancel = new CancellationTokenSource(); + var cancelled = new CountingList(3, source, 3, index => + { + if (index == 1) + { + cancel.Cancel(); + } + }); + Assert.That(Support.Throws(() => fake.Runtime.Prepare(cancelled, fake.Options, cancel.Token)).Message, Is.EqualTo(Core.Aborted)); + Assert.That(cancelled.Indices, Is.EqualTo(new[] { 0, 1 })); + Assert.That(fake.Calls("--version"), Is.EqualTo(0)); + } + + /// A fake whose run leaves ten files in its store and fails. + private static FakeCore FailingStore(string root) => FakeCore.Create(root, Compat.CoreVersion, null, + "check) echo 'first : () ->{} Int' ;;\nhash) echo '0:first " + FakeCore.First + "' ;;\n" + + "run) mkdir -p \"$4\" && for i in 0 1 2 3 4 5 6 7 8 9; do : > \"$4/f$i\"; done; exit 4 ;;"); + + /// + /// A failed preparation's cleanup removes its staging directory, but a + /// cancellation during the cleanup ends it at once (a test seam sets the + /// stop after the second removal), leaving the unpublished staging + /// directory; nothing is published either way. + /// + [Test] + public void AStopDuringTheFailureCleanupEndsIt() + { + using FakeCore fake = FailingStore(At("cleanup")); + var removed = new List(); + Runtime.CleanupHook = removed.Add; + Assert.That(fake.PrepareFails().Message, Does.EndWith("Core run failed: exit status 4")); + Assert.That(removed, Has.Count.EqualTo(13)); + Assert.That(fake.CacheEntries(), Is.Empty); + removed.Clear(); + using var cancel = new CancellationTokenSource(); + Runtime.CleanupHook = path => + { + removed.Add(path); + if (removed.Count == 2) + { + cancel.Cancel(); + } + }; + Assert.That(fake.PrepareFails("rules.jac", cancel.Token).Message, Is.EqualTo(Core.Aborted)); + Assert.That(removed, Has.Count.EqualTo(2)); + List left = fake.CacheEntries(); + Assert.That(left, Has.Count.EqualTo(1)); + Assert.That(left[0], Does.StartWith("prepare-")); + Assert.That(Directory.Exists(Path.Combine(fake.Root, "cache", left[0], "store")), Is.True); + } + + /// The failure cleanup removes at most the tree cap's entries (a test seam lowers it to 3), leaving the rest unpublished. + [Test] + public void TheFailureCleanupStopsAtTheCap() + { + using FakeCore fake = FailingStore(At("capped")); + var removed = new List(); + Runtime.CleanupHook = removed.Add; + Core.TreeEntryCap = 3; + Assert.That(fake.PrepareFails().Message, Does.EndWith("Core run failed: exit status 4")); + Assert.That(removed, Has.Count.EqualTo(3)); + List left = fake.CacheEntries(); + Assert.That(left, Has.Count.EqualTo(1)); + Assert.That(left[0], Does.StartWith("prepare-")); + } + + /// + /// A cache root reached through a symbolic link is checked as the + /// directory it names (the link's own mode reads 0777), and its entries + /// live there; a group-writable real root is refused with a hint. + /// + [Test] + public void ASymlinkedCacheRootIsFollowedToItsRealDirectory() + { + using FakeCore fake = FakeCore.Programs(At("link")); + string real = Directory.CreateDirectory(Path.Combine(fake.Root, "real-cache"), UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute).FullName; + string link = Path.Combine(fake.Root, "linked-cache"); + Directory.CreateSymbolicLink(link, real); + var options = new PrepareOptions { CacheDir = link }; + PreparedProgram program = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options); + Assert.That(Path.GetDirectoryName(program.Directory), Is.EqualTo(real)); + Assert.That(Directory.EnumerateFileSystemEntries(real).Count(), Is.EqualTo(1)); + Assert.That(fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options).Directory, Is.EqualTo(program.Directory)); + File.SetUnixFileMode(real, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options)).Message, + Is.EqualTo("cache path " + real + " is group- or world-writable; only its owner may write the cache (make it owner-only, for example chmod 700 " + real + ")")); + } + + /// A preparation removes only its own staging directory: directories it did not make, whatever their names or ages, are left alone. + [Test] + public void APreparationLeavesDirectoriesItDidNotMake() + { + using FakeCore fake = FakeCore.Programs(At("leaves")); + string cache = Directory.CreateDirectory(Path.Combine(fake.Root, "cache"), UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute).FullName; + string notes = Directory.CreateDirectory(Path.Combine(cache, "prepare-release-notes")).FullName; + File.WriteAllText(Path.Combine(notes, "draft.txt"), "keep"); + string left = Directory.CreateDirectory(Path.Combine(cache, "prepare-1-00000000-0")).FullName; + Directory.SetLastWriteTimeUtc(notes, DateTime.UtcNow.AddDays(-1)); + Directory.SetLastWriteTimeUtc(left, DateTime.UtcNow.AddDays(-1)); + string entry = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options).Directory; + Assert.That(File.ReadAllText(Path.Combine(notes, "draft.txt")), Is.EqualTo("keep")); + Assert.That(Directory.Exists(left), Is.True, "a directory the preparation did not make was removed"); + Assert.That(fake.CacheEntries(), Is.EqualTo(new[] { Path.GetFileName(entry), Path.GetFileName(left), Path.GetFileName(notes) }.OrderBy(name => name, StringComparer.Ordinal))); + } + + /// Once an entry is published its staging path is vacated, and whatever appears there afterwards is not the preparation's: it is left alone. + [Test] + public void ADirectoryAtTheVacatedStagingPathIsLeftAlone() + { + using FakeCore fake = FakeCore.Programs(At("vacated")); + string? replacement = null; + Runtime.PublishedHook = staging => + { + replacement = Directory.CreateDirectory(staging).FullName; + File.WriteAllText(Path.Combine(staging, "unrelated.txt"), "keep"); + }; + fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + Assert.That(File.ReadAllText(Path.Combine(replacement!, "unrelated.txt")), Is.EqualTo("keep"), "the preparation removed a directory it did not make"); + } + + /// A cancellation that lands as a warm load finishes fails the preparation instead of returning a program. + [Test] + public void AStopAsAWarmLoadFinishesIsReported() + { + using FakeCore fake = FakeCore.Programs(At("loaded")); + string[] sources = { fake.Source("rules.jac") }; + fake.Runtime.Prepare(sources, fake.Options); + using var cancel = new CancellationTokenSource(); + Runtime.LoadedHook = cancel.Cancel; + Assert.That(Support.Throws(() => fake.Runtime.Prepare(sources, fake.Options, cancel.Token)).Message, Is.EqualTo(Core.Aborted)); + Runtime.LoadedHook = null; + + // Without a stop the same entry loads. + fake.Runtime.Prepare(sources, fake.Options); + } +} From 7e75869b163564cc9235c360e420bbaaa0a5d0d6 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 07:04:33 -0400 Subject: [PATCH 04/52] Add live .NET SDK tests Port the Java SDK's live suite: quickstart calls and saved inspection files, the 64-row triage oracle in explicit batches, local refusals before any worker, binding completeness, unknown completion with raw evidence, the 25-case planner oracle, cache reuse and corruption, a signature too deep to bind, a redirected name in program.json, the full Int range and Real bits, cancellation, output overflow, unit results kept after a later batch fails, and result-mapping failures. The README examples under Readme/ run here too. --- .../JacquardHost.Tests/Readme/Documents.cs | 26 + .../JacquardHost.Tests/Readme/Incidents.cs | 54 ++ .../JacquardHost.Tests/Readme/Shipping.cs | 22 + .../JacquardHost.Tests/Sdk/SdkLiveTests.cs | 698 ++++++++++++++++++ 4 files changed, 800 insertions(+) create mode 100644 dotnet/test/JacquardHost.Tests/Readme/Documents.cs create mode 100644 dotnet/test/JacquardHost.Tests/Readme/Incidents.cs create mode 100644 dotnet/test/JacquardHost.Tests/Readme/Shipping.cs create mode 100644 dotnet/test/JacquardHost.Tests/Sdk/SdkLiveTests.cs diff --git a/dotnet/test/JacquardHost.Tests/Readme/Documents.cs b/dotnet/test/JacquardHost.Tests/Readme/Documents.cs new file mode 100644 index 0000000..21e4e05 --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Readme/Documents.cs @@ -0,0 +1,26 @@ +namespace JacquardLang.Host.Tests.Readme; + +using System.Collections.Generic; +using JacquardLang.Host; +using JacquardLang.Host.Sdk; + +public static class Documents +{ + // summarize : (Text) ->{Documents} Text, where Documents has one operation, + // read-document : (Text) -> Text. + public static Function Summarize(PreparedProgram program) + { + Operations documents = new Operations().On("read-document", Codecs.Text, Codecs.Text, (context, key) => + { + // The context reports a cancelled or ended invocation; its + // CancellationToken is cancelled then too. + if (context.IsCancelled) + { + throw HandlerException.CompletionUnknown("cancelled"); + } + + return key == "example" ? "A synthetic document supplied by .NET." : throw HandlerException.Refused("unknown document"); + }); + return program.Function("summarize", new Dictionary { ["Documents"] = documents }); + } +} diff --git a/dotnet/test/JacquardHost.Tests/Readme/Incidents.cs b/dotnet/test/JacquardHost.Tests/Readme/Incidents.cs new file mode 100644 index 0000000..28a2ef1 --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Readme/Incidents.cs @@ -0,0 +1,54 @@ +namespace JacquardLang.Host.Tests.Readme; + +using System.Collections.Generic; +using JacquardLang.Host.Sdk; + +// type Incident = | Incident(id: Text, service: Text, severity: Int, age: Int, visible: Bool, owner: Option Text) +// type Urgency = | PageNow | WorkToday | WatchOnly +// type Triage = | Routed Text Text Urgency (List Text) | InvalidIncident Text +public static class Incidents +{ + public sealed record Incident(string Id, string Service, long Severity, long Age, bool Visible, string? Owner); + + public enum Urgency + { + PageNow, + WorkToday, + WatchOnly, + } + + public abstract record Triage; + + public sealed record Routed(string Id, string Team, Urgency Urgency, IReadOnlyList Reasons) : Triage; + + public sealed record InvalidIncident(string Id) : Triage; + + // A record is one constructor: its fields in order, each with a codec and + // a reader, and a function that builds it from the decoded fields. + public static readonly RecordCodec IncidentCodec = Codecs.Record( + "Incident", + Codecs.Field("id", Codecs.Text, (Incident incident) => incident.Id), + Codecs.Field("service", Codecs.Text, (Incident incident) => incident.Service), + Codecs.Field("severity", Codecs.Int, (Incident incident) => incident.Severity), + Codecs.Field("age", Codecs.Int, (Incident incident) => incident.Age), + Codecs.Field("visible", Codecs.Bool, (Incident incident) => incident.Visible), + Codecs.Field("owner", Codecs.Nullable(Codecs.Text), (Incident incident) => incident.Owner), + (id, service, severity, age, visible, owner) => new Incident(id, service, severity, age, visible, owner)); + + // An enumeration's members name nullary constructors: PageNow is page-now. + // A union's cases are records, enumerations, or unions of its subtypes. + public static readonly UnionCodec TriageCodec = Codecs.Union( + Codecs.Record( + "Routed", + Codecs.Field("id", Codecs.Text, (Routed routed) => routed.Id), + Codecs.Field("team", Codecs.Text, (Routed routed) => routed.Team), + Codecs.Field("urgency", Codecs.Enum(), (Routed routed) => routed.Urgency), + Codecs.Field("reasons", Codecs.List(Codecs.Text), (Routed routed) => routed.Reasons), + (id, team, urgency, reasons) => new Routed(id, team, urgency, reasons)), + Codecs.Record("InvalidIncident", Codecs.Field("id", Codecs.Text, (InvalidIncident invalid) => invalid.Id), id => new InvalidIncident(id))); + + // triage.batch : (TriagePolicy, List Incident) ->{} List Triage, run in + // explicit batches of eight; each batch is checked before any runs. + public static IReadOnlyList Route(Function batch, Dynamic policy, IReadOnlyList incidents) => + batch.MapBatches(TriageCodec, IncidentCodec, incidents, 8, new Argument[] { policy }); +} diff --git a/dotnet/test/JacquardHost.Tests/Readme/Shipping.cs b/dotnet/test/JacquardHost.Tests/Readme/Shipping.cs new file mode 100644 index 0000000..8cf8f3a --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Readme/Shipping.cs @@ -0,0 +1,22 @@ +namespace JacquardLang.Host.Tests.Readme; + +using JacquardLang.Host.Sdk; + +public static class Shipping +{ + public static long Cost(string rules, string cacheDir, string inspection) + { + // Core is $JACQUARD_BIN and $JACQUARD_PRELUDE unless RuntimeOptions say otherwise. + var runtime = new Runtime(); + PreparedProgram program = runtime.Prepare(new[] { rules }, new PrepareOptions { CacheDir = cacheDir }); + Function shipping = program.Function("shipping-cost"); + + // A long converts to an Int argument; Call decodes the result. + long free = shipping.Call(Codecs.Int, 75L); + + // Invoke keeps the raw frames and evidence, saved only on request. + Invocation invocation = shipping.Invoke(35L); + invocation.Save(inspection); + return invocation.Decode(Codecs.Int) + free; + } +} diff --git a/dotnet/test/JacquardHost.Tests/Sdk/SdkLiveTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/SdkLiveTests.cs new file mode 100644 index 0000000..bd1444f --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/SdkLiveTests.cs @@ -0,0 +1,698 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; +using JacquardLang.Host.Tests.Readme; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// +/// Live SDK tests against the real Core named by JACQUARD_BIN and +/// JACQUARD_PRELUDE. Each is ignored without them, and fails instead +/// when JACQUARD_REQUIRE_LIVE is set, as CI does. +/// +[TestFixture] +[Platform("Linux,MacOsX")] +public sealed class SdkLiveTests +{ + private string root = string.Empty; + private Runtime runtime = null!; + + internal static void RequireLive() + { + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("JACQUARD_BIN")) || string.IsNullOrEmpty(Environment.GetEnvironmentVariable("JACQUARD_PRELUDE"))) + { + if (Environment.GetEnvironmentVariable("JACQUARD_REQUIRE_LIVE") is not null) + { + Assert.Fail("live tests need JACQUARD_BIN and JACQUARD_PRELUDE"); + } + + Assert.Ignore("live tests need JACQUARD_BIN and JACQUARD_PRELUDE"); + } + } + + [SetUp] + public void Live() + { + RequireLive(); + root = SdkSupport.Scratch("live"); + runtime = new Runtime(); + } + + [TearDown] + public void CleanUp() + { + Operations.AfterDecodeHook = null; + if (root.Length > 0) + { + SdkSupport.RemoveScratch(root); + } + } + + /// The dotnet/ directory this test assembly was built from. + internal static string DotnetRoot() => + typeof(SdkLiveTests).Assembly.GetCustomAttributes().Single(attribute => attribute.Key == "DotnetRoot").Value!; + + internal static string Example(string project) => Path.GetFullPath(Path.Combine(DotnetRoot(), "..", "examples", project, "rules.jac")); + + private PrepareOptions Options => new() { CacheDir = Path.Combine(root, "cache") }; + + private PreparedProgram Prepare(string source) => runtime.Prepare(new[] { source }, Options); + + private string Source(string name, string text) + { + string path = Path.Combine(root, name); + File.WriteAllText(path, text); + return path; + } + + private static Dynamic Policy(PreparedProgram program) => program.Data.Make("TriagePolicy", Dynamic.Of(4L), Dynamic.Of(60L)); + + [Test] + public void QuickstartCallsInvocationsAndSavedFrames() + { + Function shipping = Prepare(Example("quickstart")).Function("shipping-cost"); + Assert.That(shipping.Call(Codecs.Int, 35), Is.EqualTo(5L)); + Assert.That(shipping.Call(Codecs.Int, 75L), Is.EqualTo(0L)); + Invocation invocation = shipping.Invoke(35); + Assert.That(invocation.ToDynamic(), Is.EqualTo(Dynamic.Of(5L))); + Assert.That(invocation.Render(), Is.EqualTo("shipping-cost: ok")); + Assert.That(invocation.Error, Is.Null); + string path = Path.Combine(root, "inspection.json"); + invocation.Save(path); + byte[] raw = File.ReadAllBytes(path); + Assert.That(raw[^1], Is.EqualTo((byte)'\n')); + JsonObject saved = JsonCodec.DecodeObject(raw.AsSpan(0, raw.Length - 1), 128); + Assert.That(saved["schema"], Is.EqualTo(Json.Of("jacquard-host-inspection-v1"))); + Assert.That(saved["core_frames"].AsArray()!, Has.Count.EqualTo(invocation.Raw.CoreFrames.Count)); + Assert.That(File.GetUnixFileMode(path), Is.EqualTo(UnixFileMode.UserRead | UnixFileMode.UserWrite)); + } + + /// The README's SDK examples, compiled from Readme/, against Core. + [Test] + public void TheReadmeExamplesRun() + { + string saved = Path.Combine(root, "shipping.json"); + Assert.That(Shipping.Cost(Example("quickstart"), Path.Combine(root, "cache"), saved), Is.EqualTo(5L)); + Assert.That(File.Exists(saved), Is.True); + PreparedProgram documents = Prepare(Source( + "documents.jac", + "once effect Documents where {\n read-document : (Text) -> Text\n}\n\nsummarize : (Text) ->{Documents} Text\nsummarize(key) = read-document(key)\n")); + Function summarize = Documents.Summarize(documents); + Assert.That(summarize.Call(Codecs.Text, "example"), Is.EqualTo("A synthetic document supplied by .NET.")); + JacquardException refused = Support.Throws(() => summarize.Call(Codecs.Text, "other")); + Assert.That(refused.Kind, Is.EqualTo(JacquardErrorKind.Invocation)); + Assert.That(refused.Failure, Is.EqualTo(Failure.OperationFailed), refused.Message); + PreparedProgram triage = Prepare(Example("triage")); + IReadOnlyList routed = Incidents.Route( + triage.Function("triage.batch"), Policy(triage), new[] { new Incidents.Incident("inc-1", "checkout", 5, 3, true, null) }); + Assert.That(routed, Is.EqualTo(new[] { new Incidents.Routed("inc-1", "checkout-oncall", Incidents.Urgency.PageNow, new[] { "unowned", "customer-impact" }) }).Using(new RoutedComparer())); + } + + /// Compares results by value, with their reason lists by items. + private sealed class RoutedComparer : IEqualityComparer + { + public bool Equals(Incidents.Triage? x, Incidents.Triage? y) => + (x, y) switch + { + (Incidents.Routed a, Incidents.Routed b) => a.Id == b.Id && a.Team == b.Team && a.Urgency == b.Urgency && a.Reasons.SequenceEqual(b.Reasons), + _ => Equals(x, y), + }; + + public int GetHashCode(Incidents.Triage obj) => 0; + } + + private static Incidents.Incident Incident(long i) => + new(i.ToString(CultureInfo.InvariantCulture), "api", (i % 5) + 1, i * 3, i % 2 == 0, i % 3 != 0 ? "ops" : null); + + [Test] + public void RecordsEnumsOptionsAndExplicitBatches() + { + PreparedProgram program = Prepare(Example("triage")); + Function route = program.Function("triage.batch"); + Argument[] policy = { Policy(program) }; + List rows = Enumerable.Range(0, 64).Select(i => Incident(i)).ToList(); + IReadOnlyList results = route.MapBatches(Incidents.TriageCodec, Incidents.IncidentCodec, rows, 8, policy); + Assert.That(results, Has.Count.EqualTo(rows.Count)); + + // An item codec that throws on the second batch: the failure names that batch, keeps the first, and keeps the codec's exception. + int decoded = 0; + Codec failing = Codecs.Create( + (encoder, value) => Incidents.TriageCodec.Encode(encoder, value), + (decoder, value) => ++decoded > 8 ? throw new InvalidOperationException("second batch") : Incidents.TriageCodec.Decode(decoder, value)); + JacquardException batch = Support.Throws(() => route.MapBatches(failing, Incidents.IncidentCodec, rows.GetRange(0, 16), 8, policy)); + Assert.That(batch.BatchIndex, Is.EqualTo(1)); + Assert.That(batch.Completed, Has.Count.EqualTo(8)); + Assert.That(batch.InnerException, Is.InstanceOf(), "the codec's exception is not kept"); + for (int i = 0; i < rows.Count; i++) + { + Incidents.Incident row = rows[i]; + bool page = row.Visible && row.Severity >= 4; + var reasons = new List(); + if (row.Owner is null) + { + reasons.Add("unowned"); + } + + if (row.Age >= 60) + { + reasons.Add("ageing"); + } + + if (page) + { + reasons.Add("customer-impact"); + } + + Incidents.Urgency urgency = page ? Incidents.Urgency.PageNow : row.Age >= 60 || row.Severity == 5 ? Incidents.Urgency.WorkToday : Incidents.Urgency.WatchOnly; + Assert.That(results[i], Is.EqualTo(new Incidents.Routed(row.Id, row.Owner ?? "api-oncall", urgency, reasons)).Using(new RoutedComparer()), "row " + row.Id); + } + + Assert.That(route.MapBatches(Incidents.TriageCodec, Incidents.IncidentCodec, Array.Empty(), 8, policy), Is.Empty); + } + + [Test] + public void InvalidInputIsRefusedBeforeAnyWorker() + { + PreparedProgram program = Prepare(Example("triage")); + + // A spawn would now fail as worker_spawn_failed, not as a local refusal. + program.SetWorkerCommand(new[] { "/no/such/jacquard" }); + Function route = program.Function("triage.batch"); + Argument[] policy = { Policy(program) }; + List rows = Enumerable.Range(0, 17).Select(i => Incident(i)).ToList(); + rows[16] = rows[16] with { Severity = Json.IntMax + 1 }; + JacquardException batch = Support.Throws(() => route.MapBatches(Incidents.TriageCodec, Incidents.IncidentCodec, rows, 8, policy)); + Assert.That(batch.BatchIndex, Is.EqualTo(-1)); + Assert.That(batch.Completed, Is.Empty); + Assert.That(batch.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(batch.Message, Does.Contain("argument[1][0].severity")); + Incidents.Incident[] many = Enumerable.Repeat(Incident(1), 64).ToArray(); + Codec> results = Codecs.List(Incidents.TriageCodec); + Assert.That( + Support.Throws(() => route.Call(results, policy[0], Codecs.List(Incidents.IncidentCodec).Arg(many))).Kind, + Is.EqualTo(JacquardErrorKind.InvalidArgument), + "over-deep single call"); + JacquardException notList = Support.Throws(() => route.Call(results, policy[0], "not a list")); + Assert.That(notList.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(notList.Message, Does.Contain("argument[1]")); + Assert.That(Support.Throws(() => route.Call(results, policy[0])).Message, Does.Contain("expected 2 arguments")); + } + + [Test] + public void MissingHandlersRefuseBindingAndPureCallsNeedNoServices() + { + PreparedProgram fulfillment = Prepare(Example("fulfillment")); + int calls = 0; + Operations inventory = new Operations().On("booking-status", Codecs.Dynamic, Codecs.Dynamic, order => + { + calls++; + return Dynamic.Unit; + }); + JacquardException missing = Support.Throws(() => fulfillment.Function("fulfillment.place", new Dictionary { ["Inventory"] = inventory })); + Assert.That(missing.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(missing.Message, Does.Contain("inventory.lookup-stock").And.Contain("dispatch.publish-booking")); + Assert.That(calls, Is.EqualTo(0), "a handler ran during binding"); + Operations typo = new Operations().On("lookup_stok", Codecs.Text, Codecs.Unit, sku => Unit.Value); + Assert.That( + Support.Throws(() => fulfillment.Function("fulfillment.place", new Dictionary { ["Inventory"] = typo })).Message, + Does.Contain("inventory has no operation lookup-stok")); + + // A service binding more names than the program has operations is refused from its count. + var wide = new Operations(); + for (int i = 0; i < 100; i++) + { + wide.On("not-an-operation-" + i.ToString(CultureInfo.InvariantCulture), Codecs.Text, Codecs.Unit, sku => Unit.Value); + } + + Assert.That( + Support.Throws(() => fulfillment.Function("fulfillment.place", new Dictionary { ["Inventory"] = wide })).Message, + Does.Contain("the service for inventory binds more operations than the program's ")); + fulfillment.Provide("Inventory", inventory); + Function valid = fulfillment.Function("order.valid-lines"); + Assert.That(valid.Call(Codecs.Bool, Codecs.List(Codecs.Dynamic).Arg(Array.Empty()), Codecs.List(Codecs.Text).Arg(Array.Empty())), Is.True); + Assert.That(Support.Throws(() => fulfillment.Provide("Unknown", new Operations())).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(Support.Throws(() => fulfillment.Function("no-such-function")).Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(fulfillment.Functions, Does.Contain("fulfillment.place").And.Not.Contain("app.any")); + } + + private static Operations Warehouse(Action reserved) => new Operations() + .On("booking-status", Codecs.Dynamic, Codecs.Dynamic, order => Dynamic.VariantOf("BookingPending")) + .On("lookup_stock", Codecs.Text, Codecs.Dynamic, sku => Dynamic.VariantOf("Stock", Dynamic.Of(20L))) + .On("reserve-order", Codecs.Dynamic, Codecs.Dynamic, (context, order) => + { + Assert.That(context.IsCancelled, Is.False); + reserved(); + return Dynamic.VariantOf("Booked", Dynamic.Of("receipt-1")); + }) + .On("publish-booking", Codecs.Text, Codecs.Text, Codecs.Unit, (string id, string receipt) => throw new IOException("synthetic after commit")); + + private static Dynamic Order(ProgramData data) => + data.Make("Order", Dynamic.Of("one"), Dynamic.ListOf(data.Make("Line", Dynamic.Of("sensor"), Dynamic.Of(2L)))); + + private static Dictionary Both(Operations service) => new() { ["Inventory"] = service, ["Dispatch"] = service }; + + [Test] + public void UnknownCompletionHappensOnceWithRawEvidence() + { + PreparedProgram fulfillment = Prepare(Example("fulfillment")); + int reservations = 0; + Function place = fulfillment.Function("fulfillment.place", Both(Warehouse(() => Interlocked.Increment(ref reservations)))); + Dynamic order = Order(fulfillment.Data); + JacquardException error = Support.Throws(() => place.Call(Codecs.Dynamic, order)); + Assert.That(error.Failure, Is.EqualTo(Failure.OperationFailed), error.Message); + InvocationResult result = error.Result!; + Assert.That(result.Code, Is.EqualTo("E1612")); + Assert.That(result.HostFrames.Any(frame => Json.Of("effect_ok").Equals(frame.Get("kind"))), Is.True); + Assert.That(reservations, Is.EqualTo(1)); + string rendered = place.Invoke(order).Render(); + Assert.That(rendered, Does.StartWith("fulfillment.place: E1612").And.Contain(". publish-booking")); + } + + /// + /// In a real invocation, a cancellation that lands after a request's + /// arguments are decoded (through the test seam) keeps the handler from + /// starting, and the invocation reports the abort. + /// + [Test] + public void NoHandlerStartsOnceTheInvocationStoppedDuringDecoding() + { + PreparedProgram fulfillment = Prepare(Example("fulfillment")); + int started = 0; + Operations service = new Operations() + .On("booking-status", Codecs.Dynamic, Codecs.Dynamic, order => + { + started++; + return Dynamic.VariantOf("BookingPending"); + }) + .On("lookup-stock", Codecs.Text, Codecs.Dynamic, sku => Dynamic.VariantOf("Stock", Dynamic.Of(20L))) + .On("reserve-order", Codecs.Dynamic, Codecs.Dynamic, order => Dynamic.VariantOf("Booked", Dynamic.Of("r"))) + .On("publish-booking", Codecs.Text, Codecs.Text, Codecs.Unit, (id, receipt) => Unit.Value); + Function place = fulfillment.Function("fulfillment.place", Both(service)); + using var cancel = new CancellationTokenSource(); + Operations.AfterDecodeHook = cancel.Cancel; + Invocation invocation = place.Invoke(new Argument[] { Order(fulfillment.Data) }, cancel.Token); + Operations.AfterDecodeHook = null; + Assert.That(started, Is.EqualTo(0), "the handler started after the invocation stopped"); + Assert.That(Support.Reason(invocation.Raw), Is.EqualTo("aborted"), invocation.ToString()); + Assert.That(invocation.Raw.Requests, Has.Count.EqualTo(1)); + } + + /// A handler's context token is cancelled when the caller cancels while it runs; the handler sees it. + [Test] + public void AHandlersTokenIsCancelledWhenTheCallerCancels() + { + PreparedProgram fulfillment = Prepare(Example("fulfillment")); + using var cancel = new CancellationTokenSource(); + using var entered = new ManualResetEventSlim(); + bool sawCancellation = false; + Operations service = new Operations() + .On("booking-status", Codecs.Dynamic, Codecs.Dynamic, (context, order) => + { + entered.Set(); + sawCancellation = context.CancellationToken.WaitHandle.WaitOne(TimeSpan.FromMinutes(1)); + return Dynamic.VariantOf("BookingPending"); + }) + .On("lookup-stock", Codecs.Text, Codecs.Dynamic, sku => Dynamic.VariantOf("Stock", Dynamic.Of(20L))) + .On("reserve-order", Codecs.Dynamic, Codecs.Dynamic, order => Dynamic.VariantOf("Booked", Dynamic.Of("r"))) + .On("publish-booking", Codecs.Text, Codecs.Text, Codecs.Unit, (id, receipt) => Unit.Value); + Function place = fulfillment.Function("fulfillment.place", Both(service)); + Thread canceller = Support.After(TimeSpan.Zero, () => + { + entered.Wait(TimeSpan.FromMinutes(1)); + cancel.Cancel(); + }); + Invocation invocation = place.Invoke(new Argument[] { Order(fulfillment.Data) }, cancel.Token); + canceller.Join(); + Assert.That(Support.Reason(invocation.Raw), Is.EqualTo("aborted")); + Assert.That(Support.Eventually(() => sawCancellation, TimeSpan.FromSeconds(10)), Is.True, "the handler's token was not cancelled"); + } + + internal sealed record Job(string Id, long Effort, long Value, long Latest); + + internal sealed record Assignment(string Id, long Slot); + + internal sealed record Plan(long Score, IReadOnlyList Assignments); + + internal abstract record Planning; + + internal sealed record Planned(Plan Plan) : Planning; + + internal sealed record InvalidPlan(string Reason) : Planning; + + private static readonly RecordCodec JobCodec = Codecs.Record( + "Job", + Codecs.Field("id", Codecs.Text, (Job job) => job.Id), + Codecs.Field("effort", Codecs.Int, (Job job) => job.Effort), + Codecs.Field("value", Codecs.Int, (Job job) => job.Value), + Codecs.Field("latest", Codecs.Int, (Job job) => job.Latest), + (id, effort, value, latest) => new Job(id, effort, value, latest)); + + private static readonly UnionCodec PlanningCodec = Codecs.Union( + Codecs.Record( + "Planned", + Codecs.Field( + "plan", + Codecs.Record( + "Plan", + Codecs.Field("score", Codecs.Int, (Plan plan) => plan.Score), + Codecs.Field( + "assignments", + Codecs.List(Codecs.Record("Assignment", Codecs.Field("id", Codecs.Text, (Assignment a) => a.Id), Codecs.Field("slot", Codecs.Int, (Assignment a) => a.Slot), (id, slot) => new Assignment(id, slot))), + (Plan plan) => plan.Assignments), + (score, assignments) => new Plan(score, assignments)), + (Planned planned) => planned.Plan), + plan => new Planned(plan)), + Codecs.Record("InvalidPlan", Codecs.Field("reason", Codecs.Text, (InvalidPlan invalid) => invalid.Reason), reason => new InvalidPlan(reason))); + + private static long Optimum(IReadOnlyList jobs, long[] caps) + { + if (jobs.Count == 0) + { + return 0; + } + + Job job = jobs[0]; + List rest = jobs.Skip(1).ToList(); + long best = Optimum(rest, caps); + for (int slot = 0; slot < Math.Min(job.Latest, caps.Length); slot++) + { + if (caps[slot] >= job.Effort) + { + long[] next = (long[])caps.Clone(); + next[slot] -= job.Effort; + best = Math.Max(best, job.Value + Optimum(rest, next)); + } + } + + return best; + } + + [Test] + public void PlannerMatchesAnIndependentExhaustiveOracle() + { + Function solve = Prepare(Example("planner")).Function("planner.plan"); + for (long seed = 0; seed < 25; seed++) + { + var jobs = new List(); + for (long i = 0; i < (seed % 6) + 1; i++) + { + jobs.Add(new Job("job-" + i.ToString(CultureInfo.InvariantCulture), ((seed + (i * 3)) % 4) + 1, ((seed * 7) + (i * 11)) % 15, ((seed + i) % 3) + 1)); + } + + long[] all = { (seed % 5) + 1, (seed % 3) + 2, 3 }; + long[] caps = all.Take((int)(seed % 3) + 1).ToArray(); + if (solve.Call(PlanningCodec, Codecs.List(JobCodec).Arg(jobs), Codecs.List(Codecs.Int).Arg(caps)) is not Planned(Plan plan)) + { + Assert.Fail("seed " + seed.ToString(CultureInfo.InvariantCulture) + ": not planned"); + return; + } + + Assert.That(plan.Score, Is.EqualTo(Optimum(jobs, caps)), "seed " + seed.ToString(CultureInfo.InvariantCulture)); + long[] loads = new long[caps.Length]; + var seen = new HashSet(); + long score = 0; + foreach (Assignment assignment in plan.Assignments) + { + Job job = jobs.Single(candidate => candidate.Id == assignment.Id); + Assert.That(seen.Add(assignment.Id) && assignment.Slot >= 1 && assignment.Slot <= Math.Min(job.Latest, caps.Length), Is.True, assignment.ToString()); + loads[assignment.Slot - 1] += job.Effort; + score += job.Value; + } + + for (int slot = 0; slot < caps.Length; slot++) + { + Assert.That(loads[slot], Is.LessThanOrEqualTo(caps[slot]), "seed " + seed.ToString(CultureInfo.InvariantCulture) + ": overloaded"); + } + + Assert.That(score, Is.EqualTo(plan.Score)); + } + } + + [Test] + public void CacheReuseInvalidationAndCorruption() + { + PreparedProgram quick = Prepare(Example("quickstart")); + int commands = runtime.CoreCommands; + PreparedProgram repeated = Prepare(Example("quickstart")); + Assert.That(runtime.CoreCommands, Is.EqualTo(commands), "a warm preparation ran Core"); + Assert.That(repeated.Directory, Is.EqualTo(quick.Directory)); + string original = File.ReadAllText(Example("quickstart")); + string changed = Source("changed.jac", original.Replace("\n", "\r\n", StringComparison.Ordinal) + "\r\n-- changed\r\n"); + PreparedProgram program = Prepare(changed); + string snapshot = File.ReadAllText(Path.Combine(program.Directory, "source.jac")); + Assert.That(program.Directory != quick.Directory && snapshot.Contains("\r\n", StringComparison.Ordinal), Is.True, "not a byte-exact new entry"); + File.WriteAllText(Path.Combine(program.Directory, "store", "unexpected"), "corrupt"); + Assert.That(Support.Throws(() => Prepare(changed), "a corrupt cache was accepted").Kind, Is.EqualTo(JacquardErrorKind.Preparation)); + Assert.That(Support.Throws(() => Prepare(Source("rules.txt", string.Empty))).Kind, Is.EqualTo(JacquardErrorKind.Preparation)); + } + + /// + /// Core accepts a parameter nesting 2,500 Options. That function is + /// unavailable, refused with a binding failure that names the depth + /// limit, while safe works on both the cold and warm paths. + /// + [Test] + public void ASignatureTooDeepToBindLeavesTheOtherFunctionsWorking() + { + const int depth = 2500; + string source = Source("deep.jac", "deep : (" + string.Concat(Enumerable.Repeat("Option (", depth)) + "Int" + new string(')', depth) + ") ->{} Int\ndeep(x) = 0\nsafe : () ->{} Int\nsafe() = 1\n"); + int afterCold = -1; + foreach (string path in new[] { "cold", "warm" }) + { + PreparedProgram program = Prepare(source); + if (afterCold < 0) + { + afterCold = runtime.CoreCommands; + } + else + { + Assert.That(runtime.CoreCommands, Is.EqualTo(afterCold), "the warm path ran Core"); + } + + Assert.That(program.Function("safe").Call(Codecs.Int), Is.EqualTo(1L), path); + JacquardException refused = Support.Throws(() => program.Function("deep"), path); + Assert.That(refused.Kind, Is.EqualTo(JacquardErrorKind.Binding), path); + Assert.That(refused.Message, Does.Contain("deeper than 64 levels, the protocol's value-depth limit")); + } + } + + /// Pointing first at second's identity in program.json would make it return 2. + [Test] + public void ARedirectedNameInProgramJsonIsRefused() + { + string source = Source("pair.jac", "first : () ->{} Int\nfirst() = 1\n\nsecond : () ->{} Int\nsecond() = 2\n"); + PreparedProgram program = Prepare(source); + Assert.That(program.Function("first").Call(Codecs.Int), Is.EqualTo(1L)); + string metadata = Path.Combine(program.Directory, "program.json"); + string original = File.ReadAllText(metadata); + string from = "[\"first\",\"" + program.Data.Identity("first").Hex + "\"]"; + Assert.That(original, Does.Contain(from)); + File.WriteAllText(metadata, original.Replace(from, "[\"first\",\"" + program.Data.Identity("second").Hex + "\"]", StringComparison.Ordinal)); + JacquardException error = Support.Throws(() => Prepare(source).Function("first").Call(Codecs.Int)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Preparation), error.Message); + Assert.That(error.Message, Does.Contain("program.json bindings differ")); + File.WriteAllText(metadata, original); + Assert.That(Prepare(source).Function("first").Call(Codecs.Int), Is.EqualTo(1L)); + } + + [Test] + public void FullIntRangeRealsAndNestedOptions() + { + PreparedProgram program = Prepare(Source( + "values.jac", + "value.int : (Int) ->{} Int\nvalue.int(x) = x\nvalue.real : (Real) ->{} Real\nvalue.real(x) = x\n" + + "value.option : (Option (Option Int)) ->{} Option (Option Int)\nvalue.option(x) = x\n")); + Function integer = program.Function("value.int"); + foreach (long value in new[] { Json.IntMin, Json.IntMax }) + { + Assert.That(integer.Call(Codecs.Int, value), Is.EqualTo(value)); + } + + Assert.That(Support.Throws(() => integer.Call(Codecs.Int, Json.IntMax + 1)).Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(Support.Throws(() => integer.Call(Codecs.Int, 1.5)).Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + JacquardException narrow = Support.Throws(() => integer.Call(Codecs.Int32, 1L << 40)); + Assert.That(narrow.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(narrow.Result, Is.Not.Null); + Function real = program.Function("value.real"); + foreach (double value in new[] { 1.0, -0.0, double.PositiveInfinity, double.NaN }) + { + Assert.That(BitConverter.DoubleToInt64Bits(real.Call(Codecs.Real, value)), Is.EqualTo(BitConverter.DoubleToInt64Bits(value))); + } + + Function option = program.Function("value.option"); + Codec>> nested = Codecs.Option(Codecs.Option(Codecs.Int)); + foreach (Option> value in new[] { Option>.None, Option.Some(Option.None), Option.Some(Option.Some(3L)) }) + { + Assert.That(option.Call(nested, nested.Arg(value)), Is.EqualTo(value)); + } + + Dynamic inner = Dynamic.Some(Dynamic.None()); + Assert.That(option.Call(Codecs.Dynamic, inner), Is.EqualTo(inner)); + } + + [Test] + public void SpawnFailuresAndCancellation() + { + PreparedProgram quick = Prepare(Example("quickstart")); + Function shipping = quick.Function("shipping-cost"); + quick.SetWorkerCommand(new[] { "/no/such/jacquard" }); + JacquardException spawn = Support.Throws(() => quick.Function("shipping-cost").Call(Codecs.Int, 10)); + Assert.That(spawn.Failure, Is.EqualTo(Failure.WorkerLost), spawn.Message); + Assert.That(Support.Reason(spawn.Result!), Is.EqualTo("worker_spawn_failed")); + using var cancel = new CancellationTokenSource(); + cancel.Cancel(); + Invocation invocation = shipping.Invoke(new Argument[] { 10 }, cancel.Token); + Assert.That(Support.Reason(invocation.Raw), Is.EqualTo("aborted")); + Assert.That(invocation.Error, Is.Not.Null); + Assert.That(Support.Throws(() => shipping.Call(Codecs.Int, new Argument[] { 10 }, cancel.Token)).Failure, Is.EqualTo(Failure.WorkerLost)); + } + + [Test] + public void OutputOverflowFailsOnceWithoutAdaptiveBatching() + { + PreparedProgram program = Prepare(Example("triage")); + Function route = program.Function("triage.batch"); + List rows = Enumerable.Range(0, 29).Select(i => new Incidents.Incident(i.ToString(CultureInfo.InvariantCulture), "api", 5, 90, true, null)).ToList(); + JacquardException error = Support.Throws(() => route.MapBatches(Incidents.TriageCodec, Incidents.IncidentCodec, rows, 29, new Argument[] { Policy(program) })); + Assert.That(error.BatchIndex, Is.EqualTo(0), error.Message); + Assert.That(error.Completed, Is.Empty); + Assert.That(error.Result!.Code, Is.EqualTo("E1602")); + } + + /// Batch 0 decodes to one unit; batch 1's result exceeds the limits (E1602), and the failure keeps batch 0's result. + [Test] + public void ALaterBatchFailureKeepsEarlierUnitResults() + { + string many = string.Join(",", Enumerable.Repeat("()", 40)); + Function units = Prepare(Source( + "units.jac", + "units : (List Int) ->{} List ()\nunits(xs) = match xs { | Cons(n, _) -> if eq(n, 1) then [()] else [" + many + "] | Nil -> [] }\n")).Function("units"); + Assert.That(units.MapBatches(Codecs.Unit, Codecs.Int, new[] { 1L }, 1), Is.EqualTo(new[] { Unit.Value })); + JacquardException error = Support.Throws(() => units.MapBatches(Codecs.Unit, Codecs.Int, new[] { 1L, 2L }, 1)); + Assert.That(error.BatchIndex, Is.EqualTo(1), error.Message); + Assert.That(error.Completed, Is.EqualTo(new object[] { Unit.Value })); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Invocation)); + Assert.That(error.Result!.Code, Is.EqualTo("E1602")); + } + + /// A dictionary that reports a huge count and fails when it is read. + private sealed class HugeBindings : IReadOnlyDictionary + { + public int Count => 1_000_000; + + public IEnumerable Keys => throw new InvalidOperationException("a binding was read"); + + public IEnumerable Values => throw new InvalidOperationException("a binding was read"); + + public Operations this[string key] => throw new InvalidOperationException("a binding was read"); + + public bool ContainsKey(string key) => throw new InvalidOperationException("a binding was read"); + + public bool TryGetValue(string key, out Operations value) => throw new InvalidOperationException("a binding was read"); + + public IEnumerator> GetEnumerator() => throw new InvalidOperationException("a binding was read"); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// More bindings than the function has effects are refused from the count, before any binding is read. + [Test] + public void TooManyBindingsAreRefusedBeforeAnyIsRead() + { + PreparedProgram program = Prepare(Example("quickstart")); + JacquardException error = Support.Throws(() => program.Function("shipping-cost", new HugeBindings())); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(error.Message, Is.EqualTo("shipping-cost: 1000000 bindings given, but it uses 0 effects")); + } + + /// A result codec that throws with a huge message gives a short binding failure that keeps the raw result and the exception. + [Test] + public void AThrowingResultCodecKeepsTheResultAndQuotesAnExcerpt() + { + Function shipping = Prepare(Example("quickstart")).Function("shipping-cost"); + Invocation invocation = shipping.Invoke(35); + Assert.That(invocation.Ok, Is.True); + Codec throwing = Codecs.Create((encoder, value) => Codecs.Int.Encode(encoder, value), (decoder, value) => throw new InvalidOperationException(new string('X', 1_000_000))); + JacquardException error = Support.Throws(() => invocation.Decode(throwing)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(error.Message, Has.Length.LessThan(400)); + Assert.That(error.Message, Does.EndWith("... (1000000 chars)")); + Assert.That(error.Result!.Ok, Is.True, "the raw result was not kept"); + Assert.That(error.InnerException, Is.InstanceOf(), "the codec's exception is not kept"); + Assert.That(Support.Throws(() => shipping.Call(throwing, 35)).InnerException, Is.InstanceOf()); + + // A mapping failure with a null message is still a binding failure keeping the result. + Codec nullMessage = Codecs.Create((encoder, value) => Codecs.Int.Encode(encoder, value), (decoder, value) => throw decoder.Mapping(null)); + JacquardException nulled = Support.Throws(() => invocation.Decode(nullMessage)); + Assert.That(nulled.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(nulled.Result!.Ok, Is.True); + + // A save path longer than any path is refused before the file system sees it. + IOException save = Support.Throws(() => invocation.Save("/" + new string('v', 1_000_000))); + Assert.That(save.Message, Has.Length.LessThan(200)); + } + + [Test] + public void ResultMappingFailureRetainsTheSuccessfulResult() + { + Invocation invocation = Prepare(Example("quickstart")).Function("shipping-cost").Invoke(35); + Assert.That(invocation.Ok, Is.True); + Assert.That(invocation.Error, Is.Null); + JacquardException error = Support.Throws(() => invocation.Decode(Codecs.Text)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(error.Result!.Ok, Is.True, error.Message); + } + + private static void CopyTree(string from, string to) + { + Directory.CreateDirectory(to); + foreach (string entry in Directory.EnumerateFileSystemEntries(from)) + { + string target = Path.Combine(to, Path.GetFileName(entry)); + if (Directory.Exists(entry)) + { + CopyTree(entry, target); + } + else + { + File.Copy(entry, target); + } + } + } + + [Test] + public void PreludeContentsKeyTheCacheAndConcurrentPreparationPublishesOnce() + { + PreparedProgram quick = Prepare(Example("quickstart")); + string copy = Path.Combine(root, "prelude"); + CopyTree(Environment.GetEnvironmentVariable("JACQUARD_PRELUDE")!, copy); + var copied = new Runtime(new RuntimeOptions { Prelude = copy }); + PreparedProgram before = copied.Prepare(new[] { Example("quickstart") }, Options); + Assert.That(before.Directory, Is.EqualTo(quick.Directory), "an identical prelude copy should share the entry"); + string license = Path.Combine(copy, "LICENSE.md"); + File.AppendAllText(license, "\n"); + PreparedProgram after = copied.Prepare(new[] { Example("quickstart") }, Options); + Assert.That(after.Directory, Is.Not.EqualTo(before.Directory), "a prelude change did not change the cache key"); + string source = Source("concurrent.jac", "shipping-cost : (Int) ->{} Int\nshipping-cost(total) = if lt(total, 50) then 5 else 0\n-- concurrent\n"); + Task[] workers = Enumerable.Range(0, 2) + .Select(_ => Task.Factory.StartNew(() => runtime.Prepare(new[] { source }, Options), CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)) + .ToArray(); + Assert.That(Task.WaitAll(workers, TimeSpan.FromMinutes(2)), Is.True); + Assert.That(workers[1].Result.Directory, Is.EqualTo(workers[0].Result.Directory)); + Assert.That(workers[1].Result.Function("shipping-cost").Call(Codecs.Int, 35), Is.EqualTo(5L)); + foreach (string entry in Directory.EnumerateFileSystemEntries(Path.Combine(root, "cache"))) + { + Assert.That(Path.GetFileName(entry), Does.Not.StartWith("prepare-"), "staging directory left behind: " + entry); + } + } +} From 4a095743cba00a7bde9c5d4df98e846310480b4a Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 07:04:33 -0400 Subject: [PATCH 05/52] Run the four example projects through the .NET SDK JacquardExamples runs quickstart, planner, triage, and fulfillment from the rules in examples/, with the same --inspect, --fault, and --deliver-pending recovery paths as the Rust and Java commands. Fulfillment's warehouse commits bookings and outbox entries as one atomic JSON snapshot, written to a temporary file it creates exclusively and removes only if it made it and did not rename it. --- dotnet/JacquardHost.sln | 17 + dotnet/examples/JacquardExamples/Examples.cs | 912 ++++++++++++++++++ .../JacquardExamples/JacquardExamples.csproj | 17 + .../JacquardHost.Tests.csproj | 1 + .../JacquardHost.Tests/Sdk/ExamplesTests.cs | 206 ++++ dotnet/test/JacquardHost.Tests/Support.cs | 2 +- 6 files changed, 1154 insertions(+), 1 deletion(-) create mode 100644 dotnet/examples/JacquardExamples/Examples.cs create mode 100644 dotnet/examples/JacquardExamples/JacquardExamples.csproj create mode 100644 dotnet/test/JacquardHost.Tests/Sdk/ExamplesTests.cs diff --git a/dotnet/JacquardHost.sln b/dotnet/JacquardHost.sln index bc797ee..67350d1 100644 --- a/dotnet/JacquardHost.sln +++ b/dotnet/JacquardHost.sln @@ -13,6 +13,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JacquardHost.StandIn", "tes EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JacquardHost.Tests", "test\JacquardHost.Tests\JacquardHost.Tests.csproj", "{3ED29B8E-500E-4D0D-B6AD-F987037225AD}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "examples", "examples", "{B36A84DF-456D-A817-6EDD-3EC3E7F6E11F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JacquardExamples", "examples\JacquardExamples\JacquardExamples.csproj", "{79FD70B9-B8BC-4217-9720-9553BA48F2BD}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -59,6 +63,18 @@ Global {3ED29B8E-500E-4D0D-B6AD-F987037225AD}.Release|x64.Build.0 = Release|Any CPU {3ED29B8E-500E-4D0D-B6AD-F987037225AD}.Release|x86.ActiveCfg = Release|Any CPU {3ED29B8E-500E-4D0D-B6AD-F987037225AD}.Release|x86.Build.0 = Release|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Debug|x64.ActiveCfg = Debug|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Debug|x64.Build.0 = Debug|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Debug|x86.ActiveCfg = Debug|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Debug|x86.Build.0 = Debug|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Release|Any CPU.Build.0 = Release|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Release|x64.ActiveCfg = Release|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Release|x64.Build.0 = Release|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Release|x86.ActiveCfg = Release|Any CPU + {79FD70B9-B8BC-4217-9720-9553BA48F2BD}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -67,5 +83,6 @@ Global {3C269BF6-7938-464F-B385-6997B38B63B9} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {CB695A56-2929-47FB-B93C-3E74883663E2} = {0C88DD14-F956-CE84-757C-A364CCF449FC} {3ED29B8E-500E-4D0D-B6AD-F987037225AD} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + {79FD70B9-B8BC-4217-9720-9553BA48F2BD} = {B36A84DF-456D-A817-6EDD-3EC3E7F6E11F} EndGlobalSection EndGlobal diff --git a/dotnet/examples/JacquardExamples/Examples.cs b/dotnet/examples/JacquardExamples/Examples.cs new file mode 100644 index 0000000..a893af6 --- /dev/null +++ b/dotnet/examples/JacquardExamples/Examples.cs @@ -0,0 +1,912 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using JacquardLang.Host; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; + +namespace JacquardLang.Host.Examples; + +/// +/// Runs the repository's example Jacquard projects through the .NET SDK: +/// quickstart, planner, triage, and fulfillment. The same rules run through +/// the Python, TypeScript, Go, Rust, and Java SDKs. +/// +/// +/// +/// dotnet examples/JacquardExamples/bin/Debug/net8.0/JacquardExamples.dll \ +/// [--workdir DIR] [--inspect PATH] [--fault NAME] [--deliver-pending] PROJECT [INPUT.json] +/// +/// Core is found through JACQUARD_BIN and JACQUARD_PRELUDE. The +/// rules are read from the repository's examples/ directory +/// (JACQUARD_EXAMPLES, else examples/ or ../examples/ +/// under the working directory). The exit status is 1 for a Jacquard or SDK +/// failure and 2 for usage or other errors. +/// +public static class Examples +{ + /// The example projects in presentation order. + public static IReadOnlyList Projects { get; } = new[] { "quickstart", "planner", "triage", "fulfillment" }; + + /// The faults the fulfillment store can inject. + private static readonly string[] Faults = { string.Empty, "publish-failed", "after-commit", "after-delivery" }; + + /// Runs one command line and exits with its status. + public static int Main(string[] args) + { + try + { + Run(args, Console.Out); + return 0; + } + catch (JacquardException failure) + { + Console.Out.Flush(); + Console.Error.WriteLine(failure.Message); + return 1; + } + catch (Exception failure) when (failure is UsageException or IOException or UnauthorizedAccessException) + { + Console.Out.Flush(); + Console.Error.WriteLine(failure.Message); + return 2; + } + } + + /// The repository's example projects. + internal static string ExamplesDirectory(string? configured) + { + configured ??= Environment.GetEnvironmentVariable("JACQUARD_EXAMPLES"); + if (!string.IsNullOrEmpty(configured)) + { + return Path.GetFullPath(configured); + } + + foreach (string candidate in new[] { "examples", Path.Combine("..", "examples") }) + { + if (File.Exists(Path.Combine(candidate, "quickstart", "rules.jac"))) + { + return Path.GetFullPath(candidate); + } + } + + throw new UsageException("cannot find the examples directory; run from the repository or dotnet/, or set JACQUARD_EXAMPLES"); + } + + // JSON input and output. + private static Json Field(JsonObject json, string key) => json.Get(key) ?? throw new UsageException("input: missing " + key); + + private static string TextOf(JsonObject json, string key) => Field(json, key).AsString() ?? throw new UsageException("input: " + key + " must be text"); + + private static long IntegerOf(JsonObject json, string key) => Field(json, key).AsInt() ?? throw new UsageException("input: " + key + " must be an integer"); + + private static bool BoolOf(JsonObject json, string key) => + Field(json, key) is JsonBool value ? value.Value : throw new UsageException("input: " + key + " must be true or false"); + + private static List Objects(JsonObject json, string key) + { + IReadOnlyList items = Field(json, key).AsArray() ?? throw new UsageException("input: " + key + " must be a list"); + return items.Select(item => item.AsObject() ?? throw new UsageException("input: " + key + " must hold objects")).ToList(); + } + + private static JsonObject Tagged(string tag, params (string Key, Json Value)[] pairs) => + Json.Object(pairs.Append(("$tag", Json.Of(tag))).ToArray()); + + internal static string Compact(Json value) => JsonCodec.Stringify(value, 16); + + private static void Pretty(Json value, int indent, StringBuilder output) + { + string pad = new(' ', indent * 2); + if (value is JsonArray array && array.Items.Count > 0) + { + output.Append("[\n"); + for (int i = 0; i < array.Items.Count; i++) + { + output.Append(pad).Append(" "); + Pretty(array.Items[i], indent + 1, output); + output.Append(i + 1 < array.Items.Count ? ",\n" : "\n"); + } + + output.Append(pad).Append(']'); + } + else if (value is JsonObject json && json.Count > 0) + { + output.Append("{\n"); + int i = 0; + foreach (KeyValuePair field in json) + { + output.Append(pad).Append(" ").Append(Compact(Json.Of(field.Key))).Append(": "); + Pretty(field.Value, indent + 1, output); + output.Append(++i < json.Count ? ",\n" : "\n"); + } + + output.Append(pad).Append('}'); + } + else + { + output.Append(Compact(value)); + } + } + + private static void PrintJson(TextWriter output, Json value) + { + var text = new StringBuilder(); + Pretty(value, 0, text); + output.Write(text.Append('\n').ToString()); + } + + // Planner. + + /// A job to place. + public sealed record Job(string Id, long Effort, long Value, long Latest); + + /// A job placed in a slot. + public sealed record Assignment(string Id, long Slot); + + /// A plan and its score. + public sealed record Plan(long Score, IReadOnlyList Assignments); + + /// The planner's result. + public abstract record Planning; + + /// A valid plan. + public sealed record Planned(Plan Plan) : Planning; + + /// Invalid input. + public sealed record InvalidPlan(string Reason) : Planning; + + internal static readonly RecordCodec JobCodec = Codecs.Record( + "Job", + Codecs.Field("id", Codecs.Text, (Job job) => job.Id), + Codecs.Field("effort", Codecs.Int, (Job job) => job.Effort), + Codecs.Field("value", Codecs.Int, (Job job) => job.Value), + Codecs.Field("latest", Codecs.Int, (Job job) => job.Latest), + (id, effort, value, latest) => new Job(id, effort, value, latest)); + + private static readonly RecordCodec AssignmentCodec = Codecs.Record( + "Assignment", + Codecs.Field("id", Codecs.Text, (Assignment assignment) => assignment.Id), + Codecs.Field("slot", Codecs.Int, (Assignment assignment) => assignment.Slot), + (id, slot) => new Assignment(id, slot)); + + private static readonly RecordCodec PlanCodec = Codecs.Record( + "Plan", + Codecs.Field("score", Codecs.Int, (Plan plan) => plan.Score), + Codecs.Field("assignments", Codecs.List(AssignmentCodec), (Plan plan) => plan.Assignments), + (score, assignments) => new Plan(score, assignments)); + + internal static readonly UnionCodec PlanningCodec = Codecs.Union( + Codecs.Record("Planned", Codecs.Field("plan", PlanCodec, (Planned planned) => planned.Plan), plan => new Planned(plan)), + Codecs.Record("InvalidPlan", Codecs.Field("reason", Codecs.Text, (InvalidPlan invalid) => invalid.Reason), reason => new InvalidPlan(reason))); + + private static JsonObject PlanningJson(Planning planning) => planning switch + { + Planned(Plan plan) => Tagged("Planned", ("plan", Json.Object( + ("assignments", Json.Of(plan.Assignments.Select(assignment => (Json)Json.Object(("id", Json.Of(assignment.Id)), ("slot", Json.Of(assignment.Slot)))))), + ("score", Json.Of(plan.Score))))), + InvalidPlan(string reason) => Tagged("InvalidPlan", ("reason", Json.Of(reason))), + _ => throw new InvalidOperationException("unknown planning result"), + }; + + // Triage. + + /// When an incident pages, and when it is stale. + public sealed record TriagePolicy(long Page, long Stale); + + /// One incident; a missing owner is null. + public sealed record Incident(string Id, string Service, long Severity, long Age, bool Visible, string? Owner); + + /// Members map by convention: PageNow is the constructor PageNow. + public enum Urgency + { + /// Page someone now. + PageNow, + + /// Work on it today. + WorkToday, + + /// Only watch it. + WatchOnly, + } + + /// The triage result. + public abstract record Triage; + + /// A routed incident. + public sealed record Routed(string Id, string Team, Urgency Urgency, IReadOnlyList Reasons) : Triage; + + /// An invalid incident. + public sealed record InvalidIncident(string Id) : Triage; + + internal static readonly RecordCodec PolicyCodec = Codecs.Record( + "TriagePolicy", + Codecs.Field("page", Codecs.Int, (TriagePolicy policy) => policy.Page), + Codecs.Field("stale", Codecs.Int, (TriagePolicy policy) => policy.Stale), + (page, stale) => new TriagePolicy(page, stale)); + + internal static readonly RecordCodec IncidentCodec = Codecs.Record( + "Incident", + Codecs.Field("id", Codecs.Text, (Incident incident) => incident.Id), + Codecs.Field("service", Codecs.Text, (Incident incident) => incident.Service), + Codecs.Field("severity", Codecs.Int, (Incident incident) => incident.Severity), + Codecs.Field("age", Codecs.Int, (Incident incident) => incident.Age), + Codecs.Field("visible", Codecs.Bool, (Incident incident) => incident.Visible), + Codecs.Field("owner", Codecs.Nullable(Codecs.Text), (Incident incident) => incident.Owner), + (id, service, severity, age, visible, owner) => new Incident(id, service, severity, age, visible, owner)); + + internal static readonly UnionCodec TriageCodec = Codecs.Union( + Codecs.Record( + "Routed", + Codecs.Field("id", Codecs.Text, (Routed routed) => routed.Id), + Codecs.Field("team", Codecs.Text, (Routed routed) => routed.Team), + Codecs.Field("urgency", Codecs.Enum(), (Routed routed) => routed.Urgency), + Codecs.Field("reasons", Codecs.List(Codecs.Text), (Routed routed) => routed.Reasons), + (id, team, urgency, reasons) => new Routed(id, team, urgency, reasons)), + Codecs.Record("InvalidIncident", Codecs.Field("id", Codecs.Text, (InvalidIncident invalid) => invalid.Id), id => new InvalidIncident(id))); + + private static JsonObject TriageJson(Triage triage) => triage switch + { + Routed routed => Tagged( + "Routed", + ("id", Json.Of(routed.Id)), + ("team", Json.Of(routed.Team)), + ("urgency", Json.Of(routed.Urgency switch + { + Urgency.PageNow => "PAGE", + Urgency.WorkToday => "TODAY", + _ => "WATCH", + })), + ("reasons", Json.Of(routed.Reasons.Select(reason => (Json)Json.Of(reason))))), + InvalidIncident(string id) => Tagged("InvalidIncident", ("id", Json.Of(id))), + _ => throw new InvalidOperationException("unknown triage result"), + }; + + // Fulfillment. + + /// One order line. + public sealed record Line(string Sku, long Units); + + /// An order. + public sealed record Order(string Id, IReadOnlyList Lines); + + /// A stock lookup: Stock is also the type's name, so the .NET case is , mapped explicitly. + public abstract record Stock; + + /// Units available. + public sealed record Available(long Units) : Stock; + + /// An unknown SKU. + public sealed record UnknownSku : Stock; + + /// A reservation. + public abstract record Reservation; + + /// No booking yet. + public sealed record BookingPending : Reservation; + + /// Booked, with its receipt. + public sealed record Booked(string Receipt) : Reservation; + + /// The same order was booked before. + public sealed record RepeatBooking(string Receipt) : Reservation; + + /// Another order was booked under this id. + public sealed record BookingConflict : Reservation; + + /// Stock changed before the commit. + public sealed record StockChanged : Reservation; + + /// The fulfillment result. + public abstract record Fulfillment; + + /// Fulfilled, with its receipt. + public sealed record Fulfilled(string Receipt) : Fulfillment; + + /// A repeated order. + public sealed record DuplicateOrder(string Receipt) : Fulfillment; + + /// A rejected order. + public sealed record OrderRejected(string Reason) : Fulfillment; + + private static readonly RecordCodec LineCodec = Codecs.Record( + "Line", + Codecs.Field("sku", Codecs.Text, (Line line) => line.Sku), + Codecs.Field("units", Codecs.Int, (Line line) => line.Units), + (sku, units) => new Line(sku, units)); + + internal static readonly RecordCodec OrderCodec = Codecs.Record( + "Order", + Codecs.Field("id", Codecs.Text, (Order order) => order.Id), + Codecs.Field("lines", Codecs.List(LineCodec), (Order order) => order.Lines), + (id, lines) => new Order(id, lines)); + + private static readonly UnionCodec StockCodec = Codecs.Union( + Codecs.Record("Stock", Codecs.Field("units", Codecs.Int, (Available available) => available.Units), units => new Available(units)), + Codecs.Record("UnknownSku", () => new UnknownSku())); + + private static readonly UnionCodec ReservationCodec = Codecs.Union( + Codecs.Record("BookingPending", () => new BookingPending()), + Codecs.Record("Booked", Codecs.Field("receipt", Codecs.Text, (Booked booked) => booked.Receipt), receipt => new Booked(receipt)), + Codecs.Record("RepeatBooking", Codecs.Field("receipt", Codecs.Text, (RepeatBooking repeat) => repeat.Receipt), receipt => new RepeatBooking(receipt)), + Codecs.Record("BookingConflict", () => new BookingConflict()), + Codecs.Record("StockChanged", () => new StockChanged())); + + internal static readonly UnionCodec FulfillmentCodec = Codecs.Union( + Codecs.Record("Fulfilled", Codecs.Field("receipt", Codecs.Text, (Fulfilled done) => done.Receipt), receipt => new Fulfilled(receipt)), + Codecs.Record("DuplicateOrder", Codecs.Field("receipt", Codecs.Text, (DuplicateOrder repeat) => repeat.Receipt), receipt => new DuplicateOrder(receipt)), + Codecs.Record("OrderRejected", Codecs.Field("reason", Codecs.Text, (OrderRejected rejected) => rejected.Reason), reason => new OrderRejected(reason))); + + private static JsonObject FulfillmentJson(Fulfillment fulfillment) => fulfillment switch + { + Fulfilled(string receipt) => Tagged("Fulfilled", ("receipt", Json.Of(receipt))), + DuplicateOrder(string receipt) => Tagged("DuplicateOrder", ("receipt", Json.Of(receipt))), + OrderRejected(string reason) => Tagged("OrderRejected", ("reason", Json.Of(reason))), + _ => throw new InvalidOperationException("unknown fulfillment result"), + }; + + /// Binds fulfillment.place to a shared warehouse serving both effects. + public static Function BindFulfillment(PreparedProgram program, Warehouse warehouse) + { + ArgumentNullException.ThrowIfNull(program); + ArgumentNullException.ThrowIfNull(warehouse); + Operations service = new Operations() + .On("booking-status", OrderCodec, ReservationCodec, warehouse.BookingStatus) + .On("lookup-stock", Codecs.Text, StockCodec, warehouse.LookupStock) + .On("reserve-order", OrderCodec, ReservationCodec, warehouse.ReserveOrder) + .On("publish-booking", Codecs.Text, Codecs.Text, Codecs.Unit, warehouse.PublishBooking); + return program.Function("fulfillment.place", new Dictionary { ["Inventory"] = service, ["Dispatch"] = service }); + } + + private static void Save(Invocation invocation, string? path) + { + if (path is null) + { + return; + } + + try + { + invocation.Save(path); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw new UsageException(error.Message); + } + } + + private static byte[] Read(string path) + { + try + { + return File.ReadAllBytes(path); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw new UsageException(error.Message); + } + } + + /// Runs one command line, writing its output to . + /// A Jacquard or SDK failure (exit status 1). + /// A usage, input, or I/O failure (exit status 2). + [SuppressMessage("Maintainability", "CA1502", Justification = "One command line, one case per project, as in the other SDKs' commands.")] + public static void Run(IReadOnlyList arguments, TextWriter output, string? examples = null) + { + ArgumentNullException.ThrowIfNull(arguments); + ArgumentNullException.ThrowIfNull(output); + string workdir = Path.Combine(".scratch", "dotnet-examples"); + string? inspect = null; + string fault = string.Empty; + bool deliverPending = false; + var positional = new List(); + for (int i = 0; i < arguments.Count; i++) + { + string argument = arguments[i]; + switch (argument) + { + case "--workdir" or "--inspect" or "--fault": + if (i + 1 >= arguments.Count) + { + throw new UsageException(argument + " needs a value"); + } + + string value = arguments[++i]; + if (argument == "--workdir") + { + workdir = value; + } + else if (argument == "--inspect") + { + inspect = value; + } + else + { + fault = value; + } + + break; + case "--deliver-pending": + deliverPending = true; + break; + default: + if (argument.StartsWith("--", StringComparison.Ordinal)) + { + throw new UsageException("unknown flag " + argument); + } + + positional.Add(argument); + break; + } + } + + string project = positional.Count == 0 ? string.Empty : positional[0]; + if (!Projects.Contains(project) || positional.Count > 2) + { + throw new UsageException("choose quickstart, triage, planner, or fulfillment"); + } + + if (!Faults.Contains(fault)) + { + throw new UsageException("unknown fault"); + } + + string root = Path.GetFullPath(workdir); + string directory = ExamplesDirectory(examples); + string rules = Path.Combine(directory, project, "rules.jac"); + var runtime = new Runtime(); + PreparedProgram program = runtime.Prepare(new[] { rules }, new PrepareOptions { CacheDir = Path.Combine(root, "programs") }); + if (project == "quickstart") + { + Function shipping = program.Function("shipping-cost"); + foreach (long total in new long[] { 35, 75 }) + { + long cost = shipping.Call(Codecs.Int, total); + output.Write("Order total " + total.ToString(CultureInfo.InvariantCulture) + ": shipping " + cost.ToString(CultureInfo.InvariantCulture) + "\n"); + } + + return; + } + + string inputPath = positional.Count > 1 ? positional[1] : Path.Combine(directory, project, "input.json"); + JsonObject input; + try + { + input = JsonCodec.Decode(Read(inputPath), 16).AsObject() ?? throw new UsageException("input must be a JSON object"); + } + catch (ProtocolException error) + { + throw new UsageException("input: " + error.Message); + } + + switch (project) + { + case "planner": + { + List jobs = Objects(input, "jobs") + .Select(job => new Job(TextOf(job, "id"), IntegerOf(job, "effort"), IntegerOf(job, "value"), IntegerOf(job, "latest"))) + .ToList(); + IReadOnlyList capacityItems = Field(input, "capacities").AsArray() ?? throw new UsageException("input: capacities must be a list"); + List capacities = capacityItems.Select(capacity => capacity.AsInt() ?? throw new UsageException("input: capacities must be integers")).ToList(); + Invocation invocation = program.Function("planner.plan").Invoke(Codecs.List(JobCodec).Arg(jobs), Codecs.List(Codecs.Int).Arg(capacities)); + Save(invocation, inspect); + PrintJson(output, PlanningJson(invocation.Decode(PlanningCodec))); + output.Write(invocation.Render() + "\n"); + break; + } + + case "triage": + { + JsonObject policyObject = Field(input, "policy").AsObject() ?? throw new UsageException("input: policy must be an object"); + var policy = new TriagePolicy(IntegerOf(policyObject, "page"), IntegerOf(policyObject, "stale")); + var incidents = new List(); + foreach (JsonObject incident in Objects(input, "incidents")) + { + // A missing owner is null, as in the Go command. + Json owner = incident.Get("owner") ?? Json.Null; + string? ownerValue = owner is JsonNull ? null : owner.AsString() ?? throw new UsageException("input: owner must be text or null"); + incidents.Add(new Incident( + TextOf(incident, "id"), TextOf(incident, "service"), IntegerOf(incident, "severity"), IntegerOf(incident, "age"), BoolOf(incident, "visible"), ownerValue)); + } + + Function route = program.Function("triage.batch"); + IReadOnlyList results; + if (inspect is null) + { + results = route.MapBatches(TriageCodec, IncidentCodec, incidents, 8, new[] { PolicyCodec.Arg(policy) }); + } + else + { + var collected = new List(); + for (int start = 0, index = 0; start < incidents.Count; start += 8, index++) + { + List chunk = incidents.GetRange(start, Math.Min(8, incidents.Count - start)); + Invocation invocation = route.Invoke(PolicyCodec.Arg(policy), Codecs.List(IncidentCodec).Arg(chunk)); + Save(invocation, inspect + "." + index.ToString(CultureInfo.InvariantCulture) + ".json"); + collected.AddRange(invocation.Decode(Codecs.List(TriageCodec))); + } + + results = collected; + } + + PrintJson(output, Json.Of(results.Select(result => (Json)TriageJson(result)))); + break; + } + + default: + { + List lines = Objects(input, "lines").Select(line => new Line(TextOf(line, "sku"), IntegerOf(line, "units"))).ToList(); + var order = new Order(TextOf(input, "id"), lines); + Warehouse warehouse = Warehouse.Open(Path.Combine(root, "inventory.json"), fault); + try + { + if (deliverPending) + { + int delivered; + try + { + delivered = warehouse.DeliverPending(); + } + catch (Exception error) when (error is HandlerException or IOException or UnauthorizedAccessException) + { + // Like the Go command, a failed delivery prints no count. + throw new UsageException("delivery failed: " + Describe(error)); + } + + output.Write(Compact(Json.Object(("delivered", Json.Of(delivered)))) + "\n"); + return; + } + + Invocation invocation = BindFulfillment(program, warehouse).Invoke(OrderCodec.Arg(order)); + Save(invocation, inspect); + JacquardException? failure = null; + try + { + PrintJson(output, FulfillmentJson(invocation.Decode(FulfillmentCodec))); + } + catch (JacquardException error) + { + failure = error; + } + + output.Write(invocation.Render() + "\n"); + if (failure is not null) + { + throw failure; + } + } + finally + { + output.Write(Compact(warehouse.Summary()) + "\n"); + } + + break; + } + } + } + + private static string Describe(Exception error) => + error is HandlerException failure ? failure.Category.WireName() + ": " + failure.Message : "completion unknown"; + + /// The canonical payload of an order: its lines sorted by SKU. + internal static string Payload(Order order) => + Compact(Json.Of(order.Lines.OrderBy(line => line.Sku, StringComparer.Ordinal) + .Select(line => (Json)Json.Of(new Json[] { Json.Of(line.Sku), Json.Of(line.Units) })))); + + internal static string Sha256(string text) => Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(text))).ToLowerInvariant(); + + /// + /// A single-process synthetic Inventory and Dispatch service. Bookings and + /// outbox entries are committed together as one atomic JSON snapshot; + /// deliveries are deduplicated by receipt. It has no cross-process + /// locking or power-loss durability claim. Its methods are synchronized, + /// because each handler runs on its own thread. + /// + public sealed class Warehouse + { + internal const string InventorySchema = "jacquard-example-inventory-v1"; + private static long next; + private readonly object sync = new(); + private readonly string path; + private readonly string fault; + private Inventory state; + + private Warehouse(string path, string fault, Inventory state) + { + this.path = path; + this.fault = fault; + this.state = state; + } + + private sealed record Booking(string Id, string Payload, string Receipt); + + private sealed record Outbox(string Id, string Receipt, bool Delivered); + + private sealed record Inventory(SortedDictionary Stock, List Skus, List Bookings, List Outbox, List Deliveries) + { + internal JsonObject ToJson() => Json.Object( + ("schema", Json.Of(InventorySchema)), + ("stock", Json.Of(Skus.Select(sku => (Json)Json.Object(("sku", Json.Of(sku)), ("available", Json.Of(Stock[sku])))))), + ("bookings", Json.Of(Bookings.Select(booking => (Json)Json.Object( + ("id", Json.Of(booking.Id)), ("payload", Json.Of(booking.Payload)), ("receipt", Json.Of(booking.Receipt)))))), + ("outbox", Json.Of(Outbox.Select(entry => (Json)Json.Object( + ("id", Json.Of(entry.Id)), ("receipt", Json.Of(entry.Receipt)), ("delivered", Json.Of(entry.Delivered)))))), + ("deliveries", Json.Of(Deliveries.Select(delivery => (Json)Json.Of(delivery))))); + + [SuppressMessage("Design", "CA1031", Justification = "Any malformed snapshot is reported as a version mismatch.")] + internal static Inventory? FromJson(Json value) + { + if (value.AsObject() is not { } json || !Json.Of(InventorySchema).Equals(json.Get("schema"))) + { + return null; + } + + try + { + var stock = new SortedDictionary(StringComparer.Ordinal); + var skus = new List(); + foreach (Json item in json["stock"].AsArray()!) + { + string sku = item.AsObject()!["sku"].AsString()!; + stock[sku] = item.AsObject()!["available"].AsInt()!.Value; + skus.Add(sku); + } + + List bookings = json["bookings"].AsArray()!.Select(item => item.AsObject()!) + .Select(fields => new Booking(fields["id"].AsString()!, fields["payload"].AsString()!, fields["receipt"].AsString()!)).ToList(); + List outbox = json["outbox"].AsArray()!.Select(item => item.AsObject()!) + .Select(fields => new Outbox(fields["id"].AsString()!, fields["receipt"].AsString()!, ((JsonBool)fields["delivered"]).Value)).ToList(); + List deliveries = json["deliveries"].AsArray()!.Select(item => item.AsString()!).ToList(); + return new Inventory(stock, skus, bookings, outbox, deliveries); + } + catch (Exception) + { + return null; + } + } + + internal Inventory Copy() => new( + new SortedDictionary(Stock, StringComparer.Ordinal), new List(Skus), new List(Bookings), new List(Outbox), new List(Deliveries)); + } + + /// Loads the snapshot at or starts synthetic stock. + /// The snapshot cannot be read or is not this version. + public static Warehouse Open(string path, string fault) + { + ArgumentNullException.ThrowIfNull(path); + ArgumentNullException.ThrowIfNull(fault); + Inventory state; + try + { + byte[] raw = File.ReadAllBytes(path); + Inventory? parsed; + try + { + parsed = Inventory.FromJson(JsonCodec.Decode(raw, 16)); + } + catch (ProtocolException) + { + parsed = null; + } + + state = parsed ?? throw new UsageException("inventory snapshot version mismatch"); + } + catch (FileNotFoundException) + { + state = new Inventory( + new SortedDictionary(StringComparer.Ordinal) { ["sensor"] = 20, ["cable"] = 30, ["battery"] = 15 }, + new List { "sensor", "cable", "battery" }, + new List(), + new List(), + new List()); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + throw new UsageException(error.Message); + } + + return new Warehouse(path, fault, state.Copy()); + } + + /// + /// Commits : a fresh owner-only file, created + /// exclusively, then renamed over the snapshot. Only a file this call + /// created is ever removed, and not once it was renamed. + /// + private void Commit(Inventory nextState) + { + string parent = Path.GetDirectoryName(Path.GetFullPath(path))!; + Directory.CreateDirectory(parent); + byte[] text = Encoding.UTF8.GetBytes(Compact(nextState.ToJson()) + "\n"); + string temporary = Path.Combine(parent, Path.GetFileName(path) + "." + Environment.ProcessId.ToString(CultureInfo.InvariantCulture) + "-" + + Interlocked.Increment(ref next).ToString(CultureInfo.InvariantCulture) + ".tmp"); + var options = new FileStreamOptions { Mode = FileMode.CreateNew, Access = FileAccess.Write }; + if (!OperatingSystem.IsWindows()) + { + options.UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite; + } + + // Throws if the name exists, so nothing below runs for a file this call did not make. + var stream = new FileStream(temporary, options); + bool moved = false; + try + { + using (stream) + { + stream.Write(text); + } + + File.Move(temporary, path, overwrite: true); + moved = true; + } + finally + { + if (!moved) + { + File.Delete(temporary); + } + } + + state = nextState; + } + + private Booking? Find(string id) => state.Bookings.FirstOrDefault(booking => booking.Id == id); + + /// Serves Inventory.booking-status. + public Reservation BookingStatus(Order order) + { + ArgumentNullException.ThrowIfNull(order); + lock (sync) + { + Booking? previous = Find(order.Id); + if (previous is null) + { + return new BookingPending(); + } + + return previous.Payload == Payload(order) ? new RepeatBooking(previous.Receipt) : new BookingConflict(); + } + } + + /// Serves Inventory.lookup-stock. + public Stock LookupStock(string sku) + { + lock (sync) + { + return state.Stock.TryGetValue(sku, out long available) ? new Available(available) : new UnknownSku(); + } + } + + /// Serves Inventory.reserve-order, committing booking and outbox together. + /// An invalid reservation, or the injected fault after the commit. + public Reservation ReserveOrder(Order order) + { + ArgumentNullException.ThrowIfNull(order); + lock (sync) + { + var seen = new HashSet(StringComparer.Ordinal); + bool valid = order.Id.Length > 0 && order.Lines.Count >= 1 && order.Lines.Count <= 20 + && order.Lines.All(line => seen.Add(line.Sku) && line.Units >= 1 && line.Units <= 20); + if (!valid) + { + throw HandlerException.Refused("invalid synthetic reservation"); + } + + if (Find(order.Id) is not null) + { + return BookingStatus(order); + } + + Inventory nextState = state.Copy(); + foreach (Line line in order.Lines) + { + if (!nextState.Stock.TryGetValue(line.Sku, out long available) || available < line.Units) + { + return new StockChanged(); + } + + nextState.Stock[line.Sku] = available - line.Units; + } + + string body = Payload(order); + string receipt = Sha256(order.Id + "\0" + body); + nextState.Bookings.Add(new Booking(order.Id, body, receipt)); + nextState.Outbox.Add(new Outbox(order.Id, receipt, false)); + Commit(nextState); + if (fault == "after-commit") + { + throw HandlerException.CompletionUnknown("synthetic fault after the booking committed"); + } + + return new Booked(receipt); + } + } + + /// Serves Dispatch.publish-booking. + /// The injected delivery failure, no matching entry, or the fault after the delivery. + public Unit PublishBooking(string id, string receipt) + { + lock (sync) + { + if (fault == "publish-failed") + { + throw HandlerException.OutsideFailed("synthetic delivery unavailable; outbox retained"); + } + + Inventory nextState = state.Copy(); + int index = nextState.Outbox.FindIndex(entry => entry.Receipt == receipt && entry.Id == id); + if (index < 0) + { + throw HandlerException.Refused("no matching outbox entry"); + } + + nextState.Outbox[index] = new Outbox(id, receipt, true); + if (!nextState.Deliveries.Contains(receipt)) + { + nextState.Deliveries.Add(receipt); + } + + Commit(nextState); + if (fault == "after-delivery") + { + throw HandlerException.CompletionUnknown("synthetic fault after the delivery committed"); + } + + return Unit.Value; + } + } + + /// Publishes every undelivered outbox entry explicitly, returning how many were delivered. + public int DeliverPending() + { + lock (sync) + { + List pending = state.Outbox.Where(entry => !entry.Delivered).ToList(); + foreach (Outbox entry in pending) + { + PublishBooking(entry.Id, entry.Receipt); + } + + return pending.Count; + } + } + + /// Counts bookings, pending deliveries, and deliveries. + public JsonObject Summary() + { + lock (sync) + { + return Json.Object( + ("bookings", Json.Of(state.Bookings.Count)), + ("deliveries", Json.Of(state.Deliveries.Count)), + ("pendingDeliveries", Json.Of(state.Outbox.Count(entry => !entry.Delivered)))); + } + } + } +} + +/// A usage, input, or I/O failure: exit status 2. +public sealed class UsageException : Exception +{ + /// A failure with a message. + public UsageException() + { + } + + /// A failure with a message. + public UsageException(string message) + : base(message) + { + } + + /// A failure with a message and its cause. + public UsageException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/dotnet/examples/JacquardExamples/JacquardExamples.csproj b/dotnet/examples/JacquardExamples/JacquardExamples.csproj new file mode 100644 index 0000000..db872b7 --- /dev/null +++ b/dotnet/examples/JacquardExamples/JacquardExamples.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + JacquardLang.Host.Examples + JacquardExamples + false + + false + + + + + + + diff --git a/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj b/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj index 6ae2d10..3cb197a 100644 --- a/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj +++ b/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj @@ -31,6 +31,7 @@ + diff --git a/dotnet/test/JacquardHost.Tests/Sdk/ExamplesTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/ExamplesTests.cs new file mode 100644 index 0000000..cd1040c --- /dev/null +++ b/dotnet/test/JacquardHost.Tests/Sdk/ExamplesTests.cs @@ -0,0 +1,206 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using JacquardLang.Host.Examples; +using JacquardLang.Host.Protocol; +using JacquardLang.Host.Sdk; +using NUnit.Framework; + +namespace JacquardLang.Host.Tests.Sdk; + +/// +/// The example projects run as commands through the .NET SDK against the +/// real Core named by JACQUARD_BIN and JACQUARD_PRELUDE: the +/// live tests are ignored without them, and fail instead when +/// JACQUARD_REQUIRE_LIVE is set. The usage test needs no Core. +/// +[TestFixture] +[Platform("Linux,MacOsX")] +public sealed class ExamplesTests +{ + private string root = string.Empty; + + private static string ExamplesDirectory => Path.GetFullPath(Path.Combine(SdkLiveTests.DotnetRoot(), "..", "examples")); + + [SetUp] + public void Prepare() => root = SdkSupport.Scratch("examples"); + + [TearDown] + public void CleanUp() => SdkSupport.RemoveScratch(root); + + /// One command's output and its failure, if any. + private sealed record Outcome(string Output, Exception? Failure) + { + internal bool Ok => Failure is null; + } + + private static Outcome Command(params string[] arguments) + { + var output = new StringWriter(); + try + { + Examples.Examples.Run(arguments, output, ExamplesDirectory); + return new Outcome(output.ToString(), null); + } + catch (Exception error) when (error is JacquardException or UsageException) + { + return new Outcome(output.ToString(), error); + } + } + + private static JsonObject Summary(string output) => + JsonCodec.DecodeObject(System.Text.Encoding.UTF8.GetBytes(output.Trim().Split('\n')[^1]), 4); + + [Test] + public void AllFourProjectsRunAsCommands() + { + SdkLiveTests.RequireLive(); + foreach (string project in Examples.Examples.Projects) + { + string workdir = Path.Combine(root, project); + Outcome outcome = Command("--workdir", workdir, project); + Assert.That(outcome.Ok, Is.True, project + ": " + outcome.Failure + "\n" + outcome.Output); + switch (project) + { + case "quickstart": + Assert.That(outcome.Output, Is.EqualTo("Order total 35: shipping 5\nOrder total 75: shipping 0\n")); + break; + case "planner": + Assert.That(outcome.Output, Does.Contain("\"score\": 16").And.Contain("planner.plan: ok")); + break; + case "triage": + IReadOnlyList rows = JsonCodec.Decode(System.Text.Encoding.UTF8.GetBytes(outcome.Output.Trim()), 16).AsArray()!; + Assert.That(rows.Select(row => row.AsObject()!["urgency"].AsString()), Is.EqualTo(new[] { "PAGE", "TODAY", "WATCH" }), outcome.Output); + break; + default: + Assert.That(outcome.Output, Does.Contain("\"$tag\": \"Fulfilled\"")); + Outcome repeated = Command("--workdir", workdir, project); + Assert.That(repeated.Ok && repeated.Output.Contains("DuplicateOrder", StringComparison.Ordinal) && repeated.Output.Contains("\"deliveries\":1", StringComparison.Ordinal), Is.True, repeated.Output); + break; + } + } + + string recovery = Path.Combine(root, "recovery"); + Outcome failed = Command("--workdir", recovery, "--fault", "publish-failed", "fulfillment"); + Assert.That(failed.Failure, Is.InstanceOf(), "publish failure: " + failed.Output); + var error = (JacquardException)failed.Failure!; + Assert.That(error.Failure, Is.EqualTo(Failure.OperationFailed)); + Assert.That(error.Result!.Code, Is.EqualTo("E1613")); + Assert.That(failed.Output, Does.Contain("\"pendingDeliveries\":1")); + + // A failed explicit delivery prints no count. + Outcome undelivered = Command("--workdir", recovery, "--fault", "publish-failed", "--deliver-pending", "fulfillment"); + Assert.That(undelivered.Failure is UsageException && !undelivered.Output.Contains("delivered", StringComparison.Ordinal), Is.True, undelivered.Output); + foreach (string expected in new[] { "{\"delivered\":1}", "{\"delivered\":0}" }) + { + Outcome delivered = Command("--workdir", recovery, "--deliver-pending", "fulfillment"); + Assert.That(delivered.Ok && delivered.Output.Contains(expected, StringComparison.Ordinal), Is.True, "deliver pending: " + delivered.Failure + " " + delivered.Output); + } + + string inspect = Path.Combine(root, "triage-inspect"); + Outcome inspected = Command("--workdir", Path.Combine(root, "triage"), "--inspect", inspect, "triage"); + Assert.That(inspected.Ok, Is.True, inspected.Failure?.ToString()); + Assert.That(File.Exists(inspect + ".0.json"), Is.True, "inspection not saved"); + Assert.That(File.GetUnixFileMode(inspect + ".0.json"), Is.EqualTo(UnixFileMode.UserRead | UnixFileMode.UserWrite)); + Assert.That(Command("unknown").Failure, Is.InstanceOf()); + Assert.That(Command("--fault", "nope", "fulfillment").Failure, Is.InstanceOf()); + } + + [Test] + public void PartialCompletionFaultsRecoverExplicitlyAcrossRestart() + { + SdkLiveTests.RequireLive(); + foreach ((string fault, string code, long delivered) in new[] { ("after-commit", "E1612", 1L), ("publish-failed", "E1613", 1L), ("after-delivery", "E1612", 0L) }) + { + string workdir = Path.Combine(root, fault); + Outcome faulted = Command("--workdir", workdir, "--fault", fault, "fulfillment"); + Assert.That(faulted.Failure, Is.InstanceOf(), fault); + Assert.That(((JacquardException)faulted.Failure!).Result!.Code, Is.EqualTo(code), fault); + Assert.That(faulted.Output, Does.Contain("\"bookings\":1"), fault); + + // A restart sees the committed booking; recovery is explicit. + Outcome recovered = Command("--workdir", workdir, "--deliver-pending", "fulfillment"); + Assert.That(recovered.Ok && recovered.Output.Contains("{\"delivered\":" + delivered + "}", StringComparison.Ordinal), Is.True, fault + ": " + recovered.Output); + Outcome repeated = Command("--workdir", workdir, "fulfillment"); + Assert.That(repeated.Ok && repeated.Output.Contains("DuplicateOrder", StringComparison.Ordinal) && repeated.Output.Contains("\"pendingDeliveries\":0", StringComparison.Ordinal), Is.True, fault + ": " + repeated.Output); + Assert.That(Summary(repeated.Output)["deliveries"], Is.EqualTo(Json.Of(1)), fault); + } + } + + /// + /// The store's temporary file is created exclusively, so a file already + /// at its name (not the store's) makes the commit fail and is left as it + /// was. + /// + [Test] + public void TheStoreRemovesOnlyTheTemporaryFileItCreated() + { + string inventory = Path.Combine(root, "inventory.json"); + Examples.Examples.Warehouse warehouse = Examples.Examples.Warehouse.Open(inventory, string.Empty); + + // The name the next commit will use: the store's counter plus one. + long upcoming = (long)typeof(Examples.Examples.Warehouse).GetField("next", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!.GetValue(null)! + 1; + string occupied = inventory + "." + Environment.ProcessId + "-" + upcoming + ".tmp"; + File.WriteAllText(occupied, "caller-owned"); + var order = new Examples.Examples.Order("o", new[] { new Examples.Examples.Line("sensor", 1) }); + Support.Throws(() => warehouse.ReserveOrder(order)); + Assert.That(File.ReadAllText(occupied), Is.EqualTo("caller-owned"), "a file the store did not create was changed or removed"); + + // The next commit takes the next name and succeeds. + Assert.That(warehouse.ReserveOrder(order), Is.InstanceOf()); + Assert.That(File.Exists(occupied), Is.True); + } + + /// Runs the command in a fresh process; returns its exit status, stdout, and stderr. + private (int Status, string Stdout, string Stderr) Process(params string[] arguments) + { + var info = new ProcessStartInfo(Support.Muxer()) + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + WorkingDirectory = root, + }; + foreach (string argument in new[] { "exec", "--runtimeconfig", Path.Combine(AppContext.BaseDirectory, "JacquardLang.Host.Tests.runtimeconfig.json"), Path.Combine(AppContext.BaseDirectory, "JacquardExamples.dll") }.Concat(arguments)) + { + info.ArgumentList.Add(argument); + } + + info.Environment["JACQUARD_EXAMPLES"] = ExamplesDirectory; + info.Environment["DOTNET_GCHeapHardLimit"] = "0x10000000"; + using System.Diagnostics.Process process = System.Diagnostics.Process.Start(info)!; + var stdout = System.Threading.Tasks.Task.Run(() => process.StandardOutput.ReadToEnd()); + var stderr = System.Threading.Tasks.Task.Run(() => process.StandardError.ReadToEnd()); + if (!process.WaitForExit(120_000)) + { + process.Kill(entireProcessTree: true); + Assert.Fail("the command did not finish"); + } + + process.WaitForExit(); + return (process.ExitCode, stdout.Result, stderr.Result); + } + + [Test] + public void TheCommandExits2ForUsageErrors() + { + foreach (string[] arguments in new[] { new[] { "unknown" }, new[] { "--fault", "nope", "fulfillment" }, new[] { "--workdir" }, Array.Empty() }) + { + (int status, _, string stderr) = Process(arguments); + Assert.That(status, Is.EqualTo(2), string.Join(" ", arguments) + ": " + stderr); + Assert.That(stderr.Trim(), Is.Not.Empty, string.Join(" ", arguments) + ": no message"); + } + } + + [Test] + public void TheCommandExits1ForAJacquardFailure() + { + SdkLiveTests.RequireLive(); + (int status, string stdout, string stderr) = Process("--workdir", Path.Combine(root, "work"), "--fault", "publish-failed", "fulfillment"); + Assert.That(status, Is.EqualTo(1), stderr); + Assert.That(stderr, Does.Contain("E1613")); + Assert.That(stdout, Does.Contain("\"pendingDeliveries\":1")); + } +} diff --git a/dotnet/test/JacquardHost.Tests/Support.cs b/dotnet/test/JacquardHost.Tests/Support.cs index 34ed3a8..6109817 100644 --- a/dotnet/test/JacquardHost.Tests/Support.cs +++ b/dotnet/test/JacquardHost.Tests/Support.cs @@ -60,7 +60,7 @@ internal static CallInterface Echo(bool withWorld = true) } /// The dotnet executable that runs the stand-in. - private static string Muxer() + internal static string Muxer() { string? host = Environment.GetEnvironmentVariable("DOTNET_HOST_PATH"); if (!string.IsNullOrEmpty(host) && File.Exists(host)) From 1219896cf0b0c29131f5a0f2f0fac0320fb849ea Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 07:05:46 -0400 Subject: [PATCH 06/52] Test that a reused group id is not killed Once a Core command's leader is reaped, a live process holding its pid means the group id was reused; the group kill must then send nothing. An unrelated group leader stands in for that process. --- dotnet/src/JacquardHost/Sdk/CoreCommand.cs | 2 +- .../Sdk/CoreCommandTests.cs | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs index ce0d153..2a0adda 100644 --- a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs +++ b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs @@ -414,7 +414,7 @@ private static (int Pid, int Parent, int Group, string Start, bool Zombie)? Read /// while a group has members its id cannot be reused, so a reaped /// leader's pid held by a live process means the old group is empty. /// - private static void KillGroup(int group, Process leader) + internal static void KillGroup(int group, Process leader) { if (HasExited(leader) && ReadStat("/proc/" + group.ToString(CultureInfo.InvariantCulture)) is { Zombie: false }) { diff --git a/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs index 63f2c92..9998cb0 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs @@ -141,6 +141,31 @@ public void ABackgroundChildIsKilledWhenTheCommandExits() Assert.That(Support.Alive(survivor), Is.True, "without the exit kill the child should survive"); } + /// + /// Once the leader has exited and been reaped, a live process holding its + /// pid means the id was reused: the old group is empty, and the group + /// kill sends nothing. Here an unrelated group leader stands for the + /// process that took the pid after a wraparound; without the check the + /// kill would end its group. + /// + [Test] + public void AGroupIdReusedAfterTheLeaderIsReapedIsNotKilled() + { + Assume.That(HasSetsid, "no setsid at /usr/bin or /bin"); + using Process leader = Process.Start(new ProcessStartInfo("/bin/true") { UseShellExecute = false })!; + leader.WaitForExit(); + Assert.That(leader.HasExited, Is.True); + string pidFile = PidPath("reused"); + using Process other = Process.Start(new ProcessStartInfo("/bin/sh", new[] { "-c", "exec " + CoreCommand.Setsid + " sh -c 'echo $$ > \"" + pidFile + "\"; exec sleep 300'" }) + { + UseShellExecute = false, + })!; + int unrelated = SdkSupport.PidFile(pidFile); + CoreCommand.KillGroup(unrelated, leader); + Thread.Sleep(200); + Assert.That(Support.Alive(unrelated), Is.True, "a group whose id was reused was killed"); + } + /// A child that closed its output and keeps running is in the group, so it is killed at the exit too. [Test] public void ABackgroundChildThatClosedItsOutputIsKilled() From 9ee0e10618208d40896c79df0d2049fcf1866599 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 07:26:34 -0400 Subject: [PATCH 07/52] Rewrite the .NET README around the SDK Document preparation and its bounds, the verified owner-only cache, codecs and the reason for them, money as integer minor units, services, errors, batches, the example projects, and how the SDK runs Core's commands, and keep the protocol-level host, handler, carrier, and kit sections. The README's four C# examples are the compiled sources under Readme/, which DocumentTests checks in order and the live tests run. --- dotnet/README.md | 558 +++++++++++++++++- dotnet/src/JacquardHost/Sdk/Codec.cs | 4 +- .../test/JacquardHost.Tests/DocumentTests.cs | 29 +- 3 files changed, 547 insertions(+), 44 deletions(-) diff --git a/dotnet/README.md b/dotnet/README.md index 6edbe11..b61446b 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -1,32 +1,479 @@ # Jacquard Host for .NET -An independent .NET adapter for Jacquard Core's experimental `jacquard-host-v0` -protocol. It implements the pinned protocol on its own, without calling -Python, Node.js, Go, Rust, or Java, and has no runtime dependencies beyond -the .NET base class library. It targets .NET 8 and runs on .NET 8 and 10. It -runs workers on Linux and macOS: the carrier calls the C library's `poll(2)` -(see "Process carrier" below). - -This is the protocol-level adapter, comparable to the Go `host` package, the -Rust `host` module, and the Java adapter. It includes strict codecs, the host -session state machine, and a bounded process carrier for `jacquard host -worker`. It also provides a `Host` with a closed operation registry and a -replayer for Core's pinned conformance kit. There is no SDK layer yet. It does -not compile programs, map native types, or cache stores. You prepare a store -with the Core CLI and pass exact identities and boundary types. - -**Experimental:** the protocol is Core's pinned `jacquard-host-v0`, and Core is -installed separately. Each invocation runs one fresh, serial worker. The host -and operating system are trusted. Linux is exercised; macOS should work but has -no CI; Windows is not supported (the worker reports a spawn failure). See +Call checked Jacquard functions from .NET, bind .NET handlers to their +declared operations, and inspect the exact protocol evidence. This is an +independent .NET implementation of the pinned `jacquard-host-v0` boundary. It +has no runtime dependencies beyond the .NET base class library and runs Core +directly, without Python, Node.js, Go, Rust, or Java. It targets .NET 8 and +runs on .NET 8 and 10. + +The assembly has two layers: +- **The SDK** (`JacquardLang.Host.Sdk`) prepares and caches checked programs, + maps .NET values through explicit codecs, and binds handlers to exact + operation identities. It is comparable to the Python, TypeScript, Go, Rust, + and Java SDKs. +- **The protocol-level host** (`JacquardLang.Host`) takes exact identities, + boundary types, and protocol values for a store you prepare with the Core + CLI. It is comparable to the Go `host` package, the Rust `host` module, and + the Java adapter. + +Linux is exercised. macOS should work but has no CI: the carrier and the SDK +call the C library (`poll(2)`, and for the SDK `kill(2)`, `geteuid(2)`, +`mkdir(2)`, and `stat(2)`; see [process carrier](#process-carrier) and +[Core's commands](#cores-commands)), and the macOS status calls are compiled +but not run by any test. Windows is not supported: the worker reports a spawn +failure and `Prepare` refuses. **Experimental:** the protocol is Core's pinned +`jacquard-host-v0`, and Core is installed separately. Each invocation runs one +fresh, serial worker. The host and operating system are trusted. See [limits](../docs/LIMITS.md). -## Use +## Install The package is not published to a registry. Build it from a clone (`dotnet pack src/JacquardHost -c Release` in this directory) and reference -the `JacquardLang.Host` package or the project. The example below invokes a -checked function whose single operation is served by a .NET handler: +the `JacquardLang.Host` package or the project. + +From a clone, install the pinned Core release (see [setup](../docs/SETUP.md)), +then run the quickstart: + +```sh +eval "$(python3 scripts/core_pin.py install)" +cd dotnet +dotnet build examples/JacquardExamples +dotnet examples/JacquardExamples/bin/Debug/net8.0/JacquardExamples.dll quickstart +``` + +Expected output: + +```text +Order total 35: shipping 5 +Order total 75: shipping 0 +``` + +The same command runs the other example projects; see +[run the projects](#run-the-projects). + +## Call a checked function + +```csharp +using JacquardLang.Host.Sdk; + +public static class Shipping +{ + public static long Cost(string rules, string cacheDir, string inspection) + { + // Core is $JACQUARD_BIN and $JACQUARD_PRELUDE unless RuntimeOptions say otherwise. + var runtime = new Runtime(); + PreparedProgram program = runtime.Prepare(new[] { rules }, new PrepareOptions { CacheDir = cacheDir }); + Function shipping = program.Function("shipping-cost"); + + // A long converts to an Int argument; Call decodes the result. + long free = shipping.Call(Codecs.Int, 75L); + + // Invoke keeps the raw frames and evidence, saved only on request. + Invocation invocation = shipping.Invoke(35L); + invocation.Save(inspection); + return invocation.Decode(Codecs.Int) + free; + } +} +``` + +The shipping rule is the [quickstart source](../examples/quickstart/rules.jac). +`Prepare` takes `.jac` paths (or `.jqd` paths) in dependency order. It uses +Core's public `check`, `hash`, and `run` commands, and keeps a verified cache +keyed by: +- the source bytes; +- the Core executable bytes; +- every prelude file; +- the protocol and preparation format. + +At most 1,024 sources are taken (`Runtime.SourceFiles`); a longer list is +refused from its `Count` before any element is read, and the token is +observed before each element is checked. A path that is empty, holds a NUL, +or is longer than 4,096 chars as given or made absolute (a source, the cache +directory, the prelude, the executable, or a save path) is refused with an +excerpt before the file system sees it. Each source must be a regular file (a +symbolic link to one is followed), and all of them together at most 16 MiB +(`Runtime.SourceLimit`); both are checked from the files' status before a +source is read, so a FIFO or device is refused rather than opened, and no read +goes past the ceiling. The Core executable is fingerprinted in 64 KiB pieces +up to 512 MiB: a larger file is refused from its size before it is read, the +ceiling is enforced again while it is read, and the token is observed before +each piece. The prelude and cache walks stream each directory on an explicit +stack (no directory depth can exhaust the call stack) and read regular files +only (no symbolic links, FIFOs, or devices), at most 100,000 entries (empty +files and directories count) and 256 MiB under one root; each file's size is +checked before it is read and the ceiling again while it is read, and the +token is observed before each entry and after each piece. A source replaced by a +FIFO between that check and its open can still block the open. + +A preparation stages its entry in a `prepare-*` directory of the cache root, +created exclusively (`mkdir(2)`), so it never adopts a directory it did not +make. One that fails removes that directory in a streamed walk of at most +100,000 entries that ends at the first cancellation, so a stopped preparation +leaves its staging directory behind. Such a directory is never loaded: +entries are named by their identity digest. Nothing removes it later, because +a name and an age cannot show that the SDK made a directory or that no +preparation is using it; remove leftover `prepare-*` directories yourself when +no preparation is running. Once an entry is published, its vacated staging +path is not touched again. A cancellation that arrives as a preparation +finishes, a warm load included, is reported as the stop, not as a program. + +A function whose checked signature nests types deeper than 64 levels (the +protocol's value-depth limit, `max_json_depth`, so no such value could be +passed), names more than 4,096 types, or has more than 16,384 tokens is +refused before the parser recurses past that bound. Like a polymorphic +function it is left unbound: preparation succeeds, the other functions bind, +and `PreparedProgram.Function` on it fails with a `Binding` exception naming +the limit. `check --print-sigs` prints no operation signatures, and the SDK +parses none; an operation's arguments are checked by the protocol's limits +when they arrive. + +The cache defaults to `.scratch/jacquard-host-dotnet` under the working +directory; `PrepareOptions.CacheDir` changes it. A symbolic link at the cache +root is followed once, and the directory it names is checked and used; +nothing inside it is followed. A cache entry keeps Core's raw +`check --print-sigs` output and the `hash` output for the source and each +prelude file beside the store, and `program.json` records every file's digest. +Every binding (names, constructors, signatures, and declaration groups) is +derived from those outputs, on the first preparation and again on each load. +A load refuses an entry whose files do not match the recorded digests, or +whose `program.json` bindings differ from what the outputs give, rather than +repairing it. So an accidental or partial change is refused. No record inside +an entry can resist someone who can rewrite it (an output, its recorded +digest, and the bindings changed together load), so the cache's owner is +trusted and only the owner may write it: preparation refuses a cache root, an +entry, or any file or directory in one that the current user does not own or +that its group or others may write, both when it creates an entry and when it +loads one, and it makes each new entry's directories and files owner-only +(Core's store included). The owner and mode come from `statx(2)` on Linux and +`stat(2)` on macOS, without following links. The cache root's parent +directories are not checked. The `--version` check and prelude discovery run +once per Core executable and prelude in the process, shared by every +`Runtime` and thread. Each call runs one fresh worker. Prepare only +first-party declaration sources: preparation runs Core's `run` command, +without service bindings, to populate the store. + +`new Runtime()` uses `$JACQUARD_BIN` and `$JACQUARD_PRELUDE`; an empty option +means unset. The runtime refuses a Core whose `--version` differs from the +pinned release. Each Core command must finish within 30 seconds, and keeps at +most 8 MiB of stdout and 4 KiB of stderr; see [Core's commands](#cores-commands). +Cancelling the token `Prepare` takes kills the running command and its +process group, and fails the preparation; a preparation waiting for another +thread's version check or prelude discovery stops waiting too. The SDK +observes only the token: `Thread.Interrupt` is not a stop signal, and the +SDK's own waits (`poll(2)` sleeps) do not consume it. + +Arguments are `Argument`s. A `long` or `int` converts implicitly to an `Int`, +a `double` to a `Real`, a `string` to a `Text`, and a `bool`, a protocol +`Hash`, a protocol `Value`, or a `Dynamic` to itself; anything else, lists and +records included, is `codec.Arg(value)`. There is deliberately no conversion +from `decimal` (see [data](#data)). `Call` decodes the result with the codec +you pass; `Invoke` keeps the whole `Invocation`. Overloads that take an +argument list and a `CancellationToken` kill the worker when it is cancelled. +`Invocation.Save` writes the frames and evidence, including argument and +result values; a new file is created owner-only, and an existing file keeps +its permissions. The API is synchronous; there are no `Task`-returning +variants. + +## Data + +Values map through explicit `Codec` objects, built with the factories on +`Codecs`. A record is mapped by naming its constructor and giving, for each +field in the constructor's positional order, a codec and a function that +reads it, then a function that builds the record from the decoded fields. +Records are mapped without reflection (enumerations use `Enum.GetValues` and +`Enum.GetName`), and there is no attribute or source generator: a codec is an +ordinary immutable, thread-safe object, so the library builds clean under the +trimming and ahead-of-time analyzers (`IsAotCompatible`; a trimmed or +native-AOT publish is not tested), and a binding that does not follow a +convention (a constructor whose name differs from the type's, say) is one +explicit expression. Before any worker starts, the encoded arguments are +checked against the function's checked signature; Core always checks the +actual wire types too. + +| .NET | Jacquard | +|---|---| +| `long` (`Codecs.Int`) | `Int`, Core's full 63-bit range; a `long` outside it is refused. `int` (`Codecs.Int32`) refuses a result that does not fit | +| `double` (`Codecs.Real`) | `Real` as binary64 bits, including NaN payloads and signed zero; an integer at a `Real` parameter is refused | +| `string`, `bool`, protocol `Hash` | `Text` (a lone surrogate is refused), `Bool`, `Hash` | +| `IReadOnlyList` (`Codecs.List`) | `List T`; decoding returns a new array | +| `Option` (`Codecs.Option`) | `Option T`; `Option>` keeps nested options distinct. `Codecs.Nullable` and `Codecs.NullableValue` map `null` to `None` for one level | +| `Unit` (`Codecs.Unit`), value tuples (`Codecs.Tuple`, two to four items) | unit and tuples | +| a record (`Codecs.Record`, `Codecs.Field`) | one constructor, named by public name, its fields in the order given | +| an enumeration (`Codecs.Enum`) | nullary constructors by convention: member `PageNow` is the constructor `PageNow` (store spelling `page-now`), or an explicit map | +| a type hierarchy (`Codecs.Union`) | the constructors of one data type: each case is a record, enumeration, or union codec of a subtype | +| a recursive type (`Codecs.Lazy`) | a codec made once, thread-safely, on first use, so a field can refer to its own type's codec | +| `Dynamic` (`Codecs.Dynamic`) | any of the above by public name, for code with no application type | +| protocol `Value` (`Codecs.Value`) | passed through unchanged | + +```csharp +using System.Collections.Generic; +using JacquardLang.Host.Sdk; + +// type Incident = | Incident(id: Text, service: Text, severity: Int, age: Int, visible: Bool, owner: Option Text) +// type Urgency = | PageNow | WorkToday | WatchOnly +// type Triage = | Routed Text Text Urgency (List Text) | InvalidIncident Text +public static class Incidents +{ + public sealed record Incident(string Id, string Service, long Severity, long Age, bool Visible, string? Owner); + + public enum Urgency + { + PageNow, + WorkToday, + WatchOnly, + } + + public abstract record Triage; + + public sealed record Routed(string Id, string Team, Urgency Urgency, IReadOnlyList Reasons) : Triage; + + public sealed record InvalidIncident(string Id) : Triage; + + // A record is one constructor: its fields in order, each with a codec and + // a reader, and a function that builds it from the decoded fields. + public static readonly RecordCodec IncidentCodec = Codecs.Record( + "Incident", + Codecs.Field("id", Codecs.Text, (Incident incident) => incident.Id), + Codecs.Field("service", Codecs.Text, (Incident incident) => incident.Service), + Codecs.Field("severity", Codecs.Int, (Incident incident) => incident.Severity), + Codecs.Field("age", Codecs.Int, (Incident incident) => incident.Age), + Codecs.Field("visible", Codecs.Bool, (Incident incident) => incident.Visible), + Codecs.Field("owner", Codecs.Nullable(Codecs.Text), (Incident incident) => incident.Owner), + (id, service, severity, age, visible, owner) => new Incident(id, service, severity, age, visible, owner)); + + // An enumeration's members name nullary constructors: PageNow is page-now. + // A union's cases are records, enumerations, or unions of its subtypes. + public static readonly UnionCodec TriageCodec = Codecs.Union( + Codecs.Record( + "Routed", + Codecs.Field("id", Codecs.Text, (Routed routed) => routed.Id), + Codecs.Field("team", Codecs.Text, (Routed routed) => routed.Team), + Codecs.Field("urgency", Codecs.Enum(), (Routed routed) => routed.Urgency), + Codecs.Field("reasons", Codecs.List(Codecs.Text), (Routed routed) => routed.Reasons), + (id, team, urgency, reasons) => new Routed(id, team, urgency, reasons)), + Codecs.Record("InvalidIncident", Codecs.Field("id", Codecs.Text, (InvalidIncident invalid) => invalid.Id), id => new InvalidIncident(id))); + + // triage.batch : (TriagePolicy, List Incident) ->{} List Triage, run in + // explicit batches of eight; each batch is checked before any runs. + public static IReadOnlyList Route(Function batch, Dynamic policy, IReadOnlyList incidents) => + batch.MapBatches(TriageCodec, IncidentCodec, incidents, 8, new Argument[] { policy }); +} +``` + +The fields given must follow the Jacquard constructor's positional order; the +pinned CLI exports no field types, so the order is not checked against Core. +Constructor names may use display spelling (`WorkToday`) or store spelling +(`work-today`). A constructor that is not in the prepared program is refused +when a value is encoded, and a result built by a different constructor fails +to decode. `null` is never a Jacquard value (except through a nullable +codec). **Money crosses as integer minor units** (cents, say) in an `Int`: +Jacquard's `Int` is 63-bit and its `Real` is a binary64 `double`, and there is +no decimal type. The SDK has no `decimal` codec and no conversion from +`decimal`, so a `decimal` is never converted silently; a test checks that none +exists. + +Every encoding is bounded by the selected limits as it goes: the JSON depth +the wire form needs, `max_collection_items` for each list, tuple, and +constructor, `max_value_nodes` values, `max_text_bytes` (in UTF-8 bytes) for +each text, and `max_frame_bytes` for all texts together. A list, tuple, +record, or constructor over its count limit is refused from its count before +its items or fields are read, a text over its limit before it is scanned, and +nesting before the nested value is encoded, so a small .NET value that denotes +a huge one (1024 references to one tuple of 1024 references, say) costs no +more than one frame, and no value or codec nesting can recurse past the depth +limit. `ProgramData.Decode` measures a value within the same node and depth +limits before it decodes it. A custom codec may return a `Value` it built +itself rather than through the `Encoder`; such a value is still measured: +before a function call checks the arguments' types, it walks every argument +value under one shared budget (nodes, collection items, text bytes alone and +together, and depth), whatever codec built it, so a small shared graph that +stands for a huge value is refused before the type check walks it. The invoke +frame is then measured as it is rendered, before any worker starts, and the +host measures a handler's result the same way. A value returned by +`ProgramData.Encode` is not measured again. Raw values, decoded values, +nested unions (flattened on first use to their distinct cases), chains of +lazy codecs, and a `Dynamic`'s equality and text are walked iteratively, so +none can exhaust the call stack. `PreparedProgram.Data` gives each public +name's exact identity. `ProgramData.Make` builds a `Dynamic.Variant` after +checking that its constructor exists, and `Invocation.ToDynamic` decodes any +result into `Dynamic`. A `Dynamic` list, tuple, or variant with more items +than the v0 hard limit (1,024) is refused when it is made, from its count +where the collection gives one and otherwise after at most one item past the +limit. + +## Services and failures + +```csharp +using System.Collections.Generic; +using JacquardLang.Host; +using JacquardLang.Host.Sdk; + +public static class Documents +{ + // summarize : (Text) ->{Documents} Text, where Documents has one operation, + // read-document : (Text) -> Text. + public static Function Summarize(PreparedProgram program) + { + Operations documents = new Operations().On("read-document", Codecs.Text, Codecs.Text, (context, key) => + { + // The context reports a cancelled or ended invocation; its + // CancellationToken is cancelled then too. + if (context.IsCancelled) + { + throw HandlerException.CompletionUnknown("cancelled"); + } + + return key == "example" ? "A synthetic document supplied by .NET." : throw HandlerException.Refused("unknown document"); + }); + return program.Function("summarize", new Dictionary { ["Documents"] = documents }); + } +} +``` + +An `Operations` value maps operation names (store spelling such as +`read-document`, or display spelling; `_` is accepted for `-`) to handlers. +One `Operations` may serve several effects. + +- **Parameters:** `On` takes one codec per parameter (up to six) and one for + the result. The handler may take the `HandlerContext` first; the remaining + parameters are decoded from the request. The context's `IsCancelled` and + `CancellationToken` report a cancelled, expired, or ended invocation (see + [handlers and cancellation](#handlers-and-cancellation)). +- **Results:** the handler returns a value for its result codec + (`Unit.Value` for `Codecs.Unit`). +- **Failures:** throw `HandlerException.Unsupported`, `Refused`, + `OutsideFailed`, or `CompletionUnknown` to report that failure. Any other + exception is reported to Core as completion unknown without its message. +- **Binding:** every operation in each checked root effect needs a handler + before the first invocation, including operations a given path never + reaches. A name that is no operation of the program is refused, and a + service binding more names than the program has operations is refused from + its count before its handlers are copied. `PreparedProgram.Provide` shares + services across functions; each function receives only its own checked + effects. + +Handlers run on their own threads, so share state through a lock, as the +fulfillment example's warehouse does. `RuntimeOptions.Deadline` (default +30 s) bounds each worker lifetime. + +## Errors + +Every failure to prepare, bind, encode, call, or decode is a +`JacquardException`, including any exception (other than +`OutOfMemoryException`) thrown by your own codec, a record field's reader, a +record's constructor function, or a lazy codec's factory: while a value is +encoded, by a function or by `ProgramData.Encode`, it is an `InvalidArgument` +naming the value, and while a value is decoded, by `ProgramData.Decode`, +`Invocation.Decode`, `Call`, or `MapBatches`, a `Binding` failure (from +`Invocation.Decode`, `Call`, and `MapBatches`, with the raw result kept); +either way your exception is its `InnerException`. Names and exception +messages the SDK quotes (a constructor, type, field, effect, operation, or +executable name, or your exception's message) appear as an excerpt of at most +64 chars and its length. A message you pass to `Encoder.Invalid` or +`Decoder.Mapping` is kept as written, and no message is longer than +`JacquardException.MessageLimit` (8,192 chars), the start kept and the whole +length noted, once, however many prefixes are joined to it. Some failures +keep their usual .NET types: passing `null` where an object is required +throws `ArgumentNullException`; building a codec that cannot work (an +enumeration map missing a member, a union case that is not a subtype) or a +`Dynamic` over the hard limit throws `ArgumentException` at construction; +`Invocation.Save` throws `IOException` or `UnauthorizedAccessException`; and +an exception from a collection you pass (its `Count`, indexer, or +enumerator) propagates unchanged, except for `MapBatches` rows, which are read +while a batch is encoded and so are reported as a codec failure. A +`JacquardException`'s `Kind` is: +- `InvalidArgument`: a local refusal, such as a value of the wrong type or + one too large or too deep for a frame. No worker starts. `Invoke` throws + only this; otherwise the `Invocation` holds the raw result. +- `Invocation`: the invocation ran and did not succeed, from `Call`, + `Invocation.Decode`, and `MapBatches`. `Failure` classifies it as + `DeadlineExceeded`, `WorkerLost`, `OperationFailed`, or `Other`, and + `Result` keeps the raw result. +- `Binding`: such as an unknown name, a missing handler, invalid runtime + options, or a result that does not decode into the requested type. In the + decode case, `Result` keeps the successful Core result. +- `Preparation`: checking, the cache, and the Core commands. + +`MapBatches` runs a pure function whose final parameter and result are Lists +over explicit batches of rows, encoded with the row codec you pass, with any +other arguments first. It validates every batch before the first runs and +retries nothing. Validation keeps no batch's encoding: each batch is encoded +and checked by a step that returns nothing, and encoded and checked again +just before it runs. Before each batch, and once more after the last, it +observes the token. As in the other SDKs, validation has no deadline of its +own. A cancellation there is reported as the host's aborted invocation, +without a worker. A failed batch's exception names it in `BatchIndex` (-1 when +nothing ran: a batch was refused, or validation stopped) and keeps the decoded +values of the batches that completed in `Completed`, read-only. + +A request argument that does not decode into a handler's parameter, or a +wrong argument count, is reported as an outside failure, and that handler +does not run. Neither does a handler whose invocation was cancelled, or whose +lifetime passed, while its arguments were decoded. A result that cannot be +encoded is also reported as an outside failure, but only after the handler +returned, so any action it took has happened. None of these says anything +about earlier operations in the invocation. + +## Run the projects + +```sh +cd dotnet +dotnet build examples/JacquardExamples +RUN="dotnet examples/JacquardExamples/bin/Debug/net8.0/JacquardExamples.dll" +$RUN planner +$RUN triage +$RUN fulfillment +$RUN fulfillment +``` + +- Planner returns score 16. +- Triage returns PAGE, TODAY, and WATCH. +- Fulfillment returns Fulfilled, then DuplicateOrder with the same receipt. + +The command reads the rules and synthetic inputs from the repository's +[`examples/`](../examples/README.md), from the repository root or `dotnet/` +(or `JACQUARD_EXAMPLES=DIR`), so it runs from a clone. Custom input JSON may +follow the project name. `--inspect PATH` saves frames; triage adds a +numbered suffix per batch. State defaults to `.scratch/dotnet-examples/`. The +exit status is 1 for a Jacquard or SDK failure and 2 for usage or other +errors. + +Try explicit recovery in a fresh work directory: + +```sh +$RUN --workdir .scratch/dotnet-recovery --fault publish-failed fulfillment +$RUN --workdir .scratch/dotnet-recovery --deliver-pending fulfillment +$RUN --workdir .scratch/dotnet-recovery --deliver-pending fulfillment +``` + +The first command exits 1 with E1613 and one pending booking. Reconciliation +then delivers one receipt, and the next run delivers zero. The `after-commit` +and `after-delivery` faults exercise E1612 around the snapshot commits. + +The example's `Warehouse` is a single-process service. It writes bookings and +outbox entries together as one atomic JSON snapshot (a temporary file it +creates exclusively, then renames; it removes that file only if it created it +and did not rename it), and a simulated delivery sink deduplicates receipts. +It has no cross-process locking and makes no power-loss durability claim. + +## Namespaces + +| Namespace | Contents | +|---|---| +| `JacquardLang.Host.Sdk` | The SDK: `Runtime`, `PreparedProgram`, `ProgramData`, `Function`, `Invocation`, `Operations`, `Codec` and `Codecs`, `Dynamic`, `Option`, `Unit`, `Argument`, and `JacquardException` | +| `JacquardLang.Host` | Low-level `Host`, `OperationRegistry`, `HandlerException`, `HandlerContext`, and `InvocationResult` for exact identities | +| `JacquardLang.Host.Carrier` | The bounded process carrier for `jacquard host worker` | +| `JacquardLang.Host.Protocol` | Strict JSON, v0 values and types, limits, the `Session` state machine | +| `JacquardLang.Host.Conformance` | The embedded copy of the pinned conformance kit and its replayer | + +`Compat` holds the generated pin constants. + +## The protocol-level host + +The example below invokes a checked function whose single operation is served +by a .NET handler: ```csharp using System; @@ -116,7 +563,7 @@ size of what a handler or caller hands over: ## Handlers and cancellation -A handler returns a value or throws a `HandlerException`. The four structured +These rules apply to SDK handlers and to low-level handlers. A handler returns a value or throws a `HandlerException`. The four structured categories are `Unsupported`, `Refused`, `OutsideFailed`, and `CompletionUnknown`. Any other exception a handler throws is reported to Core as completion unknown, and its message is never sent. A returned value with @@ -176,7 +623,7 @@ Unix an anonymous pipe's asynchronous read is a blocking read on a pool thread. The carrier is built around both: - **No read or write ever blocks.** The caller's thread asks the C library's - `poll(2)` (the one native call the adapter makes) whether a pipe is ready, + `poll(2)` (the one native call the carrier makes) whether a pipe is ready, with a poll timeout of at most 5 ms that never ends after the deadline, and only then reads the bytes already in the pipe, or writes at most `PIPE_BUF` bytes (4096 on Linux, 512 on macOS), which a @@ -237,6 +684,49 @@ invocations, and check that a reparented descendant flooding stdout or stderr holds neither the caller nor the heap, at a worker's own exit, at expiry, and at cancellation. +## Core's commands + +The SDK runs Core's `--version`, `check`, `hash`, and `run` commands through +its own bounded runner, which uses the carrier's approach to pipes: stdout and +stderr are read only after `poll(2)` reports them ready, on the caller's +thread, in waits of 1 to 5 ms that recheck the 30-second timeout and the +token; .NET's own stream reads are never left waiting. Each read step takes +at most 1 MiB of each stream, keeps at most 8 MiB of stdout (the rest is read +and counted, and refused as "output exceeds 8 MiB") and 4 KiB of stderr, and +after the exit at most 4 MiB more of each is read. The command's stdin is a +pipe the host closes at once. + +On Linux, when a `setsid` is installed at `/usr/bin/setsid` or `/bin/setsid` +(util-linux's, or a compatible one such as BusyBox's; any executable there is +used), each command runs as `setsid -- `. .NET's child is never a +process group leader, so `setsid` starts a new session and execs Core without +forking: Core keeps its pid and leads a new process group whose id is that pid +(a test checks both). Every descendant that does not leave the group shares +it. The host kills that group with `kill(-pgid, SIGKILL)` when the command +exits, and on a timeout or cancellation (then also killing Core itself, +waiting up to one second for its exit, and killing the group again for +members forked meanwhile). So `sh -c 'sleep 300 & echo $!'` succeeds, as in +the Rust SDK, and its child has been sent SIGKILL by the time it returns. Once .NET has reaped +Core, a live process holding Core's pid means the id was reused, so the old +group is empty and nothing is killed; a process that took Core's pid after a +full pid wraparound, made its own group, and forked between that check and the +kill could still have that group killed, the same window as the Rust SDK's +`kill(-pgid)`. When `setsid` cannot exec Core (exit status 126 or 127 with its +own `setsid: ` message), the command is reported as a start failure. A +descendant that leaves the group (calls `setsid` itself) is neither detected +nor killed; the host does not wait for the end of the output after the exit, +and closes its ends, so that descendant's next write fails and it cannot hold +the caller. + +Without `setsid` (Linux without one at those paths), the command's +descendants are recorded from `/proc` about every 10 ms, by pid and start +time, and every recorded descendant still alive with that start time is +killed when the command returns (the start time is read just before the +kill, as the JDK's process handles do); a stop also calls +`Process.Kill(true)`. A descendant started and reparented between two records +keeps running. macOS has no `/proc`, so nothing is recorded there: only the +tree kill on a stop applies. + ## Conformance kit `Kit.Load()` verifies the embedded copy of the kit against the pinned digests. @@ -257,16 +747,20 @@ dotnet test test/JacquardHost.Tests -f net10.0 JACQUARD_REQUIRE_LIVE=1 dotnet test test/JacquardHost.Tests -f net8.0 ``` -The live tests in `LiveTests` are ignored without `JACQUARD_BIN` and -`JACQUARD_PRELUDE` (set `JACQUARD_REQUIRE_LIVE` to make them fail instead). -The offline tests drive `test/JacquardHost.StandIn`, a console program that -replays recordings or simulates faulty workers in a fresh process. Its -descendant modes use `sh` and `sleep`, so those tests run on Unix only, and -the descriptor tests run on Linux only. The build treats warnings as errors, +The live tests (`LiveTests`, `SdkLiveTests`, and the live `ExamplesTests`) +are ignored without `JACQUARD_BIN` and `JACQUARD_PRELUDE` (set +`JACQUARD_REQUIRE_LIVE` to make them fail instead). The offline tests drive +`test/JacquardHost.StandIn`, a console program that replays recordings or +simulates faulty workers in a fresh process, and a fake Core (a shell script) +for the SDK's preparation. Their descendant modes use `sh` and `sleep`, so +those tests run on Unix only, and the descriptor, Core command, and +preparation tests run on Linux only. The build treats warnings as errors, with nullable reference types and the `8.0-recommended` analyzers. `src/JacquardHost/Compat.cs` is generated, and `src/JacquardHost/kit/host-protocol-v0/` is a byte copy of the pinned kit. `scripts/core_pin.py` maintains both; do not edit them by hand. `NOTICE` must -stay identical to the repository's `NOTICE`, and the README example must stay -identical to `test/JacquardHost.Tests/Readme/Echo.cs`; tests check both. +stay identical to the repository's `NOTICE`, and the README's examples must +stay identical to `test/JacquardHost.Tests/Readme/` (`Shipping.cs`, +`Incidents.cs`, `Documents.cs`, and `Echo.cs`, in that order), which the live +tests run; tests check both. diff --git a/dotnet/src/JacquardHost/Sdk/Codec.cs b/dotnet/src/JacquardHost/Sdk/Codec.cs index 5330181..face148 100644 --- a/dotnet/src/JacquardHost/Sdk/Codec.cs +++ b/dotnet/src/JacquardHost/Sdk/Codec.cs @@ -103,8 +103,8 @@ internal Value Encode(Encoder encoder, TRecord record) /// The codec factories: built-in types, lists, options, tuples, records, /// enumerations, unions, lazy (recursive) codecs, raw protocol values, and /// . Records are mapped by explicit field readers and -/// a constructor function rather than by reflection, so mapping works under -/// trimming and ahead-of-time compilation. +/// a constructor function rather than by reflection, so the library stays +/// clean under the trimming and ahead-of-time analyzers. /// public static class Codecs { diff --git a/dotnet/test/JacquardHost.Tests/DocumentTests.cs b/dotnet/test/JacquardHost.Tests/DocumentTests.cs index d860cf0..eaaa3f5 100644 --- a/dotnet/test/JacquardHost.Tests/DocumentTests.cs +++ b/dotnet/test/JacquardHost.Tests/DocumentTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; @@ -15,18 +16,26 @@ private static string DotnetRoot() => typeof(DocumentTests).Assembly.GetCustomAttributes().Single(attribute => attribute.Key == "DotnetRoot").Value!; [Test] - public void TheReadmeExampleIsTheCompiledOne() + public void TheReadmeExamplesAreTheCompiledOnes() { - // The README's example is Readme/Echo.cs, which a live test runs - // against Core, minus its namespace line. + // The README's examples are the files below under Readme/, in this + // order, minus their namespace lines; the live tests run them. + string[] sources = { "Shipping.cs", "Incidents.cs", "Documents.cs", "Echo.cs" }; string readme = File.ReadAllText(Path.Combine(DotnetRoot(), "README.md")); - int start = readme.IndexOf("```csharp\n", StringComparison.Ordinal); - Assert.That(start, Is.GreaterThanOrEqualTo(0), "README.md has no csharp example"); - int end = readme.IndexOf("```", start + 10, StringComparison.Ordinal); - string example = readme[(start + 10)..end]; - string source = File.ReadAllText(Path.Combine(DotnetRoot(), "test", "JacquardHost.Tests", "Readme", "Echo.cs")); - string body = source[(source.IndexOf('\n', source.IndexOf("namespace ", StringComparison.Ordinal)) + 1)..].TrimStart(); - Assert.That(example, Is.EqualTo(body), "README.md's example differs from test/JacquardHost.Tests/Readme/Echo.cs"); + var examples = new List(); + for (int start = readme.IndexOf("```csharp\n", StringComparison.Ordinal); start >= 0; start = readme.IndexOf("```csharp\n", start + 10, StringComparison.Ordinal)) + { + int end = readme.IndexOf("```", start + 10, StringComparison.Ordinal); + examples.Add(readme[(start + 10)..end]); + } + + Assert.That(examples, Has.Count.EqualTo(sources.Length), "README.md's csharp examples"); + for (int i = 0; i < sources.Length; i++) + { + string source = File.ReadAllText(Path.Combine(DotnetRoot(), "test", "JacquardHost.Tests", "Readme", sources[i])); + string body = source[(source.IndexOf('\n', source.IndexOf("namespace ", StringComparison.Ordinal)) + 1)..].TrimStart(); + Assert.That(examples[i], Is.EqualTo(body), "README.md's example " + (i + 1) + " differs from test/JacquardHost.Tests/Readme/" + sources[i]); + } } [Test] From 4f6348a85a14bd7f84dde50d9398deb73ed07ea1 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 07:26:34 -0400 Subject: [PATCH 08/52] Document the .NET SDK and run it in CI Update the root and examples READMEs, CONTRIBUTING, LIMITS (.NET SDK rows replace the protocol-level-only row), EVIDENCE, and the changelog. The live CI job now runs the .NET SDK's live tests with the adapter's, and the four example projects, with duplicate fulfillment and explicit recovery, through the .NET command. The source archive prunes the example command's build output, and the distribution check looks for the SDK and command. --- .github/workflows/ci.yml | 18 +++++++++++++++++- CHANGELOG.md | 19 +++++++++++++++++-- CONTRIBUTING.md | 29 +++++++++++++++++++---------- MANIFEST.in | 2 ++ README.md | 22 ++++++++++++---------- docs/EVIDENCE.md | 22 +++++++++++++++++++++- docs/LIMITS.md | 10 ++++++++-- examples/README.md | 7 ++++--- scripts/check_distribution.py | 2 +- 9 files changed, 101 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 732b828..c3d9024 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -213,10 +213,26 @@ jobs: dotnet-version: | 8.0.x 10.0.x - - name: Run independent .NET adapter against Core + - name: Run independent .NET adapter and SDK against Core run: | dotnet test test/JacquardHost.Tests -f net8.0 dotnet test test/JacquardHost.Tests -f net10.0 working-directory: dotnet env: JACQUARD_REQUIRE_LIVE: "1" + - name: Run the four example projects through the .NET SDK + working-directory: dotnet + run: | + set -euo pipefail + dotnet build examples/JacquardExamples + run() { dotnet examples/JacquardExamples/bin/Debug/net8.0/JacquardExamples.dll --workdir "$RUNNER_TEMP/dotnet-examples" "$@"; } + run quickstart + run planner | grep -q '"score": 16' + run triage | grep -q '"urgency": "WATCH"' + run fulfillment | grep -q '"$tag": "Fulfilled"' + run fulfillment | grep -q '"$tag": "DuplicateOrder"' + status=0 + run --workdir "$RUNNER_TEMP/dotnet-recovery" --fault publish-failed fulfillment || status=$? + test "$status" = 1 + run --workdir "$RUNNER_TEMP/dotnet-recovery" --deliver-pending fulfillment | grep -q '{"delivered":1}' + run --workdir "$RUNNER_TEMP/dotnet-recovery" --deliver-pending fulfillment | grep -q '{"delivered":0}' diff --git a/CHANGELOG.md b/CHANGELOG.md index 3391481..5d5079d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,8 +78,7 @@ codecs that parse integers exactly from their text, the session state machine, a bounded process carrier for `jacquard host worker`, and a `Host` with a closed operation registry. It carries the pinned kit as embedded - resources and replays all 21 recordings. It is protocol-level only, with no - SDK layer. .NET cannot start the worker in its own process group or + resources and replays all 21 recordings. .NET cannot start the worker in its own process group or interrupt a pipe read or write, so the carrier never calls one that can block: it waits on the C library's `poll(2)` and runs workers on Linux and macOS only. It kills the worker's process tree with `Process.Kill(true)`, @@ -88,6 +87,22 @@ `scripts/core_pin.py` now generates `dotnet/src/JacquardHost/Compat.cs` and keeps the .NET kit copy in sync, and CI runs the offline tests on .NET 8 and 10 and the live tests against the pinned Core. +- .NET SDK (`JacquardLang.Host.Sdk`): prepares and caches checked programs + with Core's public commands, maps .NET values through explicit `Codec` + objects (records through field readers and a constructor function, + enumerations, type hierarchies, `Option`, `Unit`, lazy codecs for + recursive types, and `Dynamic` for untyped code) without reflection, binds + handlers to exact operation identities with the `HandlerContext` and its + `CancellationToken`, checks arguments against the checked signature before a + worker starts, and keeps each invocation's raw evidence. `MapBatches` runs + explicit batches. There is no `decimal` mapping: money crosses as integer + minor units. Core's commands run on the caller's thread through `poll(2)`, + bounded in time and output, and on Linux through `setsid`, whose process + group is killed when a command exits or is stopped. The cache is owner-only + and verified, every binding derived again from Core's cached output on load. + The four example projects run through a `JacquardExamples` command in + `dotnet/examples/`, and CI runs them and the SDK's live tests against the + pinned Core. ## 0.2.0 — 2026-10-25 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ce606f1..adc2861 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -146,12 +146,15 @@ fails if they differ). ## .NET -The independent .NET adapter lives in `dotnet/` as package `JacquardLang.Host` -(`dotnet/src/JacquardHost`). It targets .NET 8, has no runtime dependencies -beyond the .NET base class library, and does not call Python, Node.js, Go, -Rust, or Java; it calls the C library's `poll(2)` and runs workers on Linux -and macOS only. NUnit, its adapter, and the test SDK are its only (test) -dependencies. The .NET 8 or 10 SDK is required. +The independent .NET adapter and SDK live in `dotnet/` as package +`JacquardLang.Host` (`dotnet/src/JacquardHost`, the SDK in its `Sdk/` +directory). It targets .NET 8, has no runtime dependencies beyond the .NET base +class library, and does not call Python, Node.js, Go, Rust, or Java; it calls +the C library (`poll(2)`, and for the SDK `kill(2)`, `geteuid(2)`, `mkdir(2)`, +and `statx(2)` or `stat(2)`) and runs on Linux and macOS only. On Linux the +SDK runs Core's commands through util-linux `setsid` when one is installed. +NUnit, its adapter, and the test SDK are its only (test) dependencies. The +.NET 8 or 10 SDK is required. ```sh cd dotnet @@ -159,17 +162,23 @@ dotnet test test/JacquardHost.Tests -f net8.0 dotnet test test/JacquardHost.Tests -f net10.0 # with a .NET 10 SDK # With the pinned Core configured (eval "$(python3 ../scripts/core_pin.py install)"): JACQUARD_REQUIRE_LIVE=1 dotnet test test/JacquardHost.Tests -f net8.0 +dotnet build examples/JacquardExamples +dotnet examples/JacquardExamples/bin/Debug/net8.0/JacquardExamples.dll quickstart ``` -CI runs the offline tests on .NET 8 and 10, and the live tests on both in the -shared pinned-Core job. The build treats warnings as errors, with nullable +CI runs the offline tests on .NET 8 and 10, and the live tests on both and the +four example projects in the shared pinned-Core job. The build treats warnings as errors, with nullable reference types and the `8.0-recommended` analyzers. `dotnet/src/JacquardHost/Compat.cs` is generated and `dotnet/src/JacquardHost/kit/` is a byte copy of the pinned kit. `scripts/core_pin.py` maintains both; do not edit them by hand. The `JacquardHost.StandIn` program under `dotnet/test/` is a fixture for the -offline tests. Copy the repository `NOTICE` to `dotnet/NOTICE` when it changes -(a test fails if they differ). +offline tests. The `JacquardExamples` command under `dotnet/examples/` reads the +rules and inputs from `examples/` directly, so there is no copy to keep in +sync. The README's C# examples are compiled test sources under +`dotnet/test/JacquardHost.Tests/Readme/` (a test fails if they differ). Copy +the repository `NOTICE` to `dotnet/NOTICE` when it changes (a test fails if +they differ). ## Releasing diff --git a/MANIFEST.in b/MANIFEST.in index 3808d03..19294ab 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -25,3 +25,5 @@ prune dotnet/test/JacquardHost.Tests/bin prune dotnet/test/JacquardHost.Tests/obj prune dotnet/test/JacquardHost.StandIn/bin prune dotnet/test/JacquardHost.StandIn/obj +prune dotnet/examples/JacquardExamples/bin +prune dotnet/examples/JacquardExamples/obj diff --git a/README.md b/README.md index c72341d..6e6a54d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,22 @@ # Jacquard Host -Call checked Jacquard functions from Python, TypeScript, Go, Rust, or Java and implement +Call checked Jacquard functions from Python, TypeScript, Go, Rust, Java, or .NET and implement their declared host operations with ordinary services. Jacquard owns the decision logic; the host owns persistence, I/O, and application lifetimes. The SDK prepares and caches checked programs, converts native Python values, binds services to exact operation identities, and retains the raw evidence for -every invocation. Python 3.11+, Node.js 22+, Go 1.24+, Rust 1.88+, or Java 21+, -with no third-party runtime dependencies. The [TypeScript SDK](typescript/README.md), -the [Go SDK](go/README.md), the [Rust SDK](rust/README.md), and the -[Java SDK](java/README.md) implement the pinned protocol independently. The +every invocation. Python 3.11+, Node.js 22+, Go 1.24+, Rust 1.88+, Java 21+, or +.NET 8+, with no third-party runtime dependencies. The +[TypeScript SDK](typescript/README.md), the [Go SDK](go/README.md), the +[Rust SDK](rust/README.md), the [Java SDK](java/README.md), and the +[.NET SDK](dotnet/README.md) implement the pinned protocol independently. The TypeScript SDK supports async service methods, the Go SDK maps native Go types and context cancellation, the Rust SDK maps native Rust types through `record!` -and `variants!` declarations, and the Java SDK maps records, enums, and sealed -interfaces through explicit codecs. A protocol-level [.NET](dotnet/README.md) -(.NET 8+) adapter also implements the pinned protocol independently. It has a -closed operation registry and kit replay, but no SDK layer yet. +and `variants!` declarations, the Java SDK maps records, enums, and sealed +interfaces through explicit codecs, and the .NET SDK maps records, enumerations, +and type hierarchies through explicit codecs without reflection, with +`CancellationToken` cancellation. **Experimental:** the SDK targets Core's pinned `jacquard-host-v0` protocol. Core is installed separately. Each invocation runs one fresh, serial worker; @@ -25,7 +26,8 @@ the host and operating system are trusted. See [limits](docs/LIMITS.md). For Node.js, follow the [TypeScript walkthrough](typescript/README.md); for Go, the [Go walkthrough](go/README.md); for Rust, the [Rust walkthrough](rust/README.md); -for Java, the [Java walkthrough](java/README.md). For Python: +for Java, the [Java walkthrough](java/README.md); for .NET, the +[.NET walkthrough](dotnet/README.md). For Python: From a clone of this repository: diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md index a7f73fd..ef76cc1 100644 --- a/docs/EVIDENCE.md +++ b/docs/EVIDENCE.md @@ -143,6 +143,26 @@ dependencies. | Pin consistency | `dotnet/src/JacquardHost/Compat.cs` is generated, and the embedded kit copy matches the pinned digests and the Python copy (`scripts/core_pin.py check`); `tests/test_core_pin.py` covers drift in both, and the offline Python suite passes | | Distribution | `scripts/check_distribution.py` passes; the source archive carries the .NET sources, `dotnet/NOTICE`, and the kit copy | +## .NET SDK and projects (2026-10-29) + +Recorded against the same `jacquard-core-0.3.0` release asset (linux-x86_64), +on Linux 7.0 with the .NET 8.0.427 and 10.0.403 SDKs (runtimes 8.0.33 and +10.0.14). The library still has no runtime dependencies beyond the base class +library; the SDK calls the C library's `kill(2)`, `geteuid(2)`, `mkdir(2)`, +and `statx(2)` (`stat(2)` on macOS, compiled but not run), and on Linux runs +Core's commands through util-linux `setsid`, here 2.41.3. The test +dependencies are unchanged. + +| Lane | Result | +|---|---| +| .NET SDK offline (`dotnet test test/JacquardHost.Tests -f net8.0` with the .NET 8 SDK pinned as the CI 8.0 job pins it, and `-f net10.0`, without Core) | 181 tests pass on each runtime and 28 live tests are ignored: the adapter's 96 offline tests and 85 SDK ones. Core commands (17, Linux): exit status, output, and added environment, with stdin at end of file; output over its limits read and counted but not kept; a timeout and a cancellation each kill the command and its child; under `setsid` the command keeps its pid and its group and session ids equal it (with `setsid --fork` the test fails); `sh -c 'sleep 300 & echo $!'` succeeds and its child dies, and survives when the exit kill is disabled; a child that closed its output is killed too; a reaped leader's pid held by an unrelated group leader is not killed; a process outside the group holding the command's stdout is neither killed nor waited for; a stop kills a child and a grandchild; without groups a recorded descendant is killed at the exit (and survives with recording off) and a stop kills the tree; a cancellation during the exit cleanup is reported; a missing executable, and one `setsid` cannot exec, are start failures; an escaped flooder holds neither the caller nor more than the kept output; 40 commands leave the descriptor count unchanged, counted at once. Preparation against a fake Core (30, Linux): option and path validation (empty, NUL, and over-long paths refused with excerpts before the file system sees them, a relative path measured absolute); one version check across four threads and two runtimes; prelude discovery once per Core and prelude; output over 8 MiB; the 30-second bound (shortened by a seam) and cancellation killing the command's child; a waiting preparation observing its own cancellation; FIFO and directory sources refused without blocking; sources over 16 MiB refused from their sizes before reading; cancellation between sources, during source validation, during fingerprinting (one 64 KiB piece), between empty prelude files, during the failure cleanup, and as a warm load finishes; prelude walks over 256 MiB, with a FIFO, over the entry cap (lowered by a seam), and 1,500 directories deep; a changed `program.json` binding, a changed Core output, a rewritten digest, and a `program.json` replaced by a link refused; group-writable roots, entries, files, and directories refused and new entries owner-only; a foreign-owned root refused; an executable over 512 MiB refused from its size; source lists over 1,024 refused from their count with no element read; the cleanup stopping at the cap; a symbolic link at the cache root followed to its directory; directories the preparation did not make, and one made at the vacated staging path, left alone. Data (23): signatures, hash groups, and display names; round trips and type conformance; the full Int range, Real bits, and `int` narrowing; records, enumerations, unions, nested `Option`, nullable codecs, tuples, custom codecs, and `Dynamic`; no conversion from `decimal`; recursive types through a lazy codec made once across eight threads, a chain of a million lazy codecs, and one standing for itself refused; the node, text, frame, depth, and count bounds (a list whose items throw is refused from its count, tuples and records before an item is read, a constructor of 20 million fields before it is copied); a raw value 100,000 levels deep refused by its depth; a `Dynamic` over the hard limit refused before it is copied, an endless enumeration after one item past the limit; deep `Dynamic` equality and text without recursion; 64-char excerpts and the 8,192-char message cap, caller exceptions kept as inner exceptions; 100,000 layers of nested unions flattened to one case and round-tripped; a recursive union through a lazy case from eight threads; signature depth 64 accepted, 65 and 5,000 parentheses refused, 100,000 refused by the token cap. Batches and operations (13): argument counts refused before encoding; a throwing codec an invalid argument; cancellation before, during, and after batch validation, each an exact encode count with no worker; every batch validated and the first encoded again; no field or return value that could keep an encoding; a shared graph from a custom codec measured before its type is checked; handlers' decoding, results, failures, context, and token, and no handler starting once the call stopped during decoding. Examples (2): usage errors exit 2; the store removes only the temporary file it created | +| .NET SDK live (`JACQUARD_REQUIRE_LIVE=1`, `JACQUARD_BIN` and `JACQUARD_PRELUDE` set) | 20 tests pass on .NET 8 and on .NET 10: quickstart calls, `ToDynamic`, `Render`, and an owner-only saved inspection; the README examples; the 64-row triage oracle in batches of 8, with a codec failing on the second batch keeping the first and its exception; local refusals before any worker (a field over the Int range named by path, an over-deep call, a wrong type, a wrong count); missing handlers, a misspelled operation, and a 100-name service refused at binding with no handler run; unknown completion once with raw evidence; no handler starting once a real invocation stopped during decoding; a handler's token cancelled when the caller cancels; the 25-case planner oracle; cache reuse without Core, a byte-exact new entry, and corruption refused; a 2,500-`Option` signature left unbound with `safe` working cold and warm; a redirected name in `program.json` refused; the full Int range, Real bits, and nested options; spawn failure and cancellation; output overflow at batch 0; unit results kept after a later batch fails (E1602); too many bindings refused from the count; a throwing result codec keeping the result and its exception, a null mapping message, and an over-long save path; prelude contents keying the cache, and two concurrent preparations publishing one entry | +| .NET projects live | 3 further tests pass on both runtimes: all four `JacquardExamples` commands, including duplicate fulfillment, `--inspect` (owner-only), explicit delivery, a failed delivery printing no count, and usage errors; the command in a fresh process exiting 1 with E1613; and the three fulfillment faults recovered explicitly across restart, each receipt delivered once. The CI step's four projects and recovery commands also pass locally | +| Totals | 209 tests on each runtime with Core required, none ignored; 181 pass and 28 are ignored without Core | +| Regressions checked by mutation (.NET 8) | Each guard was removed in turn and the named tests run; all 42 mutations failed them: the command's exit group kill, its stop kill, `setsid` without forking, the pid-reuse check, the recorded-descendant kill, the stop check after the exit cleanup, and closing the output streams; the source kind check (the FIFO test failed after its one-minute wait), source count, source validation stop, source ceiling, tree entry and byte caps, the executable's size check and per-piece stop, the derived-binding check, the owner, mode, and owner-only checks, the cleanup's stop and cap, leaving the vacated staging path, and the final stop check; the encoder's node budget, the decoder's measurement, the text charge and length check, the list, record, and constructor counts, measuring before the type check, counting arguments before encoding, the batch stop checks before and after validation, the handler's stop check (offline and live), the `Dynamic` limit, excerpts, the message cap, wrapping a field reader's and a record constructor's exceptions, the lazy codec's cycle check (the test then never returned and was stopped), and the store's exclusive temporary file. Separately, checking the cancellation in an exception filter, which .NET runs before the preparation's cleanup, failed the failure-cleanup test until the check moved into the handler | +| Build | `TreatWarningsAsErrors`, nullable reference types, and the `8.0-recommended` analyzers are clean with both SDKs | +| Unchanged lanes | The adapter's 96 offline and 5 live tests (the README-example check now covers all four examples), the 21-recording kit replay, `scripts/core_pin.py check`, `tests/test_core_pin.py`, and `scripts/check_distribution.py` (the source archive now carries the SDK and the example command) pass | + ## Outcome frame node budget (2026-10-25) Core's `outcome` encodes the evidence's interface types and the result value @@ -232,7 +252,7 @@ reaches persistence has request ordinal 1 and one accepted `ok` observation. | Linux, Go 1.24 and stable | CI lanes | Offline Go tests with the race detector on both; live Go tests in the shared pinned-Core job | | Linux, Rust 1.88 and stable | CI lanes | fmt, clippy, and offline Rust tests on both; live Rust tests in the shared pinned-Core job | | Linux, Java 21 and 25 | CI lanes | Offline Java adapter and SDK tests on both; live Java tests and the four Java projects on Java 21 in the shared pinned-Core job; offline and live suites also run locally on Temurin 21.0.13 and 25.0.5 | -| Linux, .NET 8 and 10 | CI lanes | Offline .NET tests on both; live .NET tests on both in the shared pinned-Core job; offline and live suites also run locally on .NET 8.0.33 and 10.0.14 | +| Linux, .NET 8 and 10 | CI lanes | Offline .NET adapter and SDK tests on both; live .NET tests on both and the four .NET projects in the shared pinned-Core job; offline and live suites also run locally on .NET 8.0.33 and 10.0.14 | | Linux, Node.js 26.7 | exercised | TypeScript offline/live suites and tarball installation exercised locally; no broader runtime or OS claim | | macOS | development only | no CI; the descriptor-count assertions are skipped without `/proc` (the socket host tests skip, the kit and adapter tests skip only the count); SQLite WAL mode is recorded, not assumed | | Windows and others | no claim | see LIMITS.md | diff --git a/docs/LIMITS.md b/docs/LIMITS.md index 469a02f..da5cee6 100644 --- a/docs/LIMITS.md +++ b/docs/LIMITS.md @@ -29,7 +29,13 @@ also updates this document. | .NET worker process tree, not group | .NET cannot start the worker in its own process group, so the .NET host kills a process tree. At expiry, cancellation, or disposal of a running worker, it calls `Process.Kill(true)`, which stops the worker, lists its children from the process table, kills it, and does the same for each child. A descendant that was already reparented keeps running: one whose parent exited, or one that daemonized. So does one started after its parent was listed. Descendants are found and signalled by pid, with nothing pinning their identity: one that exits, and is reaped once the worker is killed, between the listing and the signal can have its pid reused by an unrelated process, which is then signalled (the Java host signals process handles, and the Rust and Go hosts a process group). Listing the process table takes time proportional to the number of processes on the machine. The runtime reaps the worker as soon as it exits. A descendant still holding stdout or stderr after the exit is detected and, after a one-second grace, reported as a carrier loss, as the Go and Rust hosts report it; it is not stopped. .NET reports a signal as exit status 128 + its number, so only the host's own SIGKILL is reported as a signal. | An OS-level containment mechanism (for example a cgroup or job object) with its own evidence, or a .NET process-group API. | | .NET pipe I/O goes through `poll(2)` | .NET's pipe streams have no read or write that a deadline or a cancellation can interrupt, so the .NET host never calls one that can block: the caller's thread asks the C library's `poll(2)` whether a pipe is ready, with a poll timeout of at most 5 ms, rechecking the deadline, the `CancellationToken`, and disposal between polls, and then reads only bytes already in the pipe or writes at most `PIPE_BUF` bytes (4096 on Linux, 512 on macOS), which a writable pipe accepts at once. No thread is left blocked on a worker pipe, even while an escaped descendant holds stdin without reading or floods stdout or stderr. The `pollfd` layout and event bits are the same on Linux and macOS; elsewhere, including Windows, the host does not start a worker and reports a spawn failure. .NET opens its own descriptors close-on-exec, but passes the worker any descriptor the host process itself inherited without that flag. | A .NET API for interruptible or non-blocking process pipes, and CI on each added platform. | | .NET handler cancellation is cooperative | Each .NET handler runs on its own background thread. At the deadline or on cancellation the host kills the worker's process tree, closes its input, and sends no late response. `HandlerContext.IsCancelled` turns true (it reads the deadline exactly), and stays true once the invocation ends. `HandlerContext.CancellationToken` is cancelled when the call stops or ends, never earlier (a lifetime of any length does not cancel it before its deadline). While a handler runs, the caller's thread rechecks the deadline and the caller's token between poll waits of at most 5 ms; that is the requested poll timeout, not a bound on the interval between checks, which reading pending stderr (up to 1 MiB per pass) and thread scheduling can lengthen. Whichever check sees the stop, the token's cancellation starts before the worker is killed; otherwise it starts shortly after the invocation ends. The cancellation runs on a short-lived thread the host starts for it, one per invocation that gave a handler a token, so it does not wait for the shared thread pool. If no thread can be created or started, the host queues it as a pool work item, which runs only when a pool thread is free, so a starved pool delays it. If the work item cannot be queued either, the host gives up: the token is never cancelled (`HandlerContext.IsCancelled` still turns true), `InvocationResult.HandlerTokenNotCancelled` is set, and a handler waiting only on the token keeps waiting; `Invoke` still returns its result and disposes the worker. Callbacks a handler registers on that token run on that thread, which the host never waits for; a callback that blocks holds that thread, and anything a callback throws is discarded. A callback registered after the token is cancelled runs at once on the handler's own thread and throws there, as with any `CancellationToken`. A handler that ignores both keeps running on its thread until it returns. No outside action is stopped or rolled back. A stack overflow in a handler ends the process, as it does any .NET code. | A separately enforced operation lifetime with explicit completion and reconciliation evidence. | -| .NET adapter is protocol-level only | .NET callers pass exact identities, boundary types, and protocol values to a prepared store; there is no .NET SDK for program preparation, native type mapping, or caching, and `Invoke` is synchronous. Linux runs in CI on .NET 8 and 10. macOS has no CI; the offline descendant tests need `sh`, `sleep`, and `yes`, and the descriptor tests need `/proc`. Windows is not supported. | A .NET SDK with typed-binding and preparation evidence comparable to the other SDKs, and CI on each added platform. | +| .NET SDK Core command descendants | Each Core `--version`, `check`, `hash`, and `run` command must exit within 30 seconds and keeps at most 8 MiB of stdout and 4 KiB of stderr; cancelling the token stops it. Its output is read only after `poll(2)` reports it ready, on the caller's thread. On Linux with a `setsid` at `/usr/bin/setsid` or `/bin/setsid` (util-linux's, or a compatible one such as BusyBox's; any executable there is used), the command runs as `setsid -- `, which execs without forking, so Core keeps its pid and leads its own process group; when `setsid` cannot exec Core (exit status 126 or 127 with its own `setsid: ` message) the command is reported as a start failure. When the command exits, and on a timeout or cancellation, the host sends `kill(-pgid, SIGKILL)` to that group. Once .NET has reaped Core, a live process holding Core's pid means the id was reused and nothing is killed; a process that took Core's pid after a full pid wraparound, made its own group, and forked between that check and the kill could still have that group killed, the same window as the Rust SDK's `kill(-pgid)`. A descendant that leaves the group (calls `setsid`) is neither detected nor killed; the host does not wait for the end of the output after the exit, and closes its ends, so that descendant's next write fails. Without `setsid` (Linux without one at those paths), descendants are recorded from `/proc` about every 10 ms by pid and start time, every recorded descendant still alive with that start time is killed when the command returns (the start time is read just before the kill, a short window in which a reused pid could be signalled, as with the JDK's process handles), and a stop also calls `Process.Kill(true)`; that fallback misses fast fork-and-exit chains. macOS has no `/proc`, so there only the tree kill on a stop applies. The .NET worker does not run in its own group (see the worker row above). | An OS-level containment mechanism, or a .NET process-group API, with its own evidence. | +| .NET SDK cache and source reads | A cache entry is verified against the file digests in its `program.json`, and every binding is derived again from Core's cached raw output, so an accidental or partial change is refused. No record inside the cache resists someone who can rewrite it: an output, its recorded digest, and the bindings changed together load. So the cache's owner is trusted and only the owner may write it: preparation refuses a cache root, an entry, or any file or directory in one that the current user does not own or that its group or others may write (read with `statx(2)` on Linux and `stat(2)` on macOS, without following links), on create and on load, and new entries are owner-only. A symbolic link at the cache root is followed once and the directory it names checked; nothing inside it is followed, and the cache root's parent directories are not checked. The macOS status calls are compiled but not exercised by any test. A failed preparation's cleanup removes at most 100,000 entries of its staging directory, created exclusively, and ends at a cancellation, so a stopped preparation leaves an unpublished `prepare-*` directory in the cache root; it is never loaded, and nothing removes it later (a name and an age cannot show that the SDK made a directory or that no preparation is using it), so remove leftovers yourself when no preparation is running. At most 1,024 sources are taken, refused from the list's count before any element is read, and they must be regular files totalling at most 16 MiB, the Core executable at most 512 MiB, and the prelude and cache walks read at most 100,000 entries and 256 MiB of regular files under one root, each file checked from its size before it is read and again while it is read; a source or cache file replaced by a FIFO (or, in the cache, by a link) between that check and its open can still block or redirect the open, which a cancellation does not end. | An open that cannot block on a FIFO or follow a link. | +| .NET SDK cancellation is by token only | Preparation, Core's commands, `Call`, `Invoke`, and `MapBatches` observe the `CancellationToken` they take. `Thread.Interrupt` is not a stop signal: the SDK's own waits are `poll(2)` sleeps, which neither end at an interrupt nor consume it. The API is synchronous; there are no `Task`-returning variants. | `Task`-returning variants with the same bounds, if callers need them. | +| .NET SDK signature depth | A checked function whose signature nests types deeper than 64 levels (`max_json_depth`, the protocol's value-depth limit), names more than 4,096 types, or has more than 16,384 tokens is left unbound, like a polymorphic one: preparation succeeds, the other functions bind, and binding it fails with a `Binding` exception naming the limit. The SDK parses no operation signatures (Core's `check --print-sigs` prints none). | None needed for depth, since a deeper value cannot cross the protocol; Core's structured host interface exporter for the rest. | +| .NET SDK record fields are declared, not exported | A record codec's fields, in the order given, are a constructor's fields; the pinned CLI exports no field types, so a wrong order is not detected locally, and Core refuses it only where the swapped fields' types differ. | Core's structured host interface exporter, so bindings can be generated or checked. | +| .NET fulfillment is a single-process snapshot example | Atomic JSON replacement demonstrates booking/outbox recovery and deduplication, without cross-process locking or power-loss durability evidence. | A concrete production storage contract and crash/concurrent-writer evidence. | +| .NET platforms | Linux runs in CI on .NET 8 and 10. macOS has no CI; the offline descendant tests need `sh`, `sleep`, and `yes`, and the descriptor, Core command, and preparation tests need `/proc` and run on Linux only. Windows is not supported: the worker reports a spawn failure and `Prepare` refuses. | CI, lifecycle, and cleanup evidence on each added platform. | | Node handler cancellation is cooperative | The Node adapter can expire while awaiting a Promise, signal the handler, and kill the worker; it cannot stop an outside action or preempt JavaScript that blocks the event loop. No late response or automatic retry follows expiration. | A separately enforced operation lifetime with explicit completion and reconciliation evidence. | | Socket host closes without a response on its own failures | A ceiling violation, early disconnect, or failed invocation ends the connection silently; clients see a reset, not an HTTP error. | An owner decision that the host may emit fixed minimal status lines, recorded here with the exact bytes and the rule that they carry no application meaning. | | Request bytes cross as Latin-1 `Text` | v0 has no bytes value; each byte is one codepoint 0..255 and a response codepoint above 255 is refused by the host. | A frozen bytes value in a later protocol version. | @@ -41,7 +47,7 @@ also updates this document. | No outbound network in the first application | The synthetic trial cannot call APIs, analytics, or email services. | A concrete application need and reviewed host/port authority with refusal evidence. | | Host-owned SQLite through one named operation | Jacquard receives neither raw SQL nor generic filesystem authority; `study.save-response` is given a synthetic session token by application contract (the host cannot verify what a token denotes), and its receipt is a content hash with no timestamp. | Repeated concrete operations justify a reusable storage contract with migration, recovery, and authority evidence. | | No automatic side-effect retry | An ambiguous completion fails rather than risking duplicate external work. | Operation-specific idempotency keys, authenticated receipts, and crash/recovery evidence. | -| SDK uses explicit application data mappings | The pinned CLI does not export structured constructor layouts. Python dataclasses, TypeScript record codecs, Go records, and Java record components must follow Jacquard positional field order; ambiguous names across declaration groups are refused. Core validates wire types. | Core's versioned host-format interface exporter and independent typed-binding conformance. | +| SDK uses explicit application data mappings | The pinned CLI does not export structured constructor layouts. Python dataclasses, TypeScript record codecs, Go records, Java record components, and .NET record codecs' fields must follow Jacquard positional field order; ambiguous names across declaration groups are refused. Core validates wire types. | Core's versioned host-format interface exporter and independent typed-binding conformance. | | Go fulfillment is a single-process snapshot example | Atomic JSON replacement demonstrates booking/outbox recovery and deduplication, without cross-process locking or power-loss durability evidence. | A concrete production storage contract and crash/concurrent-writer evidence. | | JavaScript numbers have a smaller exact integer range | Native safe integers and the TypeScript `int` codec use numbers; full 63-bit Int values require bigint (or the explicit bigint codec). Unsafe numeric input is refused. | An explicit application choice of bigint; no silent precision loss is allowed. | | TypeScript fulfillment is a single-process snapshot example | Atomic JSON replacement demonstrates booking/outbox recovery and deduplication, without cross-process locking or power-loss durability evidence. It is not the Python SQLite implementation. | A concrete production storage contract and crash/concurrent-writer evidence. | diff --git a/examples/README.md b/examples/README.md index f52318a..6eb0351 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,13 +1,14 @@ # Example projects -The same Jacquard rules run through all five SDKs. The commands below use +The same Jacquard rules run through all six SDKs. The commands below use Python; the [TypeScript walkthrough](../typescript/README.md#run-the-projects) includes compiled Node.js front ends for all four projects, the [Go walkthrough](../go/README.md#run-the-projects) a Go command with the rules and inputs embedded, the [Rust walkthrough](../rust/README.md#run-the-projects) a Cargo example that reads them from this directory, and the -[Java walkthrough](../java/README.md#run-the-projects) a Java command that reads -them from this directory too. Each includes an +[Java walkthrough](../java/README.md#run-the-projects) a Java command and the +[.NET walkthrough](../dotnet/README.md#run-the-projects) a .NET command that +read them from this directory too. Each includes an independent fulfillment recovery example. Start with [setup](../docs/SETUP.md), then run these from the repository root. diff --git a/scripts/check_distribution.py b/scripts/check_distribution.py index 15d963c..2f6b2ca 100644 --- a/scripts/check_distribution.py +++ b/scripts/check_distribution.py @@ -36,7 +36,7 @@ def main() -> None: # Only our freshly built archive is extracted, never an external input. archive.extractall(root / "source", filter="data") source = next((root / "source").iterdir()) - for relative in ("LICENSE", "NOTICE", "go/NOTICE", "rust/NOTICE", "java/NOTICE", "java/pom.xml", "dotnet/NOTICE", "dotnet/src/JacquardHost/JacquardHost.csproj", "dotnet/src/JacquardHost/Compat.cs", "dotnet/src/JacquardHost/kit/host-protocol-v0/kit/transcripts.json", "CONTRIBUTING.md", "CHANGELOG.md", "docs/SETUP.md", "scripts/check_examples.py", + for relative in ("LICENSE", "NOTICE", "go/NOTICE", "rust/NOTICE", "java/NOTICE", "java/pom.xml", "dotnet/NOTICE", "dotnet/src/JacquardHost/JacquardHost.csproj", "dotnet/src/JacquardHost/Compat.cs", "dotnet/src/JacquardHost/Sdk/Runtime.cs", "dotnet/examples/JacquardExamples/Examples.cs", "dotnet/src/JacquardHost/kit/host-protocol-v0/kit/transcripts.json", "CONTRIBUTING.md", "CHANGELOG.md", "docs/SETUP.md", "scripts/check_examples.py", "examples/python_sdk/app.py", "examples/python_sdk/exercise.py"): assert (source / relative).is_file(), f"sdist missing {relative}" for project in ("quickstart", "triage", "planner", "fulfillment"): From 7569323830e8520b75a4b78575507456428ca857 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:01:34 -0400 Subject: [PATCH 09/52] Save an invocation through a new owner-only file renamed over the target Saving over an existing file kept that file's permissions and owner, and a symbolic link at the target was followed. The record is now written to a file created exclusively in the target's directory, made exactly 0600, and renamed over the target. --- dotnet/src/JacquardHost/Sdk/Invocation.cs | 53 +++++++++++++++++-- .../JacquardHost.Tests/Sdk/BatchesTests.cs | 34 ++++++++++++ 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Invocation.cs b/dotnet/src/JacquardHost/Sdk/Invocation.cs index 028c63b..3b2d779 100644 --- a/dotnet/src/JacquardHost/Sdk/Invocation.cs +++ b/dotnet/src/JacquardHost/Sdk/Invocation.cs @@ -81,8 +81,16 @@ public string Render() /// /// Writes the invocation's frames and host evidence. The record contains - /// argument and result values; it is written only when called. A new file - /// is created owner-only; an existing file keeps its permissions. + /// argument and result values; it is written only when called. The + /// record is written to a new file, created exclusively in the target's + /// directory, owned by this process's user and owner-only (read and write + /// for its owner) where the file system has POSIX permissions; that file + /// is then renamed over . Whatever was at the path + /// is replaced, never written through: an existing file's permissions and + /// owner are not kept, and a symbolic link there is replaced, not + /// followed. The target's directory must be writable; where it is + /// sticky (such as /tmp), a file another user owns cannot be + /// replaced and the save fails. /// /// The file cannot be written, or the path is longer than any path. /// The file may not be written. @@ -116,8 +124,45 @@ public void Save(string path) } byte[] bytes = Encoding.UTF8.GetBytes(text + "\n"); - using var stream = new FileStream(path, Core.OwnerOnly(FileMode.Create)); - stream.Write(bytes); + string full = Path.GetFullPath(path); + string directory = Path.GetDirectoryName(full) ?? throw new IOException(Identities.Excerpt(path) + " names no file"); + string temporary = Path.Combine(directory, string.Format( + CultureInfo.InvariantCulture, + ".jacquard-save-{0}-{1}", + Environment.ProcessId, + Convert.ToHexString(System.Security.Cryptography.RandomNumberGenerator.GetBytes(8)).ToLowerInvariant())); + bool renamed = false; + try + { + using (var stream = new FileStream(temporary, Core.OwnerOnly(FileMode.CreateNew))) + { + if (!OperatingSystem.IsWindows()) + { + // Exactly owner-only, whatever the umask left. + File.SetUnixFileMode(stream.SafeFileHandle, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + + stream.Write(bytes); + } + + File.Move(temporary, full, overwrite: true); + renamed = true; + } + finally + { + if (!renamed) + { + // Only the file this call created exclusively. + try + { + File.Delete(temporary); + } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + // Left behind under its distinctive name. + } + } + } } /// diff --git a/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs index 634e08e..80a5130 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs @@ -254,4 +254,38 @@ public void MapBatchesNeedsAListFunction() Assert.That(Support.Throws(() => scalar.MapBatches(Codecs.Int, Codecs.Int, Rows(3), 1)).Kind, Is.EqualTo(JacquardErrorKind.Binding)); Assert.That(Support.Throws(() => Ints().MapBatches(Codecs.Int, Codecs.Int, Rows(3), 0)).Message, Is.EqualTo("batch size must be positive")); } + + /// + /// Save writes a new owner-only file and renames it over the target: an + /// existing world-writable file becomes owner-only (0600) with the + /// record, a symbolic link at the target is replaced by the file while + /// the link's target is left unchanged, and no temporary file is left. + /// + [Test] + [Platform("Linux")] + public void SaveReplacesTheTargetWithANewOwnerOnlyFile() + { + Invocation invocation = Ints().Invoke(Argument.Of(Codecs.List(Codecs.Int), new long[] { 1 })); + Assert.That(Support.Reason(invocation.Raw), Is.EqualTo("worker_spawn_failed")); + const UnixFileMode Open = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.OtherRead | UnixFileMode.OtherWrite; + string existing = Path.Combine(root, "existing.json"); + File.WriteAllText(existing, "old"); + File.SetUnixFileMode(existing, Open); + invocation.Save(existing); + Assert.That(File.GetUnixFileMode(existing), Is.EqualTo(UnixFileMode.UserRead | UnixFileMode.UserWrite)); + Assert.That(File.ReadAllText(existing), Does.StartWith("{").And.EndWith("}\n")); + + string victim = Path.Combine(root, "victim.txt"); + File.WriteAllText(victim, "keep"); + File.SetUnixFileMode(victim, Open); + string link = Path.Combine(root, "link.json"); + File.CreateSymbolicLink(link, victim); + invocation.Save(link); + Assert.That(new FileInfo(link).LinkTarget, Is.Null, "the link was followed"); + Assert.That(File.GetUnixFileMode(link), Is.EqualTo(UnixFileMode.UserRead | UnixFileMode.UserWrite)); + Assert.That(File.ReadAllText(link), Is.EqualTo(File.ReadAllText(existing))); + Assert.That(File.ReadAllText(victim), Is.EqualTo("keep")); + Assert.That(File.GetUnixFileMode(victim), Is.EqualTo(Open)); + Assert.That(Directory.EnumerateFiles(root).Select(Path.GetFileName), Is.EquivalentTo(new[] { "existing.json", "victim.txt", "link.json" })); + } } From 2600ef5bfb9730d0ef11a013f7cfaf7a68fcca30 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:02:51 -0400 Subject: [PATCH 10/52] Refuse a cache root, Core executable, or prelude replaced during preparation The cache root was checked once and then used by its path, so another user who may write its parent could rename it away and put their own directory there. Its device and inode are now pinned at the check and compared again before the staging directory is made, before publishing, and before and after each load. The executable and every prelude file and directory were fingerprinted before Core ran them by path, so a replacement could make artifacts from other bytes under the old fingerprint. Their identities (device, inode, size, mode, owner, modification and status-change times) are now read as they are fingerprinted and must be unchanged once Core's commands finish: inside the version check and prelude discovery, before a cold entry is published, and before a warm entry is loaded. The version check is keyed on the executable's digest and identity, and a run during which anything changed is not kept. --- dotnet/src/JacquardHost/Sdk/Core.cs | 78 +++++++++- dotnet/src/JacquardHost/Sdk/Runtime.cs | 86 +++++++++-- dotnet/src/JacquardHost/Sdk/Unix.cs | 46 +++++- .../JacquardHost.Tests/Sdk/RuntimeTests.cs | 145 ++++++++++++++++++ 4 files changed, 328 insertions(+), 27 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Core.cs b/dotnet/src/JacquardHost/Sdk/Core.cs index fcaa70f..e267ace 100644 --- a/dotnet/src/JacquardHost/Sdk/Core.cs +++ b/dotnet/src/JacquardHost/Sdk/Core.cs @@ -127,10 +127,12 @@ internal static string Run(string binary, string prelude, TimeSpan timeout, Canc /// The SHA-256 digest of the Core executable's bytes, read in pieces. A /// file larger than is refused from its size /// before it is read, and the limit is enforced again while it is read. - /// Before each piece it observes the token. + /// Before each piece it observes the token. + /// is the file's status before the read, which must be unchanged after + /// it, so the digest describes the file with that identity. /// /// A preparation failure. - internal static string FileDigest(string path, CancellationToken cancellationToken) + internal static string FileDigest(string path, CancellationToken cancellationToken, out Unix.Status identity) { JacquardException over = JacquardException.Preparation("Core executable " + path + " exceeds " + (ExecutableLimit >> 20).ToString(CultureInfo.InvariantCulture) + " MiB"); @@ -179,9 +181,64 @@ internal static string FileDigest(string path, CancellationToken cancellationTok throw JacquardException.Preparation("cannot read " + path + ": " + error.Message); } + if (!StatOrFail(path, follow: true, "cannot read ").SameFile(status)) + { + throw Changed("Core executable", path); + } + + identity = status; return Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); } + /// One file or directory whose identity a preparation keeps until Core's commands have finished. + internal readonly record struct Pin(string What, string Path, bool Follow, Unix.Status Status); + + /// The identity of as text, for a key: device, inode, size, and both times. + internal static string IdentityKey(Unix.Status status) => string.Join( + ":", + status.Device.ToString(CultureInfo.InvariantCulture), + status.Inode.ToString(CultureInfo.InvariantCulture), + status.Size.ToString(CultureInfo.InvariantCulture), + status.Modified.ToString(CultureInfo.InvariantCulture), + status.Changed.ToString(CultureInfo.InvariantCulture)); + + /// + /// Refuses the preparation unless every pinned file and directory still + /// has the identity it had when it was fingerprinted (see + /// ). It reads one status per pin, at + /// most plus one, and observes the token before + /// each. + /// + /// A preparation failure. + internal static void Unchanged(IReadOnlyList pins, CancellationToken cancellationToken) + { + foreach (Pin pin in pins) + { + if (cancellationToken.IsCancellationRequested) + { + throw JacquardException.Preparation(Aborted); + } + + Unix.Status now; + try + { + now = Unix.Stat(pin.Path, pin.Follow); + } + catch (IOException) + { + throw Changed(pin.What, pin.Path); + } + + if (!now.SameFile(pin.Status)) + { + throw Changed(pin.What, pin.Path); + } + } + } + + private static JacquardException Changed(string what, string path) => + JacquardException.Preparation(what + " " + path + " changed during preparation; prepare again once it no longer changes"); + /// Options that write a file, created owner-only (read and write for its owner) where the file system has POSIX permissions. internal static FileStreamOptions OwnerOnly(FileMode mode) { @@ -229,13 +286,16 @@ internal static Unix.Status StatOrFail(string path, bool follow, string context) /// the entry cap is held and no directory depth can exhaust the call /// stack; before each entry and each piece of a file it observes the /// token. says whether entries are also checked - /// for owner-only access, or made owner-only. + /// for owner-only access, or made owner-only. When + /// is given, the root's status and each entry's + /// status, read before the entry is digested, are added to it. /// /// A preparation failure. - internal static SortedDictionary Tree(string root, Access access, CancellationToken cancellationToken) + internal static SortedDictionary Tree(string root, Access access, CancellationToken cancellationToken, List? pins = null) { var files = new SortedDictionary(Json.CodePointOrder); Check(root, access); + pins?.Add(new Pin("prelude", root, Follow: true, StatOrFail(root, follow: true, "cannot read "))); long total = 0; int entries = 0; int cap = TreeEntryCap; @@ -265,6 +325,7 @@ internal static SortedDictionary Tree(string root, Access access string relative = Path.GetRelativePath(root, path).Replace(Path.DirectorySeparatorChar, '/'); Unix.Status status = StatOrFail(path, follow: false, "cannot read "); + pins?.Add(new Pin("prelude entry", path, Follow: false, status)); if (status.Kind == Unix.Kind.Directory) { Check(path, access); @@ -464,12 +525,13 @@ private static bool IsExecutable(string path) /// other permission. Nothing inside a cache entry can resist someone who /// may rewrite it, so the guarantee is that only the cache's owner can. /// + /// The status that was checked; empty for or where there are no POSIX permissions. /// A preparation failure. - internal static void Check(string path, Access access) + internal static Unix.Status Check(string path, Access access) { if (access == Access.Any || !Unix.Supported) { - return; + return default; } Unix.Status status = StatOrFail(path, follow: false, "cannot read "); @@ -490,7 +552,7 @@ internal static void Check(string path, Access access) } } - return; + return status; } if (status.Owner != Unix.EffectiveUser()) @@ -503,5 +565,7 @@ internal static void Check(string path, Access access) throw JacquardException.Preparation("cache path " + path + " is group- or world-writable; only its owner may write the cache" + " (make it owner-only, for example chmod " + (status.Kind == Unix.Kind.Directory ? "700 " : "600 ") + path + ")"); } + + return status; } } diff --git a/dotnet/src/JacquardHost/Sdk/Runtime.cs b/dotnet/src/JacquardHost/Sdk/Runtime.cs index c39a713..70c0a0a 100644 --- a/dotnet/src/JacquardHost/Sdk/Runtime.cs +++ b/dotnet/src/JacquardHost/Sdk/Runtime.cs @@ -134,10 +134,37 @@ public Runtime(RuntimeOptions? options = null) /// Test seam: runs after the failure cleanup removes each entry, with its path. Null outside tests. internal static volatile Action? CleanupHook; + /// Test seam: runs after each Core command succeeds, with its first argument. Null outside tests. + internal static volatile Action? CommandHook; + + /// + /// Test seam: runs just before the cache root's identity is checked + /// again, with the use that follows: stage, publish, + /// load, or loaded. Null outside tests. + /// + internal static volatile Action? RootHook; + private string Command(string executable, CancellationToken cancellationToken, params string[] arguments) { Interlocked.Increment(ref commands); - return Core.Run(executable, Prelude, CommandTimeout, cancellationToken, arguments); + string output = Core.Run(executable, Prelude, CommandTimeout, cancellationToken, arguments); + CommandHook?.Invoke(arguments[0]); + return output; + } + + /// + /// Refuses the preparation unless the cache root at + /// is still the directory that was checked (the same device and inode), + /// so a root renamed away and replaced after the check is not used. + /// + private static void SameRoot(string cache, Unix.Status pinned, string use) + { + RootHook?.Invoke(use); + Unix.Status now = Core.StatOrFail(cache, follow: false, "cannot read "); + if (now.Kind != pinned.Kind || now.Device != pinned.Device || now.Inode != pinned.Inode) + { + throw JacquardException.Preparation("cache root " + cache + " was replaced after it was checked; prepare again"); + } } /// One discovery: empty until a run succeeds; a failed run leaves it empty for the next caller. @@ -210,11 +237,17 @@ private static IReadOnlyList Once(string key, Func } } - /// Runs --version once per executable digest; accepts output whose last field is the pinned version. - private void CheckVersion(string executable, string binaryHash, CancellationToken cancellationToken) => - Once("version:" + binaryHash, () => + /// + /// Runs --version once per executable fingerprint (its digest and + /// its file identity); accepts output whose last field is the pinned + /// version. A run during which the executable changed is refused, and + /// is not kept for later callers. + /// + private void CheckVersion(string executable, string binaryKey, IReadOnlyList pins, CancellationToken cancellationToken) => + Once("version:" + binaryKey, () => { string version = Command(executable, cancellationToken, "--version").Trim(); + Core.Unchanged(pins, cancellationToken); string[] fields = version.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); if (fields.Length == 0 || !string.Equals(fields[^1], Compat.CoreVersion, StringComparison.Ordinal)) { @@ -225,8 +258,12 @@ private void CheckVersion(string executable, string binaryHash, CancellationToke return Array.Empty(); }, cancellationToken); - /// Core's hash output for each prelude file, which must parse. - private IReadOnlyList PreludeOutputs(string executable, string key, CancellationToken cancellationToken) => + /// + /// Core's hash output for each prelude file, which must parse. + /// Outputs made while the executable or the prelude changed are refused, + /// and are not kept for later callers. + /// + private IReadOnlyList PreludeOutputs(string executable, string key, IReadOnlyList pins, CancellationToken cancellationToken) => Once("prelude:" + key, () => { var outputs = new List(); @@ -237,6 +274,7 @@ private IReadOnlyList PreludeOutputs(string executable, string key, Canc outputs.Add(output); } + Core.Unchanged(pins, cancellationToken); return outputs.AsReadOnly(); }, cancellationToken); @@ -411,12 +449,18 @@ private PreparedProgram PrepareChecked(IReadOnlyList sources, PrepareOpt string configured = Path.GetFullPath(Core.CheckedPath(options.CacheDir ?? DefaultCacheDir, "cache directory")); string executable = Core.Executable(Binary); - string binaryHash = Core.FileDigest(executable, cancellationToken); - CheckVersion(executable, binaryHash, cancellationToken); + string binaryHash = Core.FileDigest(executable, cancellationToken, out Unix.Status binaryStatus); + + // What Core runs is the executable and the prelude as they are when + // each command starts; their identities, read as they are + // fingerprinted, must be unchanged once the commands have finished. + var pins = new List { new("Core executable", executable, Follow: true, binaryStatus) }; + string binaryKey = binaryHash + ":" + Core.IdentityKey(binaryStatus); + CheckVersion(executable, binaryKey, pins.ToArray(), cancellationToken); string preludeHash; try { - preludeHash = Core.MapDigest(Core.Tree(Prelude, Core.Access.Any, cancellationToken)); + preludeHash = Core.MapDigest(Core.Tree(Prelude, Core.Access.Any, cancellationToken, pins)); } catch (JacquardException error) { @@ -457,13 +501,17 @@ private PreparedProgram PrepareChecked(IReadOnlyList sources, PrepareOpt string directory = Path.Combine(cache, Core.MapDigest(identity)); // Only the cache's owner may write it; nothing inside it can resist - // someone else who may. - Core.Check(cache, Core.Access.OwnerOnly); + // someone else who may. The root is used by its path, so its + // identity is checked again before each use: another user who may + // write its parent could otherwise replace it after this check. + Unix.Status root = Core.Check(cache, Core.Access.OwnerOnly); if (Unix.Stat(directory, follow: false).Kind != Unix.Kind.Missing) { - return Load(directory, identity, executable, cancellationToken); + Core.Unchanged(pins, cancellationToken); + return Load(directory, identity, executable, cache, root, cancellationToken); } + SameRoot(cache, root, "stage"); string stage = StagingDirectory(cache); bool published = false; try @@ -471,11 +519,14 @@ private PreparedProgram PrepareChecked(IReadOnlyList sources, PrepareOpt string snapshot = Path.Combine(stage, "source" + syntax); Write(snapshot, bytes, "cannot write the source snapshot"); string check = Command(executable, cancellationToken, "check", snapshot, "--print-sigs"); - IReadOnlyList preludeHashes = PreludeOutputs(executable, binaryHash + preludeHash, cancellationToken); + IReadOnlyList preludeHashes = PreludeOutputs(executable, binaryKey + preludeHash, pins, cancellationToken); string hash = Command(executable, cancellationToken, "hash", snapshot); Bindings bindings = Derive(check, preludeHashes, hash); string store = Path.Combine(stage, "store"); Command(executable, cancellationToken, "run", snapshot, "--store", store); + + // Core's commands have finished: what they ran must be what was fingerprinted. + Core.Unchanged(pins, cancellationToken); CreateDirectory(Path.Combine(stage, "core")); Write(Path.Combine(stage, CheckOutput), Encoding.UTF8.GetBytes(check), "cannot write Core's output"); Write(Path.Combine(stage, HashOutput), Encoding.UTF8.GetBytes(hash), "cannot write Core's output"); @@ -487,6 +538,7 @@ private PreparedProgram PrepareChecked(IReadOnlyList sources, PrepareOpt // Core made the store with its own permissions; the entry is owner-only. var metadata = new Metadata(identity, bindings, Core.Tree(stage, Core.Access.Restrict, cancellationToken)); Write(Path.Combine(stage, "program.json"), Encoding.UTF8.GetBytes(metadata.ToJson()), "cannot write program.json"); + SameRoot(cache, root, "publish"); try { Directory.Move(stage, directory); @@ -512,7 +564,7 @@ private PreparedProgram PrepareChecked(IReadOnlyList sources, PrepareOpt } } - return Load(directory, identity, executable, cancellationToken); + return Load(directory, identity, executable, cache, root, cancellationToken); } private static string PreludeOutput(string file) => "core/prelude-" + file + ".out"; @@ -736,11 +788,12 @@ internal static void RemoveTree(string root, CancellationToken cancellationToken /// coordinated rewrite of several files is accepted: only the cache's /// owner can make one, and the owner is trusted. /// - private PreparedProgram Load(string directory, SortedDictionary identity, string executable, CancellationToken cancellationToken) + private PreparedProgram Load(string directory, SortedDictionary identity, string executable, string cache, Unix.Status root, CancellationToken cancellationToken) { JacquardException Fail(string detail) => JacquardException.Preparation("prepared cache " + directory + " is incomplete or changed (" + detail + "); remove that cache entry and prepare again"); + SameRoot(cache, root, "load"); SortedDictionary observed; try { @@ -808,6 +861,9 @@ JacquardException Fail(string detail) => JacquardException.Preparation("prepared throw Fail("program.json bindings differ from Core's cached output"); } + // What was read came from the root that was checked. + SameRoot(cache, root, "loaded"); + var program = new PreparedProgram(this, directory, metadata.Bindings, executable); LoadedHook?.Invoke(); return program; diff --git a/dotnet/src/JacquardHost/Sdk/Unix.cs b/dotnet/src/JacquardHost/Sdk/Unix.cs index 7e5e6c5..169c747 100644 --- a/dotnet/src/JacquardHost/Sdk/Unix.cs +++ b/dotnet/src/JacquardHost/Sdk/Unix.cs @@ -9,8 +9,9 @@ namespace JacquardLang.Host.Sdk; /// The C library calls the SDK makes beyond the carrier's poll(2): /// kill(2) to stop a Core command's process group, geteuid(2) /// and a status call (statx(2) on Linux, stat(2) and -/// lstat(2) on macOS) for the cache's owner check and for file kinds -/// and sizes, and mkdir(2) to create a staging directory exclusively. +/// lstat(2) on macOS) for the cache's owner check, for file kinds and +/// sizes, and for the identities that detect a replaced file, and +/// mkdir(2) to create a staging directory exclusively. /// /// /// The macOS calls use the 64-bit-inode struct stat layout; they are @@ -42,8 +43,23 @@ internal enum Kind Other, } - /// What the status call reports: the kind, size, owner, and permission bits. - internal readonly record struct Status(Kind Kind, long Size, uint Owner, int Mode); + /// + /// What the status call reports: the kind, size, owner, and permission + /// bits, and the file's identity: its device and inode, and its + /// modification and status-change times in nanoseconds. + /// + internal readonly record struct Status(Kind Kind, long Size, uint Owner, int Mode, ulong Device = 0, ulong Inode = 0, long Modified = 0, long Changed = 0) + { + /// + /// Whether describes the same file in the + /// same state: kind, device, inode, size, mode, owner, and both times. + /// Replacing the file changes its inode; writing it, renaming it, or + /// changing its mode or owner changes its status-change time. + /// + internal bool SameFile(Status other) => + Kind == other.Kind && Device == other.Device && Inode == other.Inode && Size == other.Size + && Mode == other.Mode && Owner == other.Owner && Modified == other.Modified && Changed == other.Changed; + } [LibraryImport("libc", EntryPoint = "kill", SetLastError = true)] private static partial int NativeKill(int pid, int signal); @@ -129,6 +145,10 @@ internal static unsafe Status Stat(string path, bool follow) int modeOffset; int ownerOffset; int sizeOffset; + ulong device; + ulong inode; + long modified; + long changed; if (OperatingSystem.IsMacOS()) { bool arm = RuntimeInformation.ProcessArchitecture == Architecture.Arm64; @@ -142,6 +162,12 @@ internal static unsafe Status Stat(string path, bool follow) modeOffset = 4; ownerOffset = 16; sizeOffset = 96; + + // st_dev (int32) at 0, st_ino at 8, st_mtimespec at 48, st_ctimespec at 64. + device = *(uint*)buffer; + inode = *(ulong*)(buffer + 8); + modified = Nanoseconds(*(long*)(buffer + 48), *(long*)(buffer + 56)); + changed = Nanoseconds(*(long*)(buffer + 64), *(long*)(buffer + 72)); } else { @@ -149,6 +175,14 @@ internal static unsafe Status Stat(string path, bool follow) modeOffset = 28; ownerOffset = 20; sizeOffset = 40; + + // stx_ino at 32, stx_ctime at 96, stx_mtime at 112 (each a + // 64-bit second and a 32-bit nanosecond), stx_dev_major and + // stx_dev_minor at 136 and 140. + inode = *(ulong*)(buffer + 32); + changed = Nanoseconds(*(long*)(buffer + 96), *(uint*)(buffer + 104)); + modified = Nanoseconds(*(long*)(buffer + 112), *(uint*)(buffer + 120)); + device = ((ulong)*(uint*)(buffer + 136) << 32) | *(uint*)(buffer + 140); } if (result != 0) @@ -172,6 +206,8 @@ internal static unsafe Status Stat(string path, bool follow) LinkType => Kind.Link, _ => Kind.Other, }; - return new Status(kind, size, owner, mode & 0xFFF); + return new Status(kind, size, owner, mode & 0xFFF, device, inode, modified, changed); } + + private static long Nanoseconds(long seconds, long nanoseconds) => unchecked((seconds * 1_000_000_000L) + nanoseconds); } diff --git a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs index 08ccd56..54d69e0 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs @@ -40,6 +40,8 @@ public void CleanUp() Runtime.LoadedHook = null; Runtime.PublishedHook = null; Runtime.CleanupHook = null; + Runtime.CommandHook = null; + Runtime.RootHook = null; Core.DigestChunkHook = null; Core.TreeFileHook = null; Core.TreeEntryCap = Core.TreeEntries; @@ -752,4 +754,147 @@ public void AStopAsAWarmLoadFinishesIsReported() // Without a stop the same entry loads. fake.Runtime.Prepare(sources, fake.Options); } + + /// Replaces with a new executable file holding , as an atomic rename does. + private static void Replace(string path, string text) + { + string next = path + ".next"; + File.WriteAllText(next, text); + File.SetUnixFileMode(next, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + File.Move(next, path, overwrite: true); + } + + private static string ChangedMessage(string what, string path) => what + " " + path + " changed during preparation; prepare again once it no longer changes"; + + /// + /// The cache root is checked once and used by its path, so its identity + /// is checked again before each use. A test seam renames the checked + /// root away and puts a new directory in its place (as another user who + /// may write its parent could) just before the staging directory is + /// made, before publishing, before a load, and once a load has read the + /// entry, on a cold and on a warm preparation; each is refused. + /// + [Test] + public void ACacheRootReplacedAfterItsCheckIsRefused() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string cache = Path.GetFullPath(fake.Options.CacheDir!); + string Swap(string use, string source) + { + bool swapped = false; + Runtime.RootHook = at => + { + if (at == use && !swapped) + { + swapped = true; + Directory.Move(cache, cache + "-" + use + "-" + Path.GetFileNameWithoutExtension(source)); + Directory.CreateDirectory(cache, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + }; + string message = fake.PrepareFails(source).Message; + Runtime.RootHook = null; + Assert.That(swapped, Is.True, use + " was never reached"); + return message; + } + + string refused = "cache root " + cache + " was replaced after it was checked; prepare again"; + foreach (string use in new[] { "stage", "publish", "load", "loaded" }) + { + Assert.That(Swap(use, "cold-" + use + ".jac"), Is.EqualTo(refused), "cold " + use); + } + + foreach (string use in new[] { "load", "loaded" }) + { + string source = "warm-" + use + ".jac"; + fake.Runtime.Prepare(new[] { fake.Source(source) }, fake.Options); + Assert.That(Swap(use, source), Is.EqualTo(refused), "warm " + use); + } + + // Unswapped, the same root is used. + fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + } + + /// + /// The executable Core runs must be the one fingerprinted: a test seam + /// replaces it (a new file renamed over it) after each kind of Core + /// command, and the preparation is refused instead of keying artifacts + /// made by other bytes under the old fingerprint. + /// + [Test] + public void AnExecutableReplacedDuringPreparationIsRefused() + { + using FakeCore fake = FakeCore.Programs(At("swap")); + string script = File.ReadAllText(fake.Binary); + foreach (string after in new[] { "--version", "check", "hash", "run" }) + { + bool replaced = false; + Runtime.CommandHook = command => + { + if (command == after && !replaced) + { + replaced = true; + Replace(fake.Binary, script + "# replaced after " + after + "\n"); + } + }; + string message = fake.PrepareFails("rules" + after.Trim('-') + ".jac").Message; + Assert.That(replaced, Is.True, after + " never ran"); + Assert.That(message, Is.EqualTo(ChangedMessage("Core executable", fake.Binary)), "after " + after); + Assert.That( + Directory.Exists(Path.Combine(fake.Root, "cache")) ? fake.CacheEntries() : new List(), + Is.Empty, + "an entry was published or a staging directory left after " + after); + } + + Runtime.CommandHook = null; + fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + } + + /// + /// The prelude Core reads must be the one fingerprinted: a prelude file + /// rewritten in place, or a file added to the prelude directory, while + /// the preparation runs is refused. + /// + [Test] + public void APreludeChangedDuringPreparationIsRefused() + { + using FakeCore fake = FakeCore.Programs(At("prelude-swap")); + string prelude = Path.Combine(fake.Root, "prelude"); + string file = Path.Combine(prelude, "01-prim.jqd"); + Runtime.CommandHook = command => + { + if (command == "check") + { + File.AppendAllText(file, "changed\n"); + } + }; + Assert.That(fake.PrepareFails("first.jac").Message, Is.EqualTo(ChangedMessage("prelude entry", file))); + Runtime.CommandHook = command => + { + if (command == "run" && !File.Exists(Path.Combine(prelude, "added.jqd"))) + { + File.WriteAllText(Path.Combine(prelude, "added.jqd"), "added\n"); + } + }; + Assert.That(fake.PrepareFails("second.jac").Message, Is.EqualTo(ChangedMessage("prelude", prelude))); + Runtime.CommandHook = null; + fake.Runtime.Prepare(new[] { fake.Source("third.jac") }, fake.Options); + } + + /// + /// The version check runs once per executable fingerprint, which + /// includes the file's identity: the same bytes with a new modification + /// time (as a replaced file restored would have) are checked again. + /// + [Test] + public void TheVersionCheckIsKeyedOnTheExecutablesIdentity() + { + using FakeCore fake = FakeCore.Programs(At("version-key")); + fake.Runtime.Prepare(new[] { fake.Source("one.jac") }, fake.Options); + fake.Runtime.Prepare(new[] { fake.Source("two.jac") }, fake.Options); + Assert.That(fake.Calls("--version"), Is.EqualTo(1)); + File.SetLastWriteTimeUtc(fake.Binary, File.GetLastWriteTimeUtc(fake.Binary).AddSeconds(-10)); + fake.Runtime.Prepare(new[] { fake.Source("three.jac") }, fake.Options); + Assert.That(fake.Calls("--version"), Is.EqualTo(2)); + } + } From ce8a2034810196b3ad8b2c330482d0946862a7c1 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:02:59 -0400 Subject: [PATCH 11/52] Bound the descendant records made where Core runs without a process group Without setsid, each record of the command's descendants read every /proc entry with no cap and no stop check, and the kills when the command exited walked every recorded process before the stop was checked. A record now reads at most 16,384 entries, 4 MiB of stat files, and 1 KiB of each, keeps at most 16,384 descendants, and ends at the deadline or a cancellation, checked before each entry; the kills at the exit check them before each descendant. On a stop, a last record bounded by the same caps replaces .NET's tree kill, which reads every process on the machine. --- dotnet/src/JacquardHost/Sdk/CoreCommand.cs | 142 ++++++++++++++---- .../Sdk/CoreCommandTests.cs | 139 +++++++++++++++++ 2 files changed, 255 insertions(+), 26 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs index 2a0adda..b167aa6 100644 --- a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs +++ b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs @@ -47,11 +47,19 @@ namespace JacquardLang.Host.Sdk; /// Elsewhere (Linux without setsid), the command's descendants are /// recorded from /proc about every 10 ms, each by pid and start time, /// and every recorded descendant still alive with that start time is killed -/// when the command returns; on timeout or cancellation .NET's -/// kills the command's tree too. A -/// descendant started and reparented between two records keeps running. On -/// macOS, which has no /proc, nothing is recorded: only the tree kill -/// on a stop applies. +/// when the command returns. Each record reads at most +/// process entries and +/// bytes (at most of each stat file), keeps +/// at most descendants in all, and observes the +/// deadline and the token before each entry, ending at once on a stop; the +/// kills at the exit observe them before each descendant too, so a stop +/// leaves the descendants not yet killed running. On timeout or +/// cancellation a last record is made (bounded by the same caps, but not +/// ended by the stop), then the command is killed, then every recorded +/// descendant; a descendant past the caps, or one started and reparented +/// between two records, keeps running. On macOS, +/// which has no /proc, nothing is recorded: only .NET's +/// tree kill on a stop applies. /// /// internal static class CoreCommand @@ -71,6 +79,15 @@ internal static class CoreCommand /// How often descendants are recorded where there are no process groups. private const long SampleIntervalMs = 10; + /// The most /proc entries one record reads, and the most descendants recorded in all. + internal const int ProcEntries = 16_384; + + /// The most bytes of stat files one record reads. + internal const int ProcBytes = 4 << 20; + + /// The most bytes read of one stat file; the fields read come well before. + internal const int StatBytes = 1024; + private const short Ready = Posix.PollIn | Posix.PollHup | Posix.PollErr | Posix.PollNval; /// Test seam: when set, commands run without a process group, as where setsid is absent. @@ -82,6 +99,22 @@ internal static class CoreCommand /// Test seam: runs after each record with every pid recorded so far. Null outside tests. internal static volatile Action>? RecordedHook; + /// Test seam: the directory read as /proc. + internal static volatile string ProcRoot = "/proc"; + + /// Test seam: the entry cap of one record. + internal static volatile int ProcEntryCap = ProcEntries; + + /// Test seam: the byte cap of one record. + internal static volatile int ProcByteCap = ProcBytes; + + /// + /// Test seam: runs after each stat file a record or the kills of + /// recorded descendants read, with its path, how many bytes were read, + /// and whether a stop ends that work. Null outside tests. + /// + internal static volatile Action? StatReadHook; + /// Test seam: receives the pid of each command once it has started. Null outside tests. internal static volatile Action? StartedHook; @@ -260,20 +293,41 @@ private sealed class Recorded private readonly Dictionary started = new(); private long next = Stopwatch.GetTimestamp(); - /// Records the command's current descendants, at most once per . - internal void Sample(int root) + /// + /// Records the command's current descendants, at most once per + /// , reading at most the entry and byte + /// caps; a stop ends the record at the next entry, recording nothing + /// from it. The record, made on a stop just + /// before the command is killed, runs whenever it is asked and to + /// its caps. + /// + internal void Sample(int root, long expires, CancellationToken cancellationToken, bool final = false) { long now = Stopwatch.GetTimestamp(); - if (!RecordDescendants || now < next || !OperatingSystem.IsLinux()) + if (!RecordDescendants || (now < next && !final) || !OperatingSystem.IsLinux()) { return; } next = now + (Stopwatch.Frequency * SampleIntervalMs / 1000); var children = new Dictionary>(); - foreach (string entry in ProcEntries()) + int cap = ProcEntryCap; + int byteCap = ProcByteCap; + int entries = 0; + long bytes = 0; + foreach (string entry in ProcDirectories()) { - if (ReadStat(entry) is { } stat) + if (!final && Stopped(expires, cancellationToken) is not null) + { + return; + } + + if (++entries > cap || bytes >= byteCap) + { + break; + } + + if (ReadStat(entry, ref bytes, stoppable: !final) is { } stat) { if (!children.TryGetValue(stat.Parent, out List<(int, string)>? list)) { @@ -299,7 +353,11 @@ internal void Sample(int root) { if (seen.Add(pid)) { - started.TryAdd(pid, start); + if (started.Count < ProcEntries) + { + started.TryAdd(pid, start); + } + queue.Enqueue(pid); } } @@ -311,13 +369,22 @@ internal void Sample(int root) /// /// Kills every recorded descendant still alive with the start time it /// was recorded with, so a reused pid is not killed (the start time is - /// read just before the kill, as the JDK's process handles do). + /// read just before the kill, as the JDK's process handles do). When + /// , a stop ends it before the next + /// descendant; on a stop's own cleanup it runs to the end, bounded by + /// the number recorded. /// - internal void KillAlive() + internal void KillAlive(bool stoppable, long expires, CancellationToken cancellationToken) { + long bytes = 0; foreach ((int pid, string start) in started) { - if (ReadStat("/proc/" + pid.ToString(CultureInfo.InvariantCulture)) is { } stat && stat.Start == start && !stat.Zombie) + if (stoppable && Stopped(expires, cancellationToken) is not null) + { + return; + } + + if (ReadStat(Path.Combine(ProcRoot, pid.ToString(CultureInfo.InvariantCulture)), ref bytes, stoppable) is { } stat && stat.Start == start && !stat.Zombie) { Unix.Kill(pid); } @@ -325,12 +392,12 @@ internal void KillAlive() } } - private static IEnumerable ProcEntries() + private static IEnumerable ProcDirectories() { IEnumerable entries; try { - entries = Directory.EnumerateDirectories("/proc"); + entries = Directory.EnumerateDirectories(ProcRoot); } catch (Exception error) when (error is IOException or UnauthorizedAccessException) { @@ -375,14 +442,30 @@ private static bool IsDigits(string text) /// /// A process's pid, parent, start time, and whether it is a zombie, from - /// /proc/<pid>/stat, read after the last ) (the - /// command name may hold any byte); null when it cannot be read. + /// at most of /proc/<pid>/stat, + /// read after the last ) (the command name may hold any byte); + /// null when it cannot be read. Adds the bytes read to + /// . /// - private static (int Pid, int Parent, int Group, string Start, bool Zombie)? ReadStat(string directory) + private static (int Pid, int Parent, int Group, string Start, bool Zombie)? ReadStat(string directory, ref long bytes, bool stoppable = false) { try { - string stat = File.ReadAllText(Path.Combine(directory, "stat")); + string path = Path.Combine(directory, "stat"); + byte[] buffer = new byte[StatBytes]; + int read = 0; + using (var stream = new FileStream(path, new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Share = FileShare.ReadWrite, BufferSize = 0 })) + { + int n; + while (read < buffer.Length && (n = stream.Read(buffer, read, buffer.Length - read)) > 0) + { + read += n; + } + } + + bytes += read; + StatReadHook?.Invoke(path, read, stoppable); + string stat = System.Text.Encoding.UTF8.GetString(buffer, 0, read); int close = stat.LastIndexOf(')'); if (close < 0 || !int.TryParse(Path.GetFileName(directory), NumberStyles.None, CultureInfo.InvariantCulture, out int pid)) { @@ -392,8 +475,9 @@ private static (int Pid, int Parent, int Group, string Start, bool Zombie)? Read // state ppid pgrp session tty_nr tpgid flags minflt cminflt majflt // cmajflt utime stime cutime cstime priority nice num_threads // itrealvalue starttime ... + // A field cut by the byte cap is not read: the start time must be followed by another field. string[] fields = stat[(close + 1)..].Trim().Split(' '); - if (fields.Length < 20 + if (fields.Length < 21 || !int.TryParse(fields[1], NumberStyles.None, CultureInfo.InvariantCulture, out int parent) || !int.TryParse(fields[2], NumberStyles.None, CultureInfo.InvariantCulture, out int group)) { @@ -416,7 +500,8 @@ private static (int Pid, int Parent, int Group, string Start, bool Zombie)? Read /// internal static void KillGroup(int group, Process leader) { - if (HasExited(leader) && ReadStat("/proc/" + group.ToString(CultureInfo.InvariantCulture)) is { Zombie: false }) + long bytes = 0; + if (HasExited(leader) && ReadStat("/proc/" + group.ToString(CultureInfo.InvariantCulture), ref bytes) is { Zombie: false }) { return; } @@ -576,13 +661,17 @@ internal static Output Run( } else { + // Not .NET's tree kill, which reads every process on the + // machine: a last bounded record, the command, then + // every recorded descendant. if (!HasExited(process)) { - KillQuietly(process, entireTree: true); + recorded!.Sample(pid, expires, cancellationToken, final: true); + KillQuietly(process, entireTree: !OperatingSystem.IsLinux()); AwaitExit(process); } - recorded!.KillAlive(); + recorded!.KillAlive(stoppable: false, expires, cancellationToken); } } @@ -618,7 +707,7 @@ private static Output Await( bool exited = HasExited(process); if (!exited) { - recorded?.Sample(pid); + recorded?.Sample(pid, expires, cancellationToken); } int read = stdout.Step(Slice) + stderr.Step(Slice); @@ -635,7 +724,8 @@ private static Output Await( } else { - recorded!.KillAlive(); + // A stop ends these kills; the stop is reported below. + recorded!.KillAlive(stoppable: true, expires, cancellationToken); } cleaned = true; diff --git a/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs index 9998cb0..dc9a5f7 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs @@ -38,6 +38,10 @@ public void CleanUp() CoreCommand.ExitedHook = null; CoreCommand.KillGroupAtExit = true; CoreCommand.KillGroupOnStop = true; + CoreCommand.ProcRoot = "/proc"; + CoreCommand.ProcEntryCap = CoreCommand.ProcEntries; + CoreCommand.ProcByteCap = CoreCommand.ProcBytes; + CoreCommand.StatReadHook = null; foreach (string file in Directory.EnumerateFiles(root, "*.pid")) { Support.Kill(int.Parse(File.ReadAllText(file).Trim(), CultureInfo.InvariantCulture)); @@ -287,6 +291,141 @@ public void AStopDuringTheExitCleanupIsReported() Assert.That(error.IsAborted, Is.True); } + /// + /// A fake /proc of processes whose parent + /// is , each stat file padded to 64 KiB. + /// Their pids start at 5,000,001, above Linux's largest pid (2^22), so a + /// kill sent to one reaches no process. + /// + private string FakeProc(string name, int count, int parent) + { + string proc = Directory.CreateDirectory(Path.Combine(root, name)).FullName; + string padding = string.Concat(Enumerable.Repeat(" 0", 32 * 1024)); + for (int i = 0; i < count; i++) + { + string pid = (5_000_001 + i).ToString(CultureInfo.InvariantCulture); + string directory = Directory.CreateDirectory(Path.Combine(proc, pid)).FullName; + File.WriteAllText( + Path.Combine(directory, "stat"), + pid + " (fake) S " + parent.ToString(CultureInfo.InvariantCulture) + " " + pid + " " + pid + " 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 " + + (1000 + i).ToString(CultureInfo.InvariantCulture) + padding + "\n"); + } + + return proc; + } + + /// A command that runs until exists, for at most 30 seconds. + private static string Waiting(string release) => + "i=0; while [ ! -e '" + release + "' ] && [ $i -lt 3000 ]; do sleep 0.01; i=$((i+1)); done"; + + /// + /// Without groups, one record reads at most its entry cap of + /// /proc entries and its byte cap of stat bytes, and at + /// most 1,024 bytes of each stat file, however many processes and + /// bytes there are (a fake /proc through a test seam, with lowered + /// caps). + /// + [Test] + public void AProcRecordReadsAtMostItsCaps() + { + CoreCommand.WithoutGroups = true; + CoreCommand.ProcRoot = FakeProc("proc", 50, 1); + foreach ((int entryCap, int byteCap, int expected) in new[] { (20, CoreCommand.ProcBytes, 20), (1000, 5 * 1024, 5) }) + { + CoreCommand.ProcEntryCap = entryCap; + CoreCommand.ProcByteCap = byteCap; + var reads = new List(); + int records = 0; + using var cancel = new CancellationTokenSource(); + CoreCommand.StatReadHook = (path, bytes, stoppable) => + { + if (stoppable && records == 0) + { + reads.Add(bytes); + } + }; + CoreCommand.RecordedHook = pids => + { + if (++records == 1) + { + cancel.Cancel(); + } + }; + CarrierException error = Support.Throws(() => Run(Waiting(Path.Combine(root, "never")), cancellationToken: cancel.Token))!; + Assert.That(error.IsAborted, Is.True); + Assert.That(reads, Has.Count.EqualTo(expected), "caps " + entryCap + "/" + byteCap); + Assert.That(reads, Is.All.EqualTo(CoreCommand.StatBytes)); + } + } + + /// + /// A cancellation that lands during a record ends it at the next entry: + /// of 50 fake entries, the record stops after the third read, and the + /// command is stopped (the last record on the stop is bounded by the cap + /// instead). + /// + [Test] + public void ACancelDuringAProcRecordEndsIt() + { + CoreCommand.WithoutGroups = true; + CoreCommand.ProcRoot = FakeProc("proc", 50, 1); + CoreCommand.ProcEntryCap = 40; + int stoppable = 0; + int final = 0; + using var cancel = new CancellationTokenSource(); + CoreCommand.StatReadHook = (path, bytes, canStop) => + { + if (!canStop) + { + final++; + } + else if (++stoppable == 3) + { + cancel.Cancel(); + } + }; + CarrierException error = Support.Throws(() => Run(Waiting(Path.Combine(root, "never")), cancellationToken: cancel.Token))!; + Assert.That(error.IsAborted, Is.True); + Assert.That(stoppable, Is.EqualTo(3)); + Assert.That(final, Is.EqualTo(40)); + } + + /// + /// The kills of recorded descendants when the command exits observe a + /// stop before each descendant: ten fake descendants are recorded, the + /// command exits, and a cancellation during the second kill check ends + /// the kills there and is reported. + /// + [Test] + public void AStopDuringTheExitKillsEndsThem() + { + CoreCommand.WithoutGroups = true; + string release = Path.Combine(root, "release"); + string proc = Path.Combine(root, "proc"); + CoreCommand.ProcRoot = proc; + CoreCommand.StartedHook = pid => FakeProc("proc", 10, pid); + int recorded = 0; + CoreCommand.RecordedHook = pids => + { + recorded = pids.Count; + CoreCommand.RecordDescendants = false; + File.WriteAllText(release, string.Empty); + }; + int checks = 0; + using var cancel = new CancellationTokenSource(); + CoreCommand.StatReadHook = (path, bytes, stoppable) => + { + if (stoppable && !CoreCommand.RecordDescendants && ++checks == 2) + { + cancel.Cancel(); + } + }; + CarrierException error = Support.Throws(() => Run(Waiting(release), cancellationToken: cancel.Token))!; + Assert.That(error.IsAborted, Is.True); + Assert.That(recorded, Is.EqualTo(10)); + Assert.That(checks, Is.EqualTo(2)); + } + [Test] public void ACommandThatCannotStartIsALoss() { From 881e79763ca2d1ad1d1b64ef6a5ab9427bc9074e Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:03:22 -0400 Subject: [PATCH 12/52] Refuse caller names over 4,096 chars before they are looked up A longer name was returned unconverted and then looked up, so its whole length was hashed. Function, effect, operation, public, and constructor names (made, dynamic, or a record codec's) over the limit are now refused first, quoting an excerpt; an encoded constructor name is an invalid argument, the others binding failures. --- dotnet/src/JacquardHost/Sdk/Encoder.cs | 5 ++++ dotnet/src/JacquardHost/Sdk/Identities.cs | 26 +++++++++++++----- dotnet/src/JacquardHost/Sdk/Operations.cs | 2 +- .../src/JacquardHost/Sdk/PreparedProgram.cs | 5 ++-- dotnet/src/JacquardHost/Sdk/ProgramData.cs | 5 ++-- .../test/JacquardHost.Tests/Sdk/DataTests.cs | 27 ++++++++++++++----- .../test/JacquardHost.Tests/Sdk/FakeCore.cs | 10 +++++++ .../JacquardHost.Tests/Sdk/RuntimeTests.cs | 23 ++++++++++++++++ 8 files changed, 85 insertions(+), 18 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Encoder.cs b/dotnet/src/JacquardHost/Sdk/Encoder.cs index e2ec488..eeb40b9 100644 --- a/dotnet/src/JacquardHost/Sdk/Encoder.cs +++ b/dotnet/src/JacquardHost/Sdk/Encoder.cs @@ -132,6 +132,11 @@ public Value Constructor(string name, IReadOnlyList fields) ArgumentNullException.ThrowIfNull(name); ArgumentNullException.ThrowIfNull(fields); + if (name.Length > Identities.NameLimit) + { + throw Invalid(Identities.TooLong("constructor", name)); + } + // Counted before the fields are copied into the value. CheckCount(fields.Count, "constructor"); if (ProgramData.Builtins.Contains(Identities.DisplayToStore(name))) diff --git a/dotnet/src/JacquardHost/Sdk/Identities.cs b/dotnet/src/JacquardHost/Sdk/Identities.cs index 5255754..853c955 100644 --- a/dotnet/src/JacquardHost/Sdk/Identities.cs +++ b/dotnet/src/JacquardHost/Sdk/Identities.cs @@ -9,7 +9,7 @@ namespace JacquardLang.Host.Sdk; /// Public names and identities printed by Core's hash and check --print-sigs commands, and the SDK's text bounds. internal static class Identities { - /// The longest name, in chars, that is converted between spellings. + /// The longest name, in chars, the SDK looks up; a longer caller name is refused before it is looked up, hashed, or converted. internal const int NameLimit = 4096; /// The longest excerpt of caller text quoted in a message, in chars. @@ -70,12 +70,25 @@ internal static string Describe(Exception error) return message is null ? type : type + ": " + Excerpt(message); } - /// Converts a display name such as WorkToday to its store spelling, work-today; lowercase names are unchanged. + /// The refusal of a caller's name longer than , quoting an excerpt. + internal static string TooLong(string what, string name) => + what + " name " + Excerpt(name) + " exceeds " + NameLimit.ToString(CultureInfo.InvariantCulture) + " chars"; + + /// , refused when it is longer than , before it is looked up or hashed. + /// A binding failure quoting an excerpt. + internal static string Bounded(string name, string what) => + name.Length > NameLimit ? throw JacquardException.Binding(TooLong(what, name)) : name; + + /// + /// Converts a display name such as WorkToday to its store + /// spelling, work-today; lowercase names are unchanged. A name + /// longer than is refused, never expanded. + /// + /// A binding failure for a name over the limit. internal static string DisplayToStore(string name) { - // A name longer than any the SDK maps is returned as given, so a huge - // one is never expanded; it then names nothing. - if (name.Length == 0 || name.Length > NameLimit) + Bounded(name, "a"); + if (name.Length == 0) { return name; } @@ -104,7 +117,8 @@ internal static string DisplayToStore(string name) } /// The form used to look up effects and operations: store spelling, with _ accepted for -. - internal static string Normal(string name) => name.Length > NameLimit ? name : DisplayToStore(name).Replace('_', '-'); + /// A binding failure for a name over the limit. + internal static string Normal(string name, string what) => DisplayToStore(Bounded(name, what)).Replace('_', '-'); /// Parses one N:name hash line, or returns null. private static (string Index, Entry Entry)? HashLine(string line) diff --git a/dotnet/src/JacquardHost/Sdk/Operations.cs b/dotnet/src/JacquardHost/Sdk/Operations.cs index d300add..db01f40 100644 --- a/dotnet/src/JacquardHost/Sdk/Operations.cs +++ b/dotnet/src/JacquardHost/Sdk/Operations.cs @@ -134,7 +134,7 @@ private Operations Bind(string operation, ICodecBox[] parameters, Codec throw HandlerException.OutsideFailed("handler result cannot be represented"); } }; - string key = Identities.Normal(operation); + string key = Identities.Normal(operation, "operation"); lock (sync) { handlers[key] = handler; diff --git a/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs b/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs index bb11311..1aa9509 100644 --- a/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs +++ b/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs @@ -148,7 +148,7 @@ public PreparedProgram Provide(string effect, Operations operations) ArgumentNullException.ThrowIfNull(operations); lock (sync) { - string key = Identities.Normal(effect); + string key = Identities.Normal(effect, "effect"); if (!effects.ContainsKey(key)) { throw JacquardException.Binding("unknown effect " + Identities.Excerpt(effect) + (unsupported.Count == 0 ? string.Empty @@ -173,6 +173,7 @@ public PreparedProgram Provide(string effect, Operations operations) public Function Function(string name, IReadOnlyDictionary? bindings = null) { ArgumentNullException.ThrowIfNull(name); + Identities.Bounded(name, "function"); bindings ??= new Dictionary(); lock (sync) { @@ -217,7 +218,7 @@ public Function Function(string name, IReadOnlyDictionary? b } ArgumentNullException.ThrowIfNull(binding.Key, nameof(bindings)); - string key = Identities.Normal(binding.Key); + string key = Identities.Normal(binding.Key, "effect"); if (!selected.ContainsKey(key)) { throw JacquardException.Binding(name + " does not use " + Identities.Excerpt(binding.Key)); diff --git a/dotnet/src/JacquardHost/Sdk/ProgramData.cs b/dotnet/src/JacquardHost/Sdk/ProgramData.cs index 863741b..0e55590 100644 --- a/dotnet/src/JacquardHost/Sdk/ProgramData.cs +++ b/dotnet/src/JacquardHost/Sdk/ProgramData.cs @@ -62,7 +62,7 @@ internal ProgramData(IReadOnlyDictionary names, IReadOnlyDictionar public Hash Identity(string name) { ArgumentNullException.ThrowIfNull(name); - return names.TryGetValue(Identities.DisplayToStore(name), out Hash? identity) + return names.TryGetValue(Identities.DisplayToStore(Identities.Bounded(name, "public")), out Hash? identity) ? identity : throw JacquardException.Binding("unknown name " + Identities.Excerpt(name)); } @@ -76,7 +76,8 @@ public Hash Identity(string name) internal Hash? Name(string name) => names.TryGetValue(name, out Hash? identity) ? identity : null; - internal Hash? ConstructorId(string name) => constructors.TryGetValue(Identities.DisplayToStore(name), out Hash? identity) ? identity : null; + /// A binding failure for a name over the limit. + internal Hash? ConstructorId(string name) => constructors.TryGetValue(Identities.DisplayToStore(Identities.Bounded(name, "constructor")), out Hash? identity) ? identity : null; /// A after checking that the constructor exists. /// diff --git a/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs index 735f4bc..e110cf7 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs @@ -122,7 +122,7 @@ public void SignaturesAndGroups() Assert.That(Identities.DisplayToStore(display), Is.EqualTo(store)); } - Assert.That(Identities.Normal("lookup_stock"), Is.EqualTo("lookup-stock")); + Assert.That(Identities.Normal("lookup_stock", "operation"), Is.EqualTo("lookup-stock")); Assert.That(Identities.DisplayToStore("\ud800X"), Is.EqualTo("\ud800X"), "a lone surrogate is not upper case"); } @@ -539,16 +539,29 @@ Dynamic Deep() } /// - /// A constructor name longer than any the SDK maps is neither expanded - /// nor copied whole into a message: it names nothing, and the refusal - /// quotes its first 64 chars and its length. + /// A caller's name over 4,096 chars is refused before it is looked up, + /// hashed, or converted, quoting its first 64 chars and its length: a + /// public name, a constructor by name (dynamic, made, or a record's), + /// and an operation. A name of exactly 4,096 chars is looked up, and + /// names nothing. /// [Test] - public void AHugeConstructorNameIsRefusedWithoutBeingExpanded() + public void AnOverlongNameIsRefusedBeforeItIsLookedUp() { string huge = new('X', 1_000_000); - Assert.That(ReferenceEquals(huge, Identities.DisplayToStore(huge)), Is.True, "the name was converted"); - Assert.That(Refusal(() => Fixture(Limits.Hard).Encode(Codecs.Dynamic, Dynamic.VariantOf(huge))), Is.EqualTo("value: unknown constructor " + new string('X', 64) + "... (1000000 chars)")); + string excerpt = new string('X', 64) + "... (1000000 chars)"; + ProgramData data = Fixture(Limits.Hard); + Assert.That(Refusal(() => Identities.DisplayToStore(huge)), Is.EqualTo("a name " + excerpt + " exceeds 4096 chars")); + Assert.That(Refusal(() => data.Identity(huge)), Is.EqualTo("public name " + excerpt + " exceeds 4096 chars")); + Assert.That(Refusal(() => data.Make(huge)), Is.EqualTo("constructor name " + excerpt + " exceeds 4096 chars")); + JacquardException dynamic = Support.Throws(() => data.Encode(Codecs.Dynamic, Dynamic.VariantOf(huge))); + Assert.That(dynamic.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(dynamic.Message, Is.EqualTo("value: constructor name " + excerpt + " exceeds 4096 chars")); + Assert.That(Refusal(() => data.Encode(Codecs.Record(huge, () => 1L), 1L)), Is.EqualTo("value: constructor name " + excerpt + " exceeds 4096 chars")); + Assert.That(Refusal(() => data.Decode(Codecs.Record(huge, () => 1L), data.Encode(Codecs.Unit, Unit.Value))), Does.Contain("constructor name " + excerpt + " exceeds 4096 chars")); + Assert.That(Refusal(() => new Operations().On(huge, Codecs.Int, () => 1L)), Is.EqualTo("operation name " + excerpt + " exceeds 4096 chars")); + string longest = new('X', 4096); + Assert.That(Refusal(() => data.Encode(Codecs.Dynamic, Dynamic.VariantOf(longest))), Does.StartWith("value: unknown constructor ")); Assert.That(Identities.DisplayToStore("JobTicket"), Is.EqualTo("job-ticket")); } diff --git a/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs b/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs index 20dfb8e..88f915a 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs @@ -73,6 +73,16 @@ internal static FakeCore Programs(string root) => Create(root, Compat.CoreVersio + Second + "' ;; esac ;;\n" + "run) mkdir -p \"$4\" && echo objects > \"$4/objects\" ;;"); + /// + /// A fake whose one function, first, uses the effect + /// Inventory, whose one operation is lookup. + /// + internal static FakeCore Effects(string root) => Create(root, Compat.CoreVersion, null, + "check) echo 'first : () ->{Inventory} Int' ;;\n" + + "hash) case \"$2\" in */prelude/*) echo '0:int " + new string('1', 64) + "' ;; *) echo '0:first " + First + "'; echo '1:inventory " + + new string('c', 64) + "'; echo '1:lookup " + new string('d', 64) + "' ;; esac ;;\n" + + "run) mkdir -p \"$4\" && echo objects > \"$4/objects\" ;;"); + internal string Source(string name) { string source = Path.Combine(Root, name); diff --git a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs index 54d69e0..d72f441 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs @@ -897,4 +897,27 @@ public void TheVersionCheckIsKeyedOnTheExecutablesIdentity() Assert.That(fake.Calls("--version"), Is.EqualTo(2)); } + /// + /// A function, effect, or operation name over 4,096 chars is refused + /// before it is looked up, quoting an excerpt; one of exactly 4,096 + /// chars is looked up (and names nothing). + /// + [Test] + public void OverlongProgramNamesAreRefusedBeforeTheyAreLookedUp() + { + using FakeCore fake = FakeCore.Effects(At("names")); + PreparedProgram program = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + string huge = new('X', 1_000_000); + Operations service = new Operations().On("lookup", Codecs.Int, () => 1L); + Assert.That(Support.Throws(() => program.Function(huge)).Message, Is.EqualTo("function name " + Excerpt('X', 1_000_000) + " exceeds 4096 chars")); + Assert.That(Support.Throws(() => program.Provide(huge, service)).Message, Is.EqualTo("effect name " + Excerpt('X', 1_000_000) + " exceeds 4096 chars")); + Assert.That( + Support.Throws(() => program.Function("first", new Dictionary { [huge] = service })).Message, + Is.EqualTo("effect name " + Excerpt('X', 1_000_000) + " exceeds 4096 chars")); + Assert.That(Support.Throws(() => new Operations().On(huge, Codecs.Int, () => 1L)).Message, Is.EqualTo("operation name " + Excerpt('X', 1_000_000) + " exceeds 4096 chars")); + string longest = new('x', 4096); + Assert.That(Support.Throws(() => program.Function(longest)).Message, Does.EndWith(": absent or unsupported closed v0 signature")); + Assert.That(Support.Throws(() => program.Provide(longest, service)).Message, Does.StartWith("unknown effect ")); + Assert.That(program.Function("first", new Dictionary { ["Inventory"] = service }).Name, Is.EqualTo("first")); + } } From 3d50e6be32d6d33e4a391b9b0ff010a5b10574b4 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:03:38 -0400 Subject: [PATCH 13/52] Refuse numeric types that have no Jacquard type instead of converting them C# chained a standard conversion into the SDK's own, so a ulong or nuint became a Real through double (2^53 + 1 arrived as 2^53), a char an Int, and a float a double. Conversions to Argument and Dynamic.Of overloads for ulong, nuint, char, float, Half, decimal, Int128, UInt128, and BigInteger are now obsolete as errors with the reason, so a use does not compile; reached by reflection, the argument is refused when encoded and Of throws. Exact overloads for the smaller integer types and uint keep them converting to Int without ambiguity. --- dotnet/src/JacquardHost/Sdk/Dynamic.cs | 63 +++++++++++ dotnet/src/JacquardHost/Sdk/Values.cs | 101 +++++++++++++++++- .../test/JacquardHost.Tests/Sdk/DataTests.cs | 59 ++++++++-- 3 files changed, 211 insertions(+), 12 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Dynamic.cs b/dotnet/src/JacquardHost/Sdk/Dynamic.cs index 043bd60..a71550c 100644 --- a/dotnet/src/JacquardHost/Sdk/Dynamic.cs +++ b/dotnet/src/JacquardHost/Sdk/Dynamic.cs @@ -32,6 +32,24 @@ private protected Dynamic() /// An Int. public static Dynamic Of(long value) => new Int(value); + /// An Int. + public static Dynamic Of(int value) => new Int(value); + + /// An Int; a widens exactly. + public static Dynamic Of(uint value) => new Int(value); + + /// An Int. + public static Dynamic Of(short value) => new Int(value); + + /// An Int. + public static Dynamic Of(ushort value) => new Int(value); + + /// An Int. + public static Dynamic Of(sbyte value) => new Int(value); + + /// An Int. + public static Dynamic Of(byte value) => new Int(value); + /// A Real. public static Dynamic Of(double value) => new Real(value); @@ -41,6 +59,51 @@ private protected Dynamic() /// A Bool. public static Dynamic Of(bool value) => new Bool(value); + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Unsigned64, error: true)] + public static Dynamic Of(ulong value) => throw new ArgumentException(NoConversion.Unsigned64, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.UnsignedNative, error: true)] + public static Dynamic Of(nuint value) => throw new ArgumentException(NoConversion.UnsignedNative, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Character, error: true)] + public static Dynamic Of(char value) => throw new ArgumentException(NoConversion.Character, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Single, error: true)] + public static Dynamic Of(float value) => throw new ArgumentException(NoConversion.Single, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.HalfFloat, error: true)] + public static Dynamic Of(Half value) => throw new ArgumentException(NoConversion.HalfFloat, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Decimal, error: true)] + public static Dynamic Of(decimal value) => throw new ArgumentException(NoConversion.Decimal, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Signed128, error: true)] + public static Dynamic Of(Int128 value) => throw new ArgumentException(NoConversion.Signed128, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Unsigned128, error: true)] + public static Dynamic Of(UInt128 value) => throw new ArgumentException(NoConversion.Unsigned128, nameof(value)); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + /// Always. + [Obsolete(NoConversion.Big, error: true)] + public static Dynamic Of(System.Numerics.BigInteger value) => throw new ArgumentException(NoConversion.Big, nameof(value)); + /// A List. /// More items than the v0 hard limit. public static Dynamic ListOf(params Dynamic[] items) => new List(items); diff --git a/dotnet/src/JacquardHost/Sdk/Values.cs b/dotnet/src/JacquardHost/Sdk/Values.cs index c210eda..d5c9a3b 100644 --- a/dotnet/src/JacquardHost/Sdk/Values.cs +++ b/dotnet/src/JacquardHost/Sdk/Values.cs @@ -74,15 +74,44 @@ public static class Option public static Option Some(T value) => Option.Some(value); } +/// +/// Why each .NET numeric type without a Jacquard type is refused. Without +/// these refusals C# would convert a or +/// silently through , a +/// to an Int, and a to a +/// with a different decimal text. The conversions to +/// and the overloads +/// for these types are marked obsolete as errors, so using one does not +/// compile; reached anyway (by reflection), the argument is refused when it +/// is encoded and the Of overload throws. +/// +internal static class NoConversion +{ + internal const string Unsigned64 = "ulong has no Jacquard type and is not converted: Int is a signed 63-bit integer; pass a range-checked long"; + internal const string UnsignedNative = "nuint has no Jacquard type and is not converted: Int is a signed 63-bit integer; pass a range-checked long"; + internal const string Character = "char has no Jacquard type and is not converted: pass a string for Text, or a long for its code"; + internal const string Single = "float is not converted: Real is a double; pass (double)value"; + internal const string HalfFloat = "Half is not converted: Real is a double; pass (double)value"; + internal const string Decimal = "decimal has no Jacquard type and is never converted: pass money as an integer count of minor units (long); Real is a double"; + internal const string Signed128 = "Int128 has no Jacquard type and is not converted: Int is a signed 63-bit integer; pass a range-checked long"; + internal const string Unsigned128 = "UInt128 has no Jacquard type and is not converted: Int is a signed 63-bit integer; pass a range-checked long"; + internal const string Big = "BigInteger has no Jacquard type and is not converted: Int is a signed 63-bit integer; pass a range-checked long"; +} + /// /// A function argument with the codec that encodes it. Plain values convert -/// implicitly: and as Int, +/// implicitly: and (and the smaller +/// integer types and , which widen exactly) as Int, /// as Real, as Text, /// , , a protocol /// , and . Anything else -/// is passed as codec.Arg(value), as are lists and records. There is -/// deliberately no conversion from : money crosses as an -/// integer count of minor units. +/// is passed as codec.Arg(value), as are lists and records. +/// , , , +/// , , , +/// , , and +/// are refused at compile time +/// instead of converting silently: money crosses as an integer count of +/// minor units. /// public abstract class Argument { @@ -96,6 +125,21 @@ private protected Argument() /// An Int. public static implicit operator Argument(int value) => Codecs.Int.Arg(value); + /// An Int; a widens exactly. + public static implicit operator Argument(uint value) => Codecs.Int.Arg(value); + + /// An Int. + public static implicit operator Argument(short value) => Codecs.Int.Arg(value); + + /// An Int. + public static implicit operator Argument(ushort value) => Codecs.Int.Arg(value); + + /// An Int. + public static implicit operator Argument(sbyte value) => Codecs.Int.Arg(value); + + /// An Int. + public static implicit operator Argument(byte value) => Codecs.Int.Arg(value); + /// A Real, the exact binary64 bits of the double. public static implicit operator Argument(double value) => Codecs.Real.Arg(value); @@ -114,6 +158,42 @@ private protected Argument() /// A value by public name. public static implicit operator Argument(Dynamic value) => Codecs.Dynamic.Arg(value); + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Unsigned64, error: true)] + public static implicit operator Argument(ulong value) => new Refused(NoConversion.Unsigned64); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.UnsignedNative, error: true)] + public static implicit operator Argument(nuint value) => new Refused(NoConversion.UnsignedNative); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Character, error: true)] + public static implicit operator Argument(char value) => new Refused(NoConversion.Character); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Single, error: true)] + public static implicit operator Argument(float value) => new Refused(NoConversion.Single); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.HalfFloat, error: true)] + public static implicit operator Argument(Half value) => new Refused(NoConversion.HalfFloat); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Decimal, error: true)] + public static implicit operator Argument(decimal value) => new Refused(NoConversion.Decimal); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Signed128, error: true)] + public static implicit operator Argument(Int128 value) => new Refused(NoConversion.Signed128); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Unsigned128, error: true)] + public static implicit operator Argument(UInt128 value) => new Refused(NoConversion.Unsigned128); + + /// Refused: the type has no Jacquard type, and using this does not compile (see the obsolete message). + [Obsolete(NoConversion.Big, error: true)] + public static implicit operator Argument(System.Numerics.BigInteger value) => new Refused(NoConversion.Big); + /// encoded with ; the same as codec.Arg(value). public static Argument Of(Codec codec, T value) { @@ -161,6 +241,19 @@ internal Typed(Codec codec, T value) internal override Value Encode(Encoder encoder) => encoder.Encode(codec, value); } + + /// A value of a type with no Jacquard type, refused when it is encoded, before any worker starts. + private sealed class Refused : Argument + { + private readonly string reason; + + internal Refused(string reason) + { + this.reason = reason; + } + + internal override Value Encode(Encoder encoder) => throw encoder.Invalid(reason); + } } /// One batch's rows as the final List argument, encoded from the caller's list without copying it. diff --git a/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs index e110cf7..fab489e 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Numerics; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -236,18 +237,60 @@ public void ExplicitlyNamedEnumsTuplesAndUnionsAgree() } /// - /// There is no implicit conversion from (or from - /// to Int): money crosses as integer minor units. + /// Numeric types with no Jacquard type are refused instead of converting + /// silently (C# would take a through + /// , so 2^53 + 1 arrives as 2^53, and a + /// to an Int): their conversions to + /// and their + /// overloads are obsolete as errors, so a use does not compile; reached by + /// reflection, the argument is refused when it is encoded and the + /// overload throws. There is no decimal codec: money crosses as integer + /// minor units. The integer types that widen exactly still convert to + /// Int. /// [Test] - public void DecimalHasNoConversion() + public void NumericTypesWithoutAJacquardTypeAreRefused() { - IEnumerable sources = typeof(Argument).GetMethods(BindingFlags.Public | BindingFlags.Static) - .Where(method => method.Name is "op_Implicit" or "op_Explicit") - .Select(method => method.GetParameters()[0].ParameterType); - Assert.That(sources, Does.Not.Contain(typeof(decimal))); - Assert.That(typeof(Dynamic).GetMethods().Where(method => method.Name == "Of").Select(method => method.GetParameters()[0].ParameterType), Does.Not.Contain(typeof(decimal))); + ProgramData data = Fixture(Limits.Hard); + const ulong Past = (1UL << 53) + 1; + var refused = new (object Value, string Reason)[] + { + (Past, NoConversion.Unsigned64), + (unchecked((nuint)Past), NoConversion.UnsignedNative), + ('x', NoConversion.Character), + (0.1f, NoConversion.Single), + ((Half)1, NoConversion.HalfFloat), + (1.10m, NoConversion.Decimal), + ((Int128)Past, NoConversion.Signed128), + ((UInt128)Past, NoConversion.Unsigned128), + (new BigInteger(Past), NoConversion.Big), + }; + foreach ((object value, string reason) in refused) + { + Type type = value.GetType(); + MethodInfo conversion = typeof(Argument).GetMethod("op_Implicit", new[] { type }) + ?? throw new AssertionException(type.Name + " has no refusing conversion"); + Assert.That(conversion.GetCustomAttribute() is { IsError: true } obsolete && obsolete.Message == reason, Is.True, type.Name + " conversion compiles"); + var argument = (Argument)conversion.Invoke(null, new[] { value })!; + JacquardException error = Support.Throws(() => argument.Encode(Encoder.Root(data, "value"))); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(error.Message, Is.EqualTo("value: " + reason)); + MethodInfo of = typeof(Dynamic).GetMethod("Of", new[] { type }) ?? throw new AssertionException(type.Name + " has no refusing Dynamic.Of"); + Assert.That(of.GetCustomAttribute() is { IsError: true }, Is.True, type.Name + " Dynamic.Of compiles"); + TargetInvocationException thrown = Support.Throws(() => of.Invoke(null, new[] { value })); + Assert.That(thrown.InnerException, Is.InstanceOf().With.Message.StartWith(reason)); + } + Assert.That(typeof(Codecs).GetProperties().Select(property => property.PropertyType), Does.Not.Contain(typeof(Codec))); + foreach (Argument exact in new Argument[] { (byte)7, (sbyte)7, (short)7, (ushort)7, 7U, 7, 7L, (nint)7 }) + { + Assert.That(exact.Encode(Encoder.Root(data, "value")), Is.EqualTo(new IntValue(7))); + } + + foreach (Dynamic exact in new[] { Dynamic.Of((byte)7), Dynamic.Of((sbyte)7), Dynamic.Of((short)7), Dynamic.Of((ushort)7), Dynamic.Of(7U), Dynamic.Of(7), Dynamic.Of((nint)7) }) + { + Assert.That(exact, Is.EqualTo(Dynamic.Of(7L))); + } } /// A recursive type maps through a lazy codec, which is made once even when many threads ask at once. From 323d6eb91fb97ee9c82b3f1346e7e279287392a8 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:03:45 -0400 Subject: [PATCH 14/52] Run a lazy codec's factory again after it throws Lazy in ExecutionAndPublication mode caches the factory's exception, so a lazy codec whose factory failed once could never be used. The factory now runs under a lock and only its codec is kept: a failure keeps nothing, the next use runs it again, and once it succeeds it never runs again. A factory that asks for its own codec is still refused. --- dotnet/src/JacquardHost/Sdk/Builtins.cs | 45 +++++++++++++++++-- .../test/JacquardHost.Tests/Sdk/DataTests.cs | 22 +++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Builtins.cs b/dotnet/src/JacquardHost/Sdk/Builtins.cs index 7cd2217..2e1930a 100644 --- a/dotnet/src/JacquardHost/Sdk/Builtins.cs +++ b/dotnet/src/JacquardHost/Sdk/Builtins.cs @@ -724,18 +724,55 @@ public override TUnion Decode(Decoder decoder, Value value) } } -/// A codec made on first use, for recursive types; made by . +/// +/// A codec made on first use, for recursive types; made by +/// . The factory runs on one thread at a time +/// and its codec, once made, is kept and shared by every thread; a factory +/// that throws keeps nothing, so the next use runs it again. +/// /// The .NET type. public sealed class LazyCodec : Codec, ICase, Builtins.ILazy { - private readonly Lazy> target; + private readonly object gate = new(); + private readonly Func> make; + private volatile Codec? made; + private bool making; internal LazyCodec(Func> make) { - target = new Lazy>(() => make() ?? throw new InvalidOperationException("a lazy codec's factory returned null"), LazyThreadSafetyMode.ExecutionAndPublication); + this.make = make; } - object Builtins.ILazy.Target => target.Value; + object Builtins.ILazy.Target => made ?? Make(); + + private Codec Make() + { + lock (gate) + { + if (made is { } done) + { + return done; + } + + // The lock is reentrant: a factory that asks for its own codec would run again. + if (making) + { + throw new InvalidOperationException("a lazy codec's factory used the codec it makes"); + } + + making = true; + try + { + Codec codec = make() ?? throw new InvalidOperationException("a lazy codec's factory returned null"); + made = codec; + return codec; + } + finally + { + making = false; + } + } + } internal override bool AcceptsNull => Resolved().AcceptsNull; diff --git a/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs index fab489e..f34ebe3 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs @@ -608,6 +608,28 @@ public void AnOverlongNameIsRefusedBeforeItIsLookedUp() Assert.That(Identities.DisplayToStore("JobTicket"), Is.EqualTo("job-ticket")); } + /// + /// A lazy codec whose factory throws keeps nothing: the next use runs + /// the factory again, and once it succeeds its codec is kept and the + /// factory never runs again, across threads. + /// + [Test] + public void ALazyCodecWhoseFactoryFailedIsMadeOnTheNextUse() + { + ProgramData data = Fixture(Limits.Hard); + int calls = 0; + LazyCodec codec = Codecs.Lazy(() => Interlocked.Increment(ref calls) <= 2 ? throw new InvalidOperationException("not ready") : Codecs.Int); + for (int attempt = 1; attempt <= 2; attempt++) + { + JacquardException error = Support.Throws(() => data.Encode(codec, 1L)); + Assert.That(error.InnerException, Is.InstanceOf().With.Message.EqualTo("not ready")); + Assert.That(calls, Is.EqualTo(attempt)); + } + + Parallel.For(0, 16, i => Assert.That(data.Decode(codec, data.Encode(codec, i)), Is.EqualTo((long)i))); + Assert.That(calls, Is.EqualTo(3), "the codec was made more than once after it succeeded"); + } + /// /// Messages quote caller text as an excerpt, and no message is kept /// past 8,192 chars: a result codec naming a huge constructor, codecs, From 76a758227fff70c1a94463192f244ec33bf93671 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:03:45 -0400 Subject: [PATCH 15/52] Compare each pair of shared Dynamic sub-values once Equality walked every path, so two separately built values that share sub-values (a tuple holding the same tuple twice, 64 levels deep) took exponential time. Pairs already compared are skipped. --- dotnet/src/JacquardHost/Sdk/Dynamic.cs | 21 ++++++++++++++-- .../test/JacquardHost.Tests/Sdk/DataTests.cs | 24 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Dynamic.cs b/dotnet/src/JacquardHost/Sdk/Dynamic.cs index a71550c..84984aa 100644 --- a/dotnet/src/JacquardHost/Sdk/Dynamic.cs +++ b/dotnet/src/JacquardHost/Sdk/Dynamic.cs @@ -13,7 +13,8 @@ namespace JacquardLang.Host.Sdk; /// . Equality is structural (a /// compares its exact bits), and both it and /// walk a value with an explicit stack, so a deep -/// value cannot exhaust the call stack. +/// value cannot exhaust the call stack; equality compares each pair of +/// shared sub-values once. /// public abstract class Dynamic : IEquatable { @@ -175,12 +176,16 @@ public bool Equals(Dynamic? other) return false; } + // Each pair of nodes is compared once: values that share sub-values + // (as decoded values and caller-built graphs may) are compared in + // time proportional to their distinct pairs, not their paths. var stack = new Stack<(Dynamic Left, Dynamic Right)>(); + var compared = new HashSet<(Dynamic Left, Dynamic Right)>(PairComparer.Instance); stack.Push((this, other)); while (stack.Count > 0) { (Dynamic left, Dynamic right) = stack.Pop(); - if (ReferenceEquals(left, right)) + if (ReferenceEquals(left, right) || !compared.Add((left, right))) { continue; } @@ -209,6 +214,18 @@ public bool Equals(Dynamic? other) /// public override bool Equals(object? obj) => obj is Dynamic other && Equals(other); + /// Pairs of nodes by reference. + private sealed class PairComparer : IEqualityComparer<(Dynamic Left, Dynamic Right)> + { + internal static readonly PairComparer Instance = new(); + + public bool Equals((Dynamic Left, Dynamic Right) x, (Dynamic Left, Dynamic Right) y) => + ReferenceEquals(x.Left, y.Left) && ReferenceEquals(x.Right, y.Right); + + public int GetHashCode((Dynamic Left, Dynamic Right) obj) => + HashCode.Combine(System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(obj.Left), System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(obj.Right)); + } + /// A hash of this node and its children's count; equal values hash alike. public override int GetHashCode() => HashCode.Combine(GetType(), NodeHash(), Children.Count); diff --git a/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs index f34ebe3..207f5eb 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs @@ -630,6 +630,30 @@ public void ALazyCodecWhoseFactoryFailedIsMadeOnTheNextUse() Assert.That(calls, Is.EqualTo(3), "the codec was made more than once after it succeeded"); } + /// + /// Equality compares each pair of shared sub-values once: two separately + /// built values of 64 levels, each level a tuple holding the level below + /// twice (2^64 paths), compare in time proportional to their nodes. + /// + [Test] + public void EqualityComparesSharedSubValuesOnce() + { + static Dynamic Layered(int depth, long leaf) + { + Dynamic node = Dynamic.Of(leaf); + for (int i = 0; i < depth; i++) + { + node = Dynamic.TupleOf(node, node); + } + + return node; + } + + Task<(bool Same, bool Different)> comparison = Task.Run(() => (Layered(64, 1).Equals(Layered(64, 1)), Layered(64, 1).Equals(Layered(64, 2)))); + Assert.That(comparison.Wait(TimeSpan.FromMinutes(1)), Is.True, "the comparison walks every path"); + Assert.That(comparison.Result, Is.EqualTo((true, false))); + } + /// /// Messages quote caller text as an excerpt, and no message is kept /// past 8,192 chars: a result codec naming a huge constructor, codecs, From c7dba6e99142272ff3616ccb88ee0aa39d757a9a Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:03:54 -0400 Subject: [PATCH 16/52] Check that this user may execute Core with access(2) The executable check accepted any execute bit, so a file only its group or others may execute was resolved and then failed to start with a less clear error. Core and setsid are now checked with access(X_OK); a regular Core executable this user may not run is refused with a message saying so. --- dotnet/src/JacquardHost/Sdk/Core.cs | 21 +++++++++++++++---- dotnet/src/JacquardHost/Sdk/CoreCommand.cs | 4 ++-- dotnet/src/JacquardHost/Sdk/Unix.cs | 11 ++++++++-- .../JacquardHost.Tests/Sdk/RuntimeTests.cs | 17 +++++++++++++++ 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Core.cs b/dotnet/src/JacquardHost/Sdk/Core.cs index e267ace..0c70344 100644 --- a/dotnet/src/JacquardHost/Sdk/Core.cs +++ b/dotnet/src/JacquardHost/Sdk/Core.cs @@ -484,11 +484,22 @@ JacquardException Missing() => JacquardException.Preparation("cannot find Core e } } - if (candidate is null || !IsExecutable(candidate)) + if (candidate is null) { throw Missing(); } + if (!IsExecutable(candidate)) + { + if (IsRegular(candidate)) + { + throw JacquardException.Preparation("Core executable " + Identities.Excerpt(binary) + + " may not be executed by the current user; give it execute permission or configure JACQUARD_BIN"); + } + + throw Missing(); + } + try { FileSystemInfo? target = new FileInfo(candidate).ResolveLinkTarget(returnFinalTarget: true); @@ -500,7 +511,10 @@ JacquardException Missing() => JacquardException.Preparation("cannot find Core e } } - private static bool IsExecutable(string path) + /// Whether is a regular file this process may execute (access(X_OK)), not merely one with some execute bit. + private static bool IsExecutable(string path) => IsRegular(path) && Unix.MayExecute(path); + + private static bool IsRegular(string path) { if (path.Length > PathLimit) { @@ -509,8 +523,7 @@ private static bool IsExecutable(string path) try { - Unix.Status status = Unix.Stat(path, follow: true); - return status.Kind == Unix.Kind.Regular && (status.Mode & 0x49) != 0; + return Unix.Stat(path, follow: true).Kind == Unix.Kind.Regular; } catch (IOException) { diff --git a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs index b167aa6..55306b9 100644 --- a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs +++ b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs @@ -151,12 +151,12 @@ internal static class CoreCommand return null; } + /// Whether is a regular file this process may execute (access(X_OK)). private static bool IsExecutable(string path) { try { - Unix.Status status = Unix.Stat(path, follow: true); - return status.Kind == Unix.Kind.Regular && (status.Mode & 0x49) != 0; + return Unix.Stat(path, follow: true).Kind == Unix.Kind.Regular && Unix.MayExecute(path); } catch (IOException) { diff --git a/dotnet/src/JacquardHost/Sdk/Unix.cs b/dotnet/src/JacquardHost/Sdk/Unix.cs index 169c747..187707c 100644 --- a/dotnet/src/JacquardHost/Sdk/Unix.cs +++ b/dotnet/src/JacquardHost/Sdk/Unix.cs @@ -10,8 +10,9 @@ namespace JacquardLang.Host.Sdk; /// kill(2) to stop a Core command's process group, geteuid(2) /// and a status call (statx(2) on Linux, stat(2) and /// lstat(2) on macOS) for the cache's owner check, for file kinds and -/// sizes, and for the identities that detect a replaced file, and -/// mkdir(2) to create a staging directory exclusively. +/// sizes, and for the identities that detect a replaced file, +/// access(2) for the execute permission, and mkdir(2) to +/// create a staging directory exclusively. /// /// /// The macOS calls use the 64-bit-inode struct stat layout; they are @@ -67,6 +68,9 @@ internal bool SameFile(Status other) => [LibraryImport("libc", EntryPoint = "geteuid")] private static partial uint NativeGeteuid(); + [LibraryImport("libc", EntryPoint = "access", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static partial int NativeAccess(string path, int mode); + [LibraryImport("libc", EntryPoint = "mkdir", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] private static partial int NativeMkdir(string path, uint mode); @@ -113,6 +117,9 @@ internal static void Kill(int pid) } } + /// Whether this process may execute , as access(path, X_OK) reports. + internal static bool MayExecute(string path) => NativeAccess(path, 1) == 0; + /// Creates with ; false when it already exists. /// The directory cannot be created for another reason. internal static bool MakeDirectory(string path, int mode) diff --git a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs index d72f441..7ab2682 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs @@ -897,6 +897,23 @@ public void TheVersionCheckIsKeyedOnTheExecutablesIdentity() Assert.That(fake.Calls("--version"), Is.EqualTo(2)); } + /// + /// The executable must be one this user may run (access(X_OK)), + /// not merely one with some execute bit: a file only its group and + /// others may execute is refused with a clear message before it runs. + /// + [Test] + public void AnExecutableTheUserMayNotRunIsRefused() + { + Assume.That(Unix.EffectiveUser(), Is.Not.EqualTo(0u), "running as root"); + using FakeCore fake = FakeCore.Programs(At("noexec")); + File.SetUnixFileMode(fake.Binary, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherExecute); + Assert.That( + fake.PrepareFails().Message, + Is.EqualTo("Core executable " + fake.Binary + " may not be executed by the current user; give it execute permission or configure JACQUARD_BIN")); + Assert.That(fake.Calls("--version"), Is.EqualTo(0)); + } + /// /// A function, effect, or operation name over 4,096 chars is refused /// before it is looked up, quoting an excerpt; one of exactly 4,096 From d1eb3764da7bcd50f26ec771ee56758e44492aaf Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:03:54 -0400 Subject: [PATCH 17/52] Read a function's bindings before taking the program's lock PreparedProgram.Function enumerated the caller's bindings dictionary while it held the lock that Provide, Functions, and ToString take, so caller code that blocked held them all. The count and at most one entry past the program's effects are now read first. --- .../src/JacquardHost/Sdk/PreparedProgram.cs | 30 ++++++++++-- .../JacquardHost.Tests/Sdk/RuntimeTests.cs | 46 +++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs b/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs index 1aa9509..5ab0147 100644 --- a/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs +++ b/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs @@ -124,6 +124,9 @@ public IReadOnlyList Functions } } + /// Test seam: whether the calling thread holds the program's lock. + internal bool LockHeld => System.Threading.Monitor.IsEntered(sync); + /// Test seam: replaces the worker command for functions bound afterwards. internal void SetWorkerCommand(IReadOnlyList command, IReadOnlyDictionary? environment = null) { @@ -175,6 +178,25 @@ public Function Function(string name, IReadOnlyDictionary? b ArgumentNullException.ThrowIfNull(name); Identities.Bounded(name, "function"); bindings ??= new Dictionary(); + + // The caller's dictionary is read here, before the program's lock is + // taken, so its code cannot hold that lock: its count once, and its + // entries only when they are no more than the program's effects, at + // most one past them. The effects are fixed when the program is made. + int declared = bindings.Count; + var given = new List>(); + if (declared <= effects.Count) + { + foreach (KeyValuePair binding in bindings) + { + given.Add(binding); + if (given.Count > effects.Count) + { + break; + } + } + } + lock (sync) { if (!interfaces.TryGetValue(name, out CallInterface? signature)) @@ -200,18 +222,18 @@ public Function Function(string name, IReadOnlyDictionary? b // Each binding must name one of the function's effects, so more // bindings than effects are refused from the count, before any is read. - if (bindings.Count > selected.Count) + if (declared > selected.Count) { - throw JacquardException.Binding(name + ": " + bindings.Count.ToString(CultureInfo.InvariantCulture) + " bindings given, but it uses " + throw JacquardException.Binding(name + ": " + declared.ToString(CultureInfo.InvariantCulture) + " bindings given, but it uses " + selected.Count.ToString(CultureInfo.InvariantCulture) + " effect" + (selected.Count == 1 ? string.Empty : "s") + (selected.Count == 0 ? string.Empty : ": " + string.Join(", ", selected.Keys))); } var chosen = new SortedDictionary(services, StringComparer.Ordinal); int read = 0; - foreach (KeyValuePair binding in bindings) + foreach (KeyValuePair binding in given) { - // A dictionary whose count understated its entries is not read past the bound. + // A dictionary whose count understated its entries was not read past the bound. if (++read > selected.Count) { throw JacquardException.Binding(name + ": more bindings than effects"); diff --git a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs index 7ab2682..2b992b3 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs @@ -914,6 +914,52 @@ public void AnExecutableTheUserMayNotRunIsRefused() Assert.That(fake.Calls("--version"), Is.EqualTo(0)); } + /// A bindings dictionary that records whether the program's lock was held while it was read. + private sealed class Watching : IReadOnlyDictionary + { + private readonly PreparedProgram program; + private readonly Operations service; + + internal Watching(PreparedProgram program, Operations service) + { + this.program = program; + this.service = service; + } + + internal bool? LockHeld { get; private set; } + + public int Count => 1; + + public IEnumerable Keys => throw new NotSupportedException(); + + public IEnumerable Values => throw new NotSupportedException(); + + public Operations this[string key] => throw new NotSupportedException(); + + public bool ContainsKey(string key) => throw new NotSupportedException(); + + public bool TryGetValue(string key, out Operations value) => throw new NotSupportedException(); + + public IEnumerator> GetEnumerator() + { + LockHeld = program.LockHeld; + yield return new("Inventory", service); + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// The caller's bindings dictionary is read before the program's lock is taken, so its code cannot hold the lock. + [Test] + public void TheCallersBindingsAreReadOutsideTheProgramsLock() + { + using FakeCore fake = FakeCore.Effects(At("bindings")); + PreparedProgram program = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + var bindings = new Watching(program, new Operations().On("lookup", Codecs.Int, () => 1L)); + Assert.That(program.Function("first", bindings).Name, Is.EqualTo("first")); + Assert.That(bindings.LockHeld, Is.False); + } + /// /// A function, effect, or operation name over 4,096 chars is refused /// before it is looked up, quoting an excerpt; one of exactly 4,096 From 3a133b398d7e919d630b92afc21b1912fbc7d237 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:03:54 -0400 Subject: [PATCH 18/52] Read the MapBatches row count once Each batch's range was computed from the list's current count, so a list that shrank after validation began made a batch of negative size. The count is read once; a list whose count differs before a later batch is refused. --- dotnet/src/JacquardHost/Sdk/Function.cs | 31 +++++++++++++---- .../JacquardHost.Tests/Sdk/BatchesTests.cs | 33 +++++++++++++++++++ 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Function.cs b/dotnet/src/JacquardHost/Sdk/Function.cs index 7d8c461..71bbe09 100644 --- a/dotnet/src/JacquardHost/Sdk/Function.cs +++ b/dotnet/src/JacquardHost/Sdk/Function.cs @@ -169,7 +169,9 @@ public TResult Call(Codec result, IReadOnlyList args /// unreachable once it is checked, and the batch is encoded and checked /// again just before it runs. Before each batch, and once more after the /// last, the validation observes the token. Validation has no deadline of - /// its own, as in the other SDKs. Nothing is retried. + /// its own, as in the other SDKs. Nothing is retried. The number of rows + /// is read once; a list whose count differs before a later batch is + /// refused. /// /// /// A refusal before any worker started ( @@ -205,11 +207,16 @@ public IReadOnlyList MapBatches( // From the count alone, before any leading argument is copied. Arity(leading.Count + 1L); List empty = Batch(leading, row, rows, 0, 0); - int count = (int)((rows.Count + (long)batchSize - 1) / batchSize); + + // Read once: every batch's range comes from this count, so a list + // that shrinks cannot make a negative batch. + int total = rows.Count; + int count = (int)((total + (long)batchSize - 1) / batchSize); for (int index = 0; index < count; index++) { RefuseIfStopped(empty, cancellationToken); - Validate(leading, row, rows, index, batchSize); + Unchanged(rows, total); + Validate(leading, row, rows, total, index, batchSize); } // A cancellation during the last batch's validation starts no worker. @@ -220,7 +227,8 @@ public IReadOnlyList MapBatches( { try { - (int start, int size) = Range(rows.Count, index, batchSize); + Unchanged(rows, total); + (int start, int size) = Range(total, index, batchSize); completed.AddRange(Run(Batch(leading, row, rows, start, size), cancellationToken).Decode(results)); } catch (JacquardException error) @@ -253,12 +261,23 @@ private void RefuseIfStopped(List empty, CancellationToken cancellationTo } /// Encodes and checks one batch, keeping nothing: the encoding is unreachable once this returns. - private void Validate(IReadOnlyList leading, Codec row, IReadOnlyList rows, int index, int batchSize) + private void Validate(IReadOnlyList leading, Codec row, IReadOnlyList rows, int total, int index, int batchSize) { - (int start, int size) = Range(rows.Count, index, batchSize); + (int start, int size) = Range(total, index, batchSize); Batch(leading, row, rows, start, size); } + /// Refuses rows whose count is no longer . + private static void Unchanged(IReadOnlyList rows, int total) + { + int now = rows.Count; + if (now != total) + { + throw JacquardException.Invalid("rows changed from " + total.ToString(CultureInfo.InvariantCulture) + " to " + + now.ToString(CultureInfo.InvariantCulture) + " during MapBatches"); + } + } + private static (int Start, int Size) Range(int total, int index, int batchSize) { long start = (long)index * batchSize; diff --git a/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs index 80a5130..62ec785 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs @@ -255,6 +255,39 @@ public void MapBatchesNeedsAListFunction() Assert.That(Support.Throws(() => Ints().MapBatches(Codecs.Int, Codecs.Int, Rows(3), 0)).Message, Is.EqualTo("batch size must be positive")); } + /// Rows whose count drops from 10 to 3 once is called. + private sealed class Shrinking : IReadOnlyList + { + private bool shrunk; + + public int Count => shrunk ? 3 : 10; + + public long this[int index] => 1L; + + internal void Shrink() => shrunk = true; + + public IEnumerator GetEnumerator() => Enumerable.Repeat(1L, Count).GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// + /// The number of rows is read once, so batches come from that count; a + /// list that shrinks while the batches are validated is refused before a + /// worker starts, instead of making a batch of negative size. + /// + [Test] + public void RowsThatShrinkAreRefused() + { + var rows = new Shrinking(); + var codec = new Counting(n => n == 1 ? (Action)rows.Shrink : null); + JacquardException error = Support.Throws(() => Ints().MapBatches(Codecs.Int, codec, rows, 4)); + Assert.That(error.Kind, Is.EqualTo(JacquardErrorKind.InvalidArgument)); + Assert.That(error.Message, Is.EqualTo("rows changed from 10 to 3 during MapBatches")); + Assert.That(error.BatchIndex, Is.EqualTo(-1)); + Assert.That(codec.Encodes, Is.EqualTo(4)); + } + /// /// Save writes a new owner-only file and renames it over the target: an /// existing world-writable file becomes owner-only (0600) with the From 96c0fc7b6bb36c6c0f6ac2a4c7a0f022e939eae3 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:18:04 -0400 Subject: [PATCH 19/52] Document the review fixes in the .NET SDK docs The README, LIMITS, and CHANGELOG now say how Save replaces its target, what the cache root and Core identity checks cover and what they do not, the bounded descendant records, the name limit, the refused numeric types, the lazy codec retry, and the execute check; the changelog's reflection claim is narrowed to member reflection. EVIDENCE records the new counts and the regressions shown by undoing each fix. --- CHANGELOG.md | 22 +++++++++--- README.md | 4 +-- docs/EVIDENCE.md | 9 ++--- docs/LIMITS.md | 4 +-- dotnet/README.md | 93 ++++++++++++++++++++++++++++++++++-------------- 5 files changed, 92 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d5079d..3e0f4ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,15 +91,27 @@ with Core's public commands, maps .NET values through explicit `Codec` objects (records through field readers and a constructor function, enumerations, type hierarchies, `Option`, `Unit`, lazy codecs for - recursive types, and `Dynamic` for untyped code) without reflection, binds + recursive types, and `Dynamic` for untyped code) without reflection over + members (enumeration codecs read `Enum` names and values, and hierarchy + codecs check case types with `Type`), binds handlers to exact operation identities with the `HandlerContext` and its `CancellationToken`, checks arguments against the checked signature before a worker starts, and keeps each invocation's raw evidence. `MapBatches` runs explicit batches. There is no `decimal` mapping: money crosses as integer - minor units. Core's commands run on the caller's thread through `poll(2)`, - bounded in time and output, and on Linux through `setsid`, whose process - group is killed when a command exits or is stopped. The cache is owner-only - and verified, every binding derived again from Core's cached output on load. + minor units, and `decimal`, `ulong`, `nuint`, `char`, `float`, and the + other numeric types with no Jacquard type do not compile as arguments. + Caller names over 4,096 chars are refused before they are looked up. + Core's commands run on the caller's thread through `poll(2)`, bounded in + time and output, and on Linux through `setsid`, whose process group is + killed when a command exits or is stopped; without `setsid`, descendants + are recorded from `/proc` within entry and byte caps. Only the cache's + owner may write it: the root, each entry, and each file must be owned by + the current user and not group- or world-writable, and the root's identity + is checked again before each use (its parent directories are not checked). + Every binding is derived again from Core's cached output on load, and a + preparation during which the Core executable or prelude changed is + refused. `Invocation.Save` writes a new owner-only file and renames it over + the target. The four example projects run through a `JacquardExamples` command in `dotnet/examples/`, and CI runs them and the SDK's live tests against the pinned Core. diff --git a/README.md b/README.md index 6e6a54d..86a018a 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ TypeScript SDK supports async service methods, the Go SDK maps native Go types and context cancellation, the Rust SDK maps native Rust types through `record!` and `variants!` declarations, the Java SDK maps records, enums, and sealed interfaces through explicit codecs, and the .NET SDK maps records, enumerations, -and type hierarchies through explicit codecs without reflection, with -`CancellationToken` cancellation. +and type hierarchies through explicit codecs without reflection over members, +with `CancellationToken` cancellation. **Experimental:** the SDK targets Core's pinned `jacquard-host-v0` protocol. Core is installed separately. Each invocation runs one fresh, serial worker; diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md index ef76cc1..e646662 100644 --- a/docs/EVIDENCE.md +++ b/docs/EVIDENCE.md @@ -149,17 +149,18 @@ Recorded against the same `jacquard-core-0.3.0` release asset (linux-x86_64), on Linux 7.0 with the .NET 8.0.427 and 10.0.403 SDKs (runtimes 8.0.33 and 10.0.14). The library still has no runtime dependencies beyond the base class library; the SDK calls the C library's `kill(2)`, `geteuid(2)`, `mkdir(2)`, -and `statx(2)` (`stat(2)` on macOS, compiled but not run), and on Linux runs +`access(2)`, and `statx(2)` (`stat(2)` on macOS, compiled but not run), and on Linux runs Core's commands through util-linux `setsid`, here 2.41.3. The test dependencies are unchanged. | Lane | Result | |---|---| -| .NET SDK offline (`dotnet test test/JacquardHost.Tests -f net8.0` with the .NET 8 SDK pinned as the CI 8.0 job pins it, and `-f net10.0`, without Core) | 181 tests pass on each runtime and 28 live tests are ignored: the adapter's 96 offline tests and 85 SDK ones. Core commands (17, Linux): exit status, output, and added environment, with stdin at end of file; output over its limits read and counted but not kept; a timeout and a cancellation each kill the command and its child; under `setsid` the command keeps its pid and its group and session ids equal it (with `setsid --fork` the test fails); `sh -c 'sleep 300 & echo $!'` succeeds and its child dies, and survives when the exit kill is disabled; a child that closed its output is killed too; a reaped leader's pid held by an unrelated group leader is not killed; a process outside the group holding the command's stdout is neither killed nor waited for; a stop kills a child and a grandchild; without groups a recorded descendant is killed at the exit (and survives with recording off) and a stop kills the tree; a cancellation during the exit cleanup is reported; a missing executable, and one `setsid` cannot exec, are start failures; an escaped flooder holds neither the caller nor more than the kept output; 40 commands leave the descriptor count unchanged, counted at once. Preparation against a fake Core (30, Linux): option and path validation (empty, NUL, and over-long paths refused with excerpts before the file system sees them, a relative path measured absolute); one version check across four threads and two runtimes; prelude discovery once per Core and prelude; output over 8 MiB; the 30-second bound (shortened by a seam) and cancellation killing the command's child; a waiting preparation observing its own cancellation; FIFO and directory sources refused without blocking; sources over 16 MiB refused from their sizes before reading; cancellation between sources, during source validation, during fingerprinting (one 64 KiB piece), between empty prelude files, during the failure cleanup, and as a warm load finishes; prelude walks over 256 MiB, with a FIFO, over the entry cap (lowered by a seam), and 1,500 directories deep; a changed `program.json` binding, a changed Core output, a rewritten digest, and a `program.json` replaced by a link refused; group-writable roots, entries, files, and directories refused and new entries owner-only; a foreign-owned root refused; an executable over 512 MiB refused from its size; source lists over 1,024 refused from their count with no element read; the cleanup stopping at the cap; a symbolic link at the cache root followed to its directory; directories the preparation did not make, and one made at the vacated staging path, left alone. Data (23): signatures, hash groups, and display names; round trips and type conformance; the full Int range, Real bits, and `int` narrowing; records, enumerations, unions, nested `Option`, nullable codecs, tuples, custom codecs, and `Dynamic`; no conversion from `decimal`; recursive types through a lazy codec made once across eight threads, a chain of a million lazy codecs, and one standing for itself refused; the node, text, frame, depth, and count bounds (a list whose items throw is refused from its count, tuples and records before an item is read, a constructor of 20 million fields before it is copied); a raw value 100,000 levels deep refused by its depth; a `Dynamic` over the hard limit refused before it is copied, an endless enumeration after one item past the limit; deep `Dynamic` equality and text without recursion; 64-char excerpts and the 8,192-char message cap, caller exceptions kept as inner exceptions; 100,000 layers of nested unions flattened to one case and round-tripped; a recursive union through a lazy case from eight threads; signature depth 64 accepted, 65 and 5,000 parentheses refused, 100,000 refused by the token cap. Batches and operations (13): argument counts refused before encoding; a throwing codec an invalid argument; cancellation before, during, and after batch validation, each an exact encode count with no worker; every batch validated and the first encoded again; no field or return value that could keep an encoding; a shared graph from a custom codec measured before its type is checked; handlers' decoding, results, failures, context, and token, and no handler starting once the call stopped during decoding. Examples (2): usage errors exit 2; the store removes only the temporary file it created | -| .NET SDK live (`JACQUARD_REQUIRE_LIVE=1`, `JACQUARD_BIN` and `JACQUARD_PRELUDE` set) | 20 tests pass on .NET 8 and on .NET 10: quickstart calls, `ToDynamic`, `Render`, and an owner-only saved inspection; the README examples; the 64-row triage oracle in batches of 8, with a codec failing on the second batch keeping the first and its exception; local refusals before any worker (a field over the Int range named by path, an over-deep call, a wrong type, a wrong count); missing handlers, a misspelled operation, and a 100-name service refused at binding with no handler run; unknown completion once with raw evidence; no handler starting once a real invocation stopped during decoding; a handler's token cancelled when the caller cancels; the 25-case planner oracle; cache reuse without Core, a byte-exact new entry, and corruption refused; a 2,500-`Option` signature left unbound with `safe` working cold and warm; a redirected name in `program.json` refused; the full Int range, Real bits, and nested options; spawn failure and cancellation; output overflow at batch 0; unit results kept after a later batch fails (E1602); too many bindings refused from the count; a throwing result codec keeping the result and its exception, a null mapping message, and an over-long save path; prelude contents keying the cache, and two concurrent preparations publishing one entry | +| .NET SDK offline (`dotnet test test/JacquardHost.Tests -f net8.0` with the .NET 8 SDK pinned as the CI 8.0 job pins it, and `-f net10.0`, without Core) | 195 tests pass on each runtime and 28 live tests are ignored: the adapter's 96 offline tests and 99 SDK ones. Core commands (20, Linux): exit status, output, and added environment, with stdin at end of file; output over its limits read and counted but not kept; a timeout and a cancellation each kill the command and its child; under `setsid` the command keeps its pid and its group and session ids equal it (with `setsid --fork` the test fails); `sh -c 'sleep 300 & echo $!'` succeeds and its child dies, and survives when the exit kill is disabled; a child that closed its output is killed too; a reaped leader's pid held by an unrelated group leader is not killed; a process outside the group holding the command's stdout is neither killed nor waited for; a stop kills a child and a grandchild; without groups a recorded descendant is killed at the exit (and survives with recording off) and a stop kills the tree; against a fake `/proc` (a seam, pids above Linux's largest), one record reads exactly its entry cap (20 of 50) and its byte cap (5 of 50 entries at 5 KiB), and 1,024 bytes of each 64 KiB `stat` file, a cancellation after the third read ends the record there (the stop's last record then reads its cap), and a cancellation during the second of ten exit kills ends them; a cancellation during the exit cleanup is reported; a missing executable, and one `setsid` cannot exec, are start failures; an escaped flooder holds neither the caller nor more than the kept output; 40 commands leave the descriptor count unchanged, counted at once. Preparation against a fake Core (37, Linux): option and path validation (empty, NUL, and over-long paths refused with excerpts before the file system sees them, a relative path measured absolute); one version check across four threads and two runtimes; prelude discovery once per Core and prelude; output over 8 MiB; the 30-second bound (shortened by a seam) and cancellation killing the command's child; a waiting preparation observing its own cancellation; FIFO and directory sources refused without blocking; sources over 16 MiB refused from their sizes before reading; cancellation between sources, during source validation, during fingerprinting (one 64 KiB piece), between empty prelude files, during the failure cleanup, and as a warm load finishes; prelude walks over 256 MiB, with a FIFO, over the entry cap (lowered by a seam), and 1,500 directories deep; a changed `program.json` binding, a changed Core output, a rewritten digest, and a `program.json` replaced by a link refused; group-writable roots, entries, files, and directories refused and new entries owner-only; a foreign-owned root refused; an executable over 512 MiB refused from its size; source lists over 1,024 refused from their count with no element read; the cleanup stopping at the cap; a symbolic link at the cache root followed to its directory; directories the preparation did not make, and one made at the vacated staging path, left alone; the cache root renamed away and replaced (a seam) before staging, publishing, and a cold or warm load, and after a load read the entry, each refused; the executable replaced by a new file after `--version`, `check`, `hash`, and `run`, a prelude file rewritten after `check`, and a file added to the prelude after `run`, each refused with nothing published; the version check run again for the same executable bytes with a new modification time; an executable only its group and others may run refused by `access(2)` before it runs; a bindings dictionary read without the program's lock held; function, effect, binding, and operation names over 4,096 chars refused and names of exactly 4,096 looked up. Data (25): signatures, hash groups, and display names; round trips and type conformance; the full Int range, Real bits, and `int` narrowing; records, enumerations, unions, nested `Option`, nullable codecs, tuples, custom codecs, and `Dynamic`; `ulong` (2^53 + 1), `nuint`, `char`, `float`, `Half`, `decimal`, `Int128`, `UInt128`, and `BigInteger` conversions and `Dynamic.Of` overloads obsolete as errors and, reached by reflection, refused, while the smaller integer types and `uint` still convert to `Int`; caller names over 4,096 chars refused before lookup (public, made, dynamic, record, and operation names); a lazy codec whose factory threw twice made on the third use and never again across 16 threads; two 64-level shared tuple graphs (2^64 paths) compared equal and unequal at once; recursive types through a lazy codec made once across eight threads, a chain of a million lazy codecs, and one standing for itself refused; the node, text, frame, depth, and count bounds (a list whose items throw is refused from its count, tuples and records before an item is read, a constructor of 20 million fields before it is copied); a raw value 100,000 levels deep refused by its depth; a `Dynamic` over the hard limit refused before it is copied, an endless enumeration after one item past the limit; deep `Dynamic` equality and text without recursion; 64-char excerpts and the 8,192-char message cap, caller exceptions kept as inner exceptions; 100,000 layers of nested unions flattened to one case and round-tripped; a recursive union through a lazy case from eight threads; signature depth 64 accepted, 65 and 5,000 parentheses refused, 100,000 refused by the token cap. Batches and operations (15): argument counts refused before encoding; a throwing codec an invalid argument; cancellation before, during, and after batch validation, each an exact encode count with no worker; every batch validated and the first encoded again; no field or return value that could keep an encoding; a shared graph from a custom codec measured before its type is checked; rows that shrink from 10 to 3 during validation refused after one batch's four encodings; `Save` over a 0666 file leaving a 0600 file, and over a symbolic link replacing the link and leaving its target unchanged; handlers' decoding, results, failures, context, and token, and no handler starting once the call stopped during decoding. Examples (2): usage errors exit 2; the store removes only the temporary file it created | +| .NET SDK live (`JACQUARD_REQUIRE_LIVE=1`, `JACQUARD_BIN` and `JACQUARD_PRELUDE` set) | 20 tests pass on .NET 8 and on .NET 10 (unchanged in this review round): quickstart calls, `ToDynamic`, `Render`, and an owner-only saved inspection; the README examples; the 64-row triage oracle in batches of 8, with a codec failing on the second batch keeping the first and its exception; local refusals before any worker (a field over the Int range named by path, an over-deep call, a wrong type, a wrong count); missing handlers, a misspelled operation, and a 100-name service refused at binding with no handler run; unknown completion once with raw evidence; no handler starting once a real invocation stopped during decoding; a handler's token cancelled when the caller cancels; the 25-case planner oracle; cache reuse without Core, a byte-exact new entry, and corruption refused; a 2,500-`Option` signature left unbound with `safe` working cold and warm; a redirected name in `program.json` refused; the full Int range, Real bits, and nested options; spawn failure and cancellation; output overflow at batch 0; unit results kept after a later batch fails (E1602); too many bindings refused from the count; a throwing result codec keeping the result and its exception, a null mapping message, and an over-long save path; prelude contents keying the cache, and two concurrent preparations publishing one entry | | .NET projects live | 3 further tests pass on both runtimes: all four `JacquardExamples` commands, including duplicate fulfillment, `--inspect` (owner-only), explicit delivery, a failed delivery printing no count, and usage errors; the command in a fresh process exiting 1 with E1613; and the three fulfillment faults recovered explicitly across restart, each receipt delivered once. The CI step's four projects and recovery commands also pass locally | -| Totals | 209 tests on each runtime with Core required, none ignored; 181 pass and 28 are ignored without Core | +| Totals | 223 tests on each runtime with Core required, none ignored; 195 pass and 28 are ignored without Core (the .NET 8 lane built with the 8.0.427 SDK through a temporary `global.json`, as CI pins it) | | Regressions checked by mutation (.NET 8) | Each guard was removed in turn and the named tests run; all 42 mutations failed them: the command's exit group kill, its stop kill, `setsid` without forking, the pid-reuse check, the recorded-descendant kill, the stop check after the exit cleanup, and closing the output streams; the source kind check (the FIFO test failed after its one-minute wait), source count, source validation stop, source ceiling, tree entry and byte caps, the executable's size check and per-piece stop, the derived-binding check, the owner, mode, and owner-only checks, the cleanup's stop and cap, leaving the vacated staging path, and the final stop check; the encoder's node budget, the decoder's measurement, the text charge and length check, the list, record, and constructor counts, measuring before the type check, counting arguments before encoding, the batch stop checks before and after validation, the handler's stop check (offline and live), the `Dynamic` limit, excerpts, the message cap, wrapping a field reader's and a record constructor's exceptions, the lazy codec's cycle check (the test then never returned and was stopped), and the store's exclusive temporary file. Separately, checking the cancellation in an exception filter, which .NET runs before the preparation's cleanup, failed the failure-cleanup test until the check moved into the handler | +| Review-round regressions (.NET 8) | Each fix was undone in turn and its tests run; all 16 undoings failed them: `Invocation.cs`, `Builtins.cs`, and `Function.cs` as they were before the fixes (the old save left the existing file at 0666; the old lazy codec never ran its factory again; the old batch count made a batch of size -1, reported as an `OverflowException` from the codec), `Dynamic.cs` without the pair set (the equality test timed out after its one-minute wait), and guards turned off: the cache root recheck, the executable and prelude recheck, the identity in the version key, the record's entry cap, byte cap, and per-file read cap, the stop check in a record and in the exit kills, the name limit, the `ulong` refusal, `access(2)` (back to any execute bit), and reading the bindings outside the lock | | Build | `TreatWarningsAsErrors`, nullable reference types, and the `8.0-recommended` analyzers are clean with both SDKs | | Unchanged lanes | The adapter's 96 offline and 5 live tests (the README-example check now covers all four examples), the 21-recording kit replay, `scripts/core_pin.py check`, `tests/test_core_pin.py`, and `scripts/check_distribution.py` (the source archive now carries the SDK and the example command) pass | diff --git a/docs/LIMITS.md b/docs/LIMITS.md index da5cee6..4d25a81 100644 --- a/docs/LIMITS.md +++ b/docs/LIMITS.md @@ -29,8 +29,8 @@ also updates this document. | .NET worker process tree, not group | .NET cannot start the worker in its own process group, so the .NET host kills a process tree. At expiry, cancellation, or disposal of a running worker, it calls `Process.Kill(true)`, which stops the worker, lists its children from the process table, kills it, and does the same for each child. A descendant that was already reparented keeps running: one whose parent exited, or one that daemonized. So does one started after its parent was listed. Descendants are found and signalled by pid, with nothing pinning their identity: one that exits, and is reaped once the worker is killed, between the listing and the signal can have its pid reused by an unrelated process, which is then signalled (the Java host signals process handles, and the Rust and Go hosts a process group). Listing the process table takes time proportional to the number of processes on the machine. The runtime reaps the worker as soon as it exits. A descendant still holding stdout or stderr after the exit is detected and, after a one-second grace, reported as a carrier loss, as the Go and Rust hosts report it; it is not stopped. .NET reports a signal as exit status 128 + its number, so only the host's own SIGKILL is reported as a signal. | An OS-level containment mechanism (for example a cgroup or job object) with its own evidence, or a .NET process-group API. | | .NET pipe I/O goes through `poll(2)` | .NET's pipe streams have no read or write that a deadline or a cancellation can interrupt, so the .NET host never calls one that can block: the caller's thread asks the C library's `poll(2)` whether a pipe is ready, with a poll timeout of at most 5 ms, rechecking the deadline, the `CancellationToken`, and disposal between polls, and then reads only bytes already in the pipe or writes at most `PIPE_BUF` bytes (4096 on Linux, 512 on macOS), which a writable pipe accepts at once. No thread is left blocked on a worker pipe, even while an escaped descendant holds stdin without reading or floods stdout or stderr. The `pollfd` layout and event bits are the same on Linux and macOS; elsewhere, including Windows, the host does not start a worker and reports a spawn failure. .NET opens its own descriptors close-on-exec, but passes the worker any descriptor the host process itself inherited without that flag. | A .NET API for interruptible or non-blocking process pipes, and CI on each added platform. | | .NET handler cancellation is cooperative | Each .NET handler runs on its own background thread. At the deadline or on cancellation the host kills the worker's process tree, closes its input, and sends no late response. `HandlerContext.IsCancelled` turns true (it reads the deadline exactly), and stays true once the invocation ends. `HandlerContext.CancellationToken` is cancelled when the call stops or ends, never earlier (a lifetime of any length does not cancel it before its deadline). While a handler runs, the caller's thread rechecks the deadline and the caller's token between poll waits of at most 5 ms; that is the requested poll timeout, not a bound on the interval between checks, which reading pending stderr (up to 1 MiB per pass) and thread scheduling can lengthen. Whichever check sees the stop, the token's cancellation starts before the worker is killed; otherwise it starts shortly after the invocation ends. The cancellation runs on a short-lived thread the host starts for it, one per invocation that gave a handler a token, so it does not wait for the shared thread pool. If no thread can be created or started, the host queues it as a pool work item, which runs only when a pool thread is free, so a starved pool delays it. If the work item cannot be queued either, the host gives up: the token is never cancelled (`HandlerContext.IsCancelled` still turns true), `InvocationResult.HandlerTokenNotCancelled` is set, and a handler waiting only on the token keeps waiting; `Invoke` still returns its result and disposes the worker. Callbacks a handler registers on that token run on that thread, which the host never waits for; a callback that blocks holds that thread, and anything a callback throws is discarded. A callback registered after the token is cancelled runs at once on the handler's own thread and throws there, as with any `CancellationToken`. A handler that ignores both keeps running on its thread until it returns. No outside action is stopped or rolled back. A stack overflow in a handler ends the process, as it does any .NET code. | A separately enforced operation lifetime with explicit completion and reconciliation evidence. | -| .NET SDK Core command descendants | Each Core `--version`, `check`, `hash`, and `run` command must exit within 30 seconds and keeps at most 8 MiB of stdout and 4 KiB of stderr; cancelling the token stops it. Its output is read only after `poll(2)` reports it ready, on the caller's thread. On Linux with a `setsid` at `/usr/bin/setsid` or `/bin/setsid` (util-linux's, or a compatible one such as BusyBox's; any executable there is used), the command runs as `setsid -- `, which execs without forking, so Core keeps its pid and leads its own process group; when `setsid` cannot exec Core (exit status 126 or 127 with its own `setsid: ` message) the command is reported as a start failure. When the command exits, and on a timeout or cancellation, the host sends `kill(-pgid, SIGKILL)` to that group. Once .NET has reaped Core, a live process holding Core's pid means the id was reused and nothing is killed; a process that took Core's pid after a full pid wraparound, made its own group, and forked between that check and the kill could still have that group killed, the same window as the Rust SDK's `kill(-pgid)`. A descendant that leaves the group (calls `setsid`) is neither detected nor killed; the host does not wait for the end of the output after the exit, and closes its ends, so that descendant's next write fails. Without `setsid` (Linux without one at those paths), descendants are recorded from `/proc` about every 10 ms by pid and start time, every recorded descendant still alive with that start time is killed when the command returns (the start time is read just before the kill, a short window in which a reused pid could be signalled, as with the JDK's process handles), and a stop also calls `Process.Kill(true)`; that fallback misses fast fork-and-exit chains. macOS has no `/proc`, so there only the tree kill on a stop applies. The .NET worker does not run in its own group (see the worker row above). | An OS-level containment mechanism, or a .NET process-group API, with its own evidence. | -| .NET SDK cache and source reads | A cache entry is verified against the file digests in its `program.json`, and every binding is derived again from Core's cached raw output, so an accidental or partial change is refused. No record inside the cache resists someone who can rewrite it: an output, its recorded digest, and the bindings changed together load. So the cache's owner is trusted and only the owner may write it: preparation refuses a cache root, an entry, or any file or directory in one that the current user does not own or that its group or others may write (read with `statx(2)` on Linux and `stat(2)` on macOS, without following links), on create and on load, and new entries are owner-only. A symbolic link at the cache root is followed once and the directory it names checked; nothing inside it is followed, and the cache root's parent directories are not checked. The macOS status calls are compiled but not exercised by any test. A failed preparation's cleanup removes at most 100,000 entries of its staging directory, created exclusively, and ends at a cancellation, so a stopped preparation leaves an unpublished `prepare-*` directory in the cache root; it is never loaded, and nothing removes it later (a name and an age cannot show that the SDK made a directory or that no preparation is using it), so remove leftovers yourself when no preparation is running. At most 1,024 sources are taken, refused from the list's count before any element is read, and they must be regular files totalling at most 16 MiB, the Core executable at most 512 MiB, and the prelude and cache walks read at most 100,000 entries and 256 MiB of regular files under one root, each file checked from its size before it is read and again while it is read; a source or cache file replaced by a FIFO (or, in the cache, by a link) between that check and its open can still block or redirect the open, which a cancellation does not end. | An open that cannot block on a FIFO or follow a link. | +| .NET SDK Core command descendants | Each Core `--version`, `check`, `hash`, and `run` command must exit within 30 seconds and keeps at most 8 MiB of stdout and 4 KiB of stderr; cancelling the token stops it. Its output is read only after `poll(2)` reports it ready, on the caller's thread. On Linux with a `setsid` at `/usr/bin/setsid` or `/bin/setsid` (util-linux's, or a compatible one such as BusyBox's; any executable there is used), the command runs as `setsid -- `, which execs without forking, so Core keeps its pid and leads its own process group; when `setsid` cannot exec Core (exit status 126 or 127 with its own `setsid: ` message) the command is reported as a start failure. When the command exits, and on a timeout or cancellation, the host sends `kill(-pgid, SIGKILL)` to that group. Once .NET has reaped Core, a live process holding Core's pid means the id was reused and nothing is killed; a process that took Core's pid after a full pid wraparound, made its own group, and forked between that check and the kill could still have that group killed, the same window as the Rust SDK's `kill(-pgid)`. A descendant that leaves the group (calls `setsid`) is neither detected nor killed; the host does not wait for the end of the output after the exit, and closes its ends, so that descendant's next write fails. Without `setsid` (Linux without one at those paths), descendants are recorded from `/proc` about every 10 ms by pid and start time, and every recorded descendant still alive with that start time is killed when the command returns (the start time is read just before the kill, a short window in which a reused pid could be signalled, as with the JDK's process handles). Each record reads at most 16,384 `/proc` entries, 4 MiB of `stat` files, and 1 KiB of each, at most 16,384 descendants are kept, and a record ends at the deadline or a cancellation, checked before each entry; the kills at the exit check them before each descendant, so a stop that lands there leaves the descendants not yet killed running. On a timeout or cancellation, a last record bounded by the same caps (not ended by the stop) is made, then the command is killed, then every recorded descendant; .NET's `Process.Kill(true)` is not used on Linux. Descendants past the caps, and fast fork-and-exit chains between records, are missed. macOS has no `/proc`, so there only `Process.Kill(true)` on a stop applies. The .NET worker does not run in its own group (see the worker row above). | An OS-level containment mechanism, or a .NET process-group API, with its own evidence. | +| .NET SDK cache and source reads | A cache entry is verified against the file digests in its `program.json`, and every binding is derived again from Core's cached raw output, so an accidental or partial change is refused. No record inside the cache resists someone who can rewrite it: an output, its recorded digest, and the bindings changed together load. So the cache's owner is trusted and only the owner may write it: preparation refuses a cache root, an entry, or any file or directory in one that the current user does not own or that its group or others may write (read with `statx(2)` on Linux and `stat(2)` on macOS, without following links), on create and on load, and new entries are owner-only. A symbolic link at the cache root is followed once and the directory it names checked; nothing inside it is followed. The root is then used by its path: its device and inode, read at the owner check, are compared again before the staging directory is made, before publishing, and before and after each load, and a replaced root is refused. Its parent directories are not checked, so another user who may write a parent can still replace the root in the few system calls between one of those comparisons and the step that follows it; a cache under directories only its owner can write is not exposed. The Core executable and every prelude file and directory are fingerprinted by path, and Core opens them by path later: each one's identity (device, inode, size, mode, owner, and modification and status-change times) is read as it is fingerprinted and compared after Core's commands finish (inside the version check and prelude discovery, before a new entry is published, and before an existing one is loaded), and any change refuses the preparation. Not detected: a file swapped and then restored with the same inode and times, which an unprivileged user can do only within the file system's timestamp granularity, since writing or renaming a file updates its status-change time (on Linux from a coarse clock, a few milliseconds); files Core loads besides the executable and prelude (its shared libraries and loader); and any change after the last comparison, since a prepared program's workers later run the executable by path without checking it again. The macOS status calls are compiled but not exercised by any test. A failed preparation's cleanup removes at most 100,000 entries of its staging directory, created exclusively, and ends at a cancellation, so a stopped preparation leaves an unpublished `prepare-*` directory in the cache root; it is never loaded, and nothing removes it later (a name and an age cannot show that the SDK made a directory or that no preparation is using it), so remove leftovers yourself when no preparation is running. At most 1,024 sources are taken, refused from the list's count before any element is read, and they must be regular files totalling at most 16 MiB, the Core executable at most 512 MiB, and the prelude and cache walks read at most 100,000 entries and 256 MiB of regular files under one root, each file checked from its size before it is read and again while it is read; a source or cache file replaced by a FIFO (or, in the cache, by a link) between that check and its open can still block or redirect the open, which a cancellation does not end. | An open that cannot block on a FIFO or follow a link; directory-relative (`openat`) access pinned to the checked root; Core reading descriptors the host opened and verified. | | .NET SDK cancellation is by token only | Preparation, Core's commands, `Call`, `Invoke`, and `MapBatches` observe the `CancellationToken` they take. `Thread.Interrupt` is not a stop signal: the SDK's own waits are `poll(2)` sleeps, which neither end at an interrupt nor consume it. The API is synchronous; there are no `Task`-returning variants. | `Task`-returning variants with the same bounds, if callers need them. | | .NET SDK signature depth | A checked function whose signature nests types deeper than 64 levels (`max_json_depth`, the protocol's value-depth limit), names more than 4,096 types, or has more than 16,384 tokens is left unbound, like a polymorphic one: preparation succeeds, the other functions bind, and binding it fails with a `Binding` exception naming the limit. The SDK parses no operation signatures (Core's `check --print-sigs` prints none). | None needed for depth, since a deeper value cannot cross the protocol; Core's structured host interface exporter for the rest. | | .NET SDK record fields are declared, not exported | A record codec's fields, in the order given, are a constructor's fields; the pinned CLI exports no field types, so a wrong order is not detected locally, and Core refuses it only where the swapped fields' types differ. | Core's structured host interface exporter, so bindings can be generated or checked. | diff --git a/dotnet/README.md b/dotnet/README.md index b61446b..f11db61 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -147,16 +147,34 @@ entry, or any file or directory in one that the current user does not own or that its group or others may write, both when it creates an entry and when it loads one, and it makes each new entry's directories and files owner-only (Core's store included). The owner and mode come from `statx(2)` on Linux and -`stat(2)` on macOS, without following links. The cache root's parent -directories are not checked. The `--version` check and prelude discovery run -once per Core executable and prelude in the process, shared by every -`Runtime` and thread. Each call runs one fresh worker. Prepare only +`stat(2)` on macOS, without following links. The cache root is then used by +its path, so its device and inode, read at that check, are compared again +before the staging directory is made, before an entry is published, and +before and after each load; a root renamed away and replaced is refused. Its +parent directories are not checked, so a root replaced between one of those +checks and the step that follows it is not detected: keep the cache under +directories only you can write. + +The executable and every prelude file and directory are fingerprinted by +path, and Core opens them by path afterwards. Each one's identity (device, +inode, size, mode, owner, and modification and status-change times) is read +as it is fingerprinted and must be unchanged once Core's commands have +finished: inside the version check and prelude discovery, before a new entry +is published, and before an existing one is loaded. A change refuses the +preparation, and a version check or prelude discovery it affected is not +kept. The `--version` check runs once per executable fingerprint (its digest +and its identity) and prelude discovery once per that fingerprint and the +prelude's digest, in the process, shared by every `Runtime` and thread. What +this does not cover is in [LIMITS](../docs/LIMITS.md). Each call runs one +fresh worker. Prepare only first-party declaration sources: preparation runs Core's `run` command, without service bindings, to populate the store. `new Runtime()` uses `$JACQUARD_BIN` and `$JACQUARD_PRELUDE`; an empty option -means unset. The runtime refuses a Core whose `--version` differs from the -pinned release. Each Core command must finish within 30 seconds, and keeps at +means unset. The Core executable must be a regular file this user may execute +(`access(2)` with `X_OK`, not merely any execute bit); one this user may not +run is refused with a message saying so. The runtime refuses a Core whose +`--version` differs from the pinned release. Each Core command must finish within 30 seconds, and keeps at most 8 MiB of stdout and 4 KiB of stderr; see [Core's commands](#cores-commands). Cancelling the token `Prepare` takes kills the running command and its process group, and fails the preparation; a preparation waiting for another @@ -164,17 +182,27 @@ thread's version check or prelude discovery stops waiting too. The SDK observes only the token: `Thread.Interrupt` is not a stop signal, and the SDK's own waits (`poll(2)` sleeps) do not consume it. -Arguments are `Argument`s. A `long` or `int` converts implicitly to an `Int`, -a `double` to a `Real`, a `string` to a `Text`, and a `bool`, a protocol -`Hash`, a protocol `Value`, or a `Dynamic` to itself; anything else, lists and -records included, is `codec.Arg(value)`. There is deliberately no conversion -from `decimal` (see [data](#data)). `Call` decodes the result with the codec -you pass; `Invoke` keeps the whole `Invocation`. Overloads that take an -argument list and a `CancellationToken` kill the worker when it is cancelled. +Arguments are `Argument`s. A `long`, `int`, `uint`, `short`, `ushort`, +`sbyte`, or `byte` converts implicitly to an `Int`, a `double` to a `Real`, a +`string` to a `Text`, and a `bool`, a protocol `Hash`, a protocol `Value`, or +a `Dynamic` to itself; anything else, lists and records included, is +`codec.Arg(value)`. A `ulong`, `nuint`, `char`, `float`, `Half`, `decimal`, +`Int128`, `UInt128`, or `BigInteger` does not compile as an argument or in +`Dynamic.Of`: those conversions are marked obsolete as errors, with the +reason, because C# would otherwise convert a `ulong` through `double` (2^53 + 1 +would arrive as 2^53) and a `char` to an `Int` (see [data](#data)). `Call` +decodes the result with the codec you pass; `Invoke` keeps the whole +`Invocation`. Overloads that take an argument list and a `CancellationToken` +kill the worker when it is cancelled. `MapBatches` reads the number of rows +once and refuses a list whose count differs before a later batch. `Invocation.Save` writes the frames and evidence, including argument and -result values; a new file is created owner-only, and an existing file keeps -its permissions. The API is synchronous; there are no `Task`-returning -variants. +result values, to a new file created exclusively in the target's directory, +made owner-only (0600) and owned by you, and then renames it over the path: +whatever was there is replaced, never written through, so an existing file's +permissions and owner are not kept and a symbolic link there is replaced +rather than followed. The directory must be writable; in a sticky directory +such as `/tmp`, a file another user owns cannot be replaced and the save +fails. The API is synchronous; there are no `Task`-returning variants. ## Data @@ -183,7 +211,8 @@ Values map through explicit `Codec` objects, built with the factories on field in the constructor's positional order, a codec and a function that reads it, then a function that builds the record from the decoded fields. Records are mapped without reflection (enumerations use `Enum.GetValues` and -`Enum.GetName`), and there is no attribute or source generator: a codec is an +`Enum.GetName`, and a type hierarchy checks its cases' types with `Type`), and +there is no attribute or source generator: a codec is an ordinary immutable, thread-safe object, so the library builds clean under the trimming and ahead-of-time analyzers (`IsAotCompatible`; a trimmed or native-AOT publish is not tested), and a binding that does not follow a @@ -203,7 +232,7 @@ actual wire types too. | a record (`Codecs.Record`, `Codecs.Field`) | one constructor, named by public name, its fields in the order given | | an enumeration (`Codecs.Enum`) | nullary constructors by convention: member `PageNow` is the constructor `PageNow` (store spelling `page-now`), or an explicit map | | a type hierarchy (`Codecs.Union`) | the constructors of one data type: each case is a record, enumeration, or union codec of a subtype | -| a recursive type (`Codecs.Lazy`) | a codec made once, thread-safely, on first use, so a field can refer to its own type's codec | +| a recursive type (`Codecs.Lazy`) | a codec made once, thread-safely, on first use, so a field can refer to its own type's codec; a factory that throws keeps nothing and runs again on the next use | | `Dynamic` (`Codecs.Dynamic`) | any of the above by public name, for code with no application type | | protocol `Value` (`Codecs.Value`) | passed through unchanged | @@ -270,9 +299,9 @@ when a value is encoded, and a result built by a different constructor fails to decode. `null` is never a Jacquard value (except through a nullable codec). **Money crosses as integer minor units** (cents, say) in an `Int`: Jacquard's `Int` is 63-bit and its `Real` is a binary64 `double`, and there is -no decimal type. The SDK has no `decimal` codec and no conversion from -`decimal`, so a `decimal` is never converted silently; a test checks that none -exists. +no decimal type. The SDK has no `decimal` codec, and a `decimal` argument or +`Dynamic.Of(decimal)` does not compile (the conversion is obsolete as an error +saying why), so a `decimal` is never converted silently; a test checks both. Every encoding is bounded by the selected limits as it goes: the JSON depth the wire form needs, `max_collection_items` for each list, tuple, and @@ -295,7 +324,9 @@ host measures a handler's result the same way. A value returned by `ProgramData.Encode` is not measured again. Raw values, decoded values, nested unions (flattened on first use to their distinct cases), chains of lazy codecs, and a `Dynamic`'s equality and text are walked iteratively, so -none can exhaust the call stack. `PreparedProgram.Data` gives each public +none can exhaust the call stack; equality compares each pair of shared +sub-values once, so values that share sub-values compare in time +proportional to their distinct nodes. `PreparedProgram.Data` gives each public name's exact identity. `ProgramData.Make` builds a `Dynamic.Variant` after checking that its constructor exists, and `Invocation.ToDynamic` decodes any result into `Dynamic`. A `Dynamic` list, tuple, or variant with more items @@ -371,7 +402,8 @@ naming the value, and while a value is decoded, by `ProgramData.Decode`, either way your exception is its `InnerException`. Names and exception messages the SDK quotes (a constructor, type, field, effect, operation, or executable name, or your exception's message) appear as an excerpt of at most -64 chars and its length. A message you pass to `Encoder.Invalid` or +64 chars and its length. A function, effect, operation, public, or constructor +name longer than 4,096 chars is refused before it is looked up or hashed. A message you pass to `Encoder.Invalid` or `Decoder.Mapping` is kept as written, and no message is longer than `JacquardException.MessageLimit` (8,192 chars), the start kept and the whole length noted, once, however many prefixes are joined to it. Some failures @@ -722,10 +754,17 @@ Without `setsid` (Linux without one at those paths), the command's descendants are recorded from `/proc` about every 10 ms, by pid and start time, and every recorded descendant still alive with that start time is killed when the command returns (the start time is read just before the -kill, as the JDK's process handles do); a stop also calls -`Process.Kill(true)`. A descendant started and reparented between two records -keeps running. macOS has no `/proc`, so nothing is recorded there: only the -tree kill on a stop applies. +kill, as the JDK's process handles do). Each record reads at most 16,384 +`/proc` entries, 4 MiB of `stat` files, and 1 KiB of each, at most 16,384 +descendants are kept, and a record ends at the deadline or a cancellation, +checked before each entry; the kills at the exit check them before each +descendant, so a stop there leaves the rest running. On a timeout or +cancellation, a last record bounded by the same caps (but not ended by the +stop) is made, then the command is killed, then every recorded descendant; +.NET's `Process.Kill(true)`, which reads every process on the machine, is not +used. A descendant past the caps, or one started and reparented between two +records, keeps running. macOS has no `/proc`, so nothing is recorded there: +only .NET's tree kill on a stop applies. ## Conformance kit From 985338d872480a2aadb3417977cc28aa7c7c5973 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:43:37 -0400 Subject: [PATCH 20/52] Limit each /proc stat read to the bytes left in the record's cap --- dotnet/src/JacquardHost/Sdk/CoreCommand.cs | 18 +++++----- .../Sdk/CoreCommandTests.cs | 33 +++++++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs index 55306b9..b1bc21d 100644 --- a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs +++ b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs @@ -49,7 +49,8 @@ namespace JacquardLang.Host.Sdk; /// and every recorded descendant still alive with that start time is killed /// when the command returns. Each record reads at most /// process entries and -/// bytes (at most of each stat file), keeps +/// bytes in all (at most of each stat file, +/// and never more than the bytes left), keeps /// at most descendants in all, and observes the /// deadline and the token before each entry, ending at once on a stop; the /// kills at the exit observe them before each descendant too, so a stop @@ -327,7 +328,8 @@ internal void Sample(int root, long expires, CancellationToken cancellationToken break; } - if (ReadStat(entry, ref bytes, stoppable: !final) is { } stat) + // Never more than the bytes left, so a record reads at most its cap. + if (ReadStat(entry, ref bytes, (int)Math.Min(StatBytes, byteCap - bytes), stoppable: !final) is { } stat) { if (!children.TryGetValue(stat.Parent, out List<(int, string)>? list)) { @@ -384,7 +386,7 @@ internal void KillAlive(bool stoppable, long expires, CancellationToken cancella return; } - if (ReadStat(Path.Combine(ProcRoot, pid.ToString(CultureInfo.InvariantCulture)), ref bytes, stoppable) is { } stat && stat.Start == start && !stat.Zombie) + if (ReadStat(Path.Combine(ProcRoot, pid.ToString(CultureInfo.InvariantCulture)), ref bytes, StatBytes, stoppable) is { } stat && stat.Start == start && !stat.Zombie) { Unix.Kill(pid); } @@ -443,16 +445,16 @@ private static bool IsDigits(string text) /// /// A process's pid, parent, start time, and whether it is a zombie, from /// at most of /proc/<pid>/stat, - /// read after the last ) (the command name may hold any byte); - /// null when it cannot be read. Adds the bytes read to - /// . + /// read after the last ) (the command name may hold any byte), + /// and at most ; null when it cannot be read. + /// Adds the bytes read to . /// - private static (int Pid, int Parent, int Group, string Start, bool Zombie)? ReadStat(string directory, ref long bytes, bool stoppable = false) + private static (int Pid, int Parent, int Group, string Start, bool Zombie)? ReadStat(string directory, ref long bytes, int limit = StatBytes, bool stoppable = false) { try { string path = Path.Combine(directory, "stat"); - byte[] buffer = new byte[StatBytes]; + byte[] buffer = new byte[Math.Clamp(limit, 0, StatBytes)]; int read = 0; using (var stream = new FileStream(path, new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Share = FileShare.ReadWrite, BufferSize = 0 })) { diff --git a/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs index dc9a5f7..7a770b2 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs @@ -358,6 +358,39 @@ public void AProcRecordReadsAtMostItsCaps() } } + /// + /// The byte cap holds when it is not a multiple of the 1,024 bytes read + /// of each stat file: with a 1,025-byte cap, a record reads 1,024 + /// bytes and then the 1 byte left, never a second full file. + /// + [Test] + public void AProcRecordNeverReadsPastANonAlignedByteCap() + { + CoreCommand.WithoutGroups = true; + CoreCommand.ProcRoot = FakeProc("proc", 10, 1); + CoreCommand.ProcByteCap = 1025; + var reads = new List(); + int records = 0; + using var cancel = new CancellationTokenSource(); + CoreCommand.StatReadHook = (path, bytes, stoppable) => + { + if (stoppable && records == 0) + { + reads.Add(bytes); + } + }; + CoreCommand.RecordedHook = pids => + { + if (++records == 1) + { + cancel.Cancel(); + } + }; + CarrierException error = Support.Throws(() => Run(Waiting(Path.Combine(root, "never")), cancellationToken: cancel.Token))!; + Assert.That(error.IsAborted, Is.True); + Assert.That(reads, Is.EqualTo(new[] { 1024, 1 })); + } + /// /// A cancellation that lands during a record ends it at the next entry: /// of 50 fake entries, the record stops after the third read, and the From 6747d0fae1cd50ee8737b56f15094f21446de51c Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:44:17 -0400 Subject: [PATCH 21/52] Freeze Core before the last /proc record on a stop and bound the cleanup after it Without setsid, a stop now sends Core SIGSTOP (guarded by its start time), makes the last record within 250 ms of the stop, kills Core and every recorded descendant within 500 ms, and waits for the exit until 750 ms. A stop during the kills at Core's own exit lets them finish within the kill budget instead of leaving the rest running. --- dotnet/src/JacquardHost/Sdk/CoreCommand.cs | 168 +++++++++++++----- dotnet/src/JacquardHost/Sdk/Unix.cs | 15 +- .../Sdk/CoreCommandTests.cs | 119 +++++++++++-- 3 files changed, 240 insertions(+), 62 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs index b1bc21d..07aaaa7 100644 --- a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs +++ b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs @@ -50,17 +50,25 @@ namespace JacquardLang.Host.Sdk; /// when the command returns. Each record reads at most /// process entries and /// bytes in all (at most of each stat file, -/// and never more than the bytes left), keeps -/// at most descendants in all, and observes the -/// deadline and the token before each entry, ending at once on a stop; the -/// kills at the exit observe them before each descendant too, so a stop -/// leaves the descendants not yet killed running. On timeout or -/// cancellation a last record is made (bounded by the same caps, but not -/// ended by the stop), then the command is killed, then every recorded -/// descendant; a descendant past the caps, or one started and reparented -/// between two records, keeps running. On macOS, -/// which has no /proc, nothing is recorded: only .NET's -/// tree kill on a stop applies. +/// and never more than the bytes left), keeps at most +/// descendants in all, and observes the deadline +/// and the token before each entry, ending at once on a stop. On a timeout +/// or cancellation (a stop), the command is first sent SIGSTOP +/// (guarded against pid reuse by its start time), so it can start no more +/// children and its descendants stay its descendants; then a last record is +/// made within the same caps and at most after the +/// stop; then the command is killed and every recorded descendant, the kills +/// ending + after the stop; +/// then the host waits for the command to exit until +/// after the stop. Each budget is checked before +/// each entry or descendant, so the cleanup returns within +/// plus one stat read and one signal. The +/// kills when the command exits by itself run to the end; a stop that lands +/// there lets them go on for at most . A descendant +/// past the caps or the budgets, or one started and reparented between two +/// records, keeps running. On macOS, which has no /proc, nothing is +/// recorded: only .NET's tree kill on a +/// stop applies. /// /// internal static class CoreCommand @@ -68,8 +76,19 @@ internal static class CoreCommand /// The most of each stream read after the command has exited. internal const int ExitDrain = 4 << 20; - /// How long the host waits for a killed command to exit. - private const int ExitGraceMs = 1000; + /// How long after a stop the last record of descendants may run (without groups). + internal const int StopScanMs = 250; + + /// How much longer after that the kills of recorded descendants may run. + internal const int StopKillMs = 250; + + /// + /// The most time from a stop to the end of the cleanup, the wait for the + /// killed command to exit included: + + /// + 250 ms, each budget checked before each + /// step. + /// + internal const int StopBudgetMs = StopScanMs + StopKillMs + 250; /// The longest single wait between checks. private const int PollSliceMs = 5; @@ -116,6 +135,12 @@ internal static class CoreCommand /// internal static volatile Action? StatReadHook; + /// Test seam: the clock the budgets after a stop are measured with, in ticks. Null outside tests. + internal static volatile Func? ClockHook; + + /// Test seam: runs before the host sends the command itself a signal on a stop without groups, with its pid and the signal. Null outside tests. + internal static volatile Action? SignalHook; + /// Test seam: receives the pid of each command once it has started. Null outside tests. internal static volatile Action? StartedHook; @@ -298,12 +323,14 @@ private sealed class Recorded /// Records the command's current descendants, at most once per /// , reading at most the entry and byte /// caps; a stop ends the record at the next entry, recording nothing - /// from it. The record, made on a stop just - /// before the command is killed, runs whenever it is asked and to - /// its caps. + /// from it. The last record, made on a stop once the command is + /// frozen, runs whenever it is asked, within the caps, until + /// (checked before each entry); what it read by + /// then is recorded. /// - internal void Sample(int root, long expires, CancellationToken cancellationToken, bool final = false) + internal void Sample(int root, long expires, CancellationToken cancellationToken, long end = long.MaxValue) { + bool final = end != long.MaxValue; long now = Stopwatch.GetTimestamp(); if (!RecordDescendants || (now < next && !final) || !OperatingSystem.IsLinux()) { @@ -318,9 +345,14 @@ internal void Sample(int root, long expires, CancellationToken cancellationToken long bytes = 0; foreach (string entry in ProcDirectories()) { - if (!final && Stopped(expires, cancellationToken) is not null) + if (final ? Now() >= end : Stopped(expires, cancellationToken) is not null) { - return; + if (!final) + { + return; + } + + break; } if (++entries > cap || bytes >= byteCap) @@ -371,22 +403,30 @@ internal void Sample(int root, long expires, CancellationToken cancellationToken /// /// Kills every recorded descendant still alive with the start time it /// was recorded with, so a reused pid is not killed (the start time is - /// read just before the kill, as the JDK's process handles do). When - /// , a stop ends it before the next - /// descendant; on a stop's own cleanup it runs to the end, bounded by - /// the number recorded. + /// read just before the kill, as the JDK's process handles do). It + /// ends at , checked before each descendant; + /// when that is unset (at the command's own exit) it runs to the end, + /// bounded by the number recorded, unless a stop lands, which lets it + /// go on for at most more. /// - internal void KillAlive(bool stoppable, long expires, CancellationToken cancellationToken) + internal void KillAlive(long expires, CancellationToken cancellationToken, long end = long.MaxValue) { long bytes = 0; + bool afterStop = end != long.MaxValue; foreach ((int pid, string start) in started) { - if (stoppable && Stopped(expires, cancellationToken) is not null) + if (!afterStop && Stopped(expires, cancellationToken) is not null) + { + afterStop = true; + end = After(Now(), StopKillMs); + } + + if (afterStop && Now() >= end) { return; } - if (ReadStat(Path.Combine(ProcRoot, pid.ToString(CultureInfo.InvariantCulture)), ref bytes, StatBytes, stoppable) is { } stat && stat.Start == start && !stat.Zombie) + if (ReadStat(Path.Combine(ProcRoot, pid.ToString(CultureInfo.InvariantCulture)), ref bytes, StatBytes, !afterStop) is { } stat && stat.Start == start && !stat.Zombie) { Unix.Kill(pid); } @@ -447,9 +487,10 @@ private static bool IsDigits(string text) /// at most of /proc/<pid>/stat, /// read after the last ) (the command name may hold any byte), /// and at most ; null when it cannot be read. - /// Adds the bytes read to . + /// Adds the bytes read to . Only reads for a + /// record or a descendant's kill are passed to . /// - private static (int Pid, int Parent, int Group, string Start, bool Zombie)? ReadStat(string directory, ref long bytes, int limit = StatBytes, bool stoppable = false) + private static (int Pid, int Parent, int Group, string Start, bool Zombie)? ReadStat(string directory, ref long bytes, int limit = StatBytes, bool stoppable = false, bool report = true) { try { @@ -466,7 +507,11 @@ private static (int Pid, int Parent, int Group, string Start, bool Zombie)? Read } bytes += read; - StatReadHook?.Invoke(path, read, stoppable); + if (report) + { + StatReadHook?.Invoke(path, read, stoppable); + } + string stat = System.Text.Encoding.UTF8.GetString(buffer, 0, read); int close = stat.LastIndexOf(')'); if (close < 0 || !int.TryParse(Path.GetFileName(directory), NumberStyles.None, CultureInfo.InvariantCulture, out int pid)) @@ -503,7 +548,7 @@ private static (int Pid, int Parent, int Group, string Start, bool Zombie)? Read internal static void KillGroup(int group, Process leader) { long bytes = 0; - if (HasExited(leader) && ReadStat("/proc/" + group.ToString(CultureInfo.InvariantCulture), ref bytes) is { Zombie: false }) + if (HasExited(leader) && ReadStat("/proc/" + group.ToString(CultureInfo.InvariantCulture), ref bytes, report: false) is { Zombie: false }) { return; } @@ -612,6 +657,10 @@ internal static Output Run( int pid = process.Id; StartedHook?.Invoke(pid); Recorded? recorded = grouped ? null : new Recorded(); + + // Read now so the SIGSTOP on a stop cannot reach a process that took a reused pid. + long unused = 0; + string? coreStart = recorded is not null && OperatingSystem.IsLinux() ? ReadStat("/proc/" + pid.ToString(CultureInfo.InvariantCulture), ref unused, report: false)?.Start : null; Stream? stdout = null; Stream? stderr = null; bool cleaned = false; @@ -645,6 +694,8 @@ internal static Output Run( { if (!cleaned) { + // A stop: everything below ends within StopBudgetMs of it. + long stop = Now(); if (grouped) { // While the command is alive, or unreaped, its group id is reserved. @@ -654,7 +705,7 @@ internal static Output Run( } KillQuietly(process, entireTree: false); - AwaitExit(process); + AwaitExit(process, After(stop, StopBudgetMs)); if (KillGroupOnStop) { // Members forked while the first kill ran. @@ -664,16 +715,24 @@ internal static Output Run( else { // Not .NET's tree kill, which reads every process on the - // machine: a last bounded record, the command, then - // every recorded descendant. + // machine: the command is frozen at once, so it starts + // nothing more and keeps its children; a last record + // within its budget; the command killed; then every + // recorded descendant within theirs. if (!HasExited(process)) { - recorded!.Sample(pid, expires, cancellationToken, final: true); + if (OperatingSystem.IsLinux()) + { + Signal(pid, coreStart, Unix.Sigstop); + recorded!.Sample(pid, expires, cancellationToken, After(stop, StopScanMs)); + SignalHook?.Invoke(pid, Unix.Sigkill); + } + KillQuietly(process, entireTree: !OperatingSystem.IsLinux()); - AwaitExit(process); } - recorded!.KillAlive(stoppable: false, expires, cancellationToken); + recorded!.KillAlive(expires, cancellationToken, After(stop, StopScanMs + StopKillMs)); + AwaitExit(process, After(stop, StopBudgetMs)); } } @@ -726,8 +785,8 @@ private static Output Await( } else { - // A stop ends these kills; the stop is reported below. - recorded!.KillAlive(stoppable: true, expires, cancellationToken); + // A stop lets these kills go on for a bounded time; it is reported below. + recorded!.KillAlive(expires, cancellationToken); } cleaned = true; @@ -793,16 +852,37 @@ private static void KillQuietly(Process process, bool entireTree) } /// - /// Waits up to one second for a killed command to exit, in short - /// poll(2) sleeps, so the wait neither blocks on a .NET wait handle - /// nor consumes a . + /// Waits until at most for a killed command to + /// exit, in short poll(2) sleeps, so the wait neither blocks on a + /// .NET wait handle nor consumes a . /// - private static void AwaitExit(Process process) + private static void AwaitExit(Process process, long end) { - long end = Stopwatch.GetTimestamp() + (Stopwatch.Frequency * ExitGraceMs / 1000); - while (!HasExited(process) && Stopwatch.GetTimestamp() < end) + while (!HasExited(process) && Now() < end) { Posix.Poll(Span.Empty, 1); } } + + /// + /// Sends to the command + /// unless it is gone, a zombie, or no longer has the start time + /// it had when it started (its pid was reused); + /// nothing is sent when that start time could not be read. + /// + private static void Signal(int pid, string? start, int signal) + { + long bytes = 0; + if (start is not null && ReadStat("/proc/" + pid.ToString(CultureInfo.InvariantCulture), ref bytes, report: false) is { Zombie: false } stat && stat.Start == start) + { + SignalHook?.Invoke(pid, signal); + Unix.Signal(pid, signal); + } + } + + /// The time the budgets after a stop are measured with. + private static long Now() => ClockHook?.Invoke() ?? Stopwatch.GetTimestamp(); + + /// The time after . + private static long After(long start, int milliseconds) => start + (Stopwatch.Frequency * milliseconds / 1000); } diff --git a/dotnet/src/JacquardHost/Sdk/Unix.cs b/dotnet/src/JacquardHost/Sdk/Unix.cs index 187707c..850fe6b 100644 --- a/dotnet/src/JacquardHost/Sdk/Unix.cs +++ b/dotnet/src/JacquardHost/Sdk/Unix.cs @@ -7,7 +7,8 @@ namespace JacquardLang.Host.Sdk; /// /// The C library calls the SDK makes beyond the carrier's poll(2): -/// kill(2) to stop a Core command's process group, geteuid(2) +/// kill(2) to stop a Core command's process group (or freeze and +/// kill the command where there is none), geteuid(2) /// and a status call (statx(2) on Linux, stat(2) and /// lstat(2) on macOS) for the cache's owner check, for file kinds and /// sizes, and for the identities that detect a replaced file, @@ -23,6 +24,9 @@ internal static partial class Unix { internal const int Sigkill = 9; + /// SIGSTOP on Linux (macOS numbers it 17; the SDK sends it only on Linux). + internal const int Sigstop = 19; + private const int Enoent = 2; private const int Enotdir = 20; private const int Eexist = 17; @@ -117,6 +121,15 @@ internal static void Kill(int pid) } } + /// Sends to , never to a pid of one or less. + internal static void Signal(int pid, int signal) + { + if (pid > 1) + { + _ = NativeKill(pid, signal); + } + } + /// Whether this process may execute , as access(path, X_OK) reports. internal static bool MayExecute(string path) => NativeAccess(path, 1) == 0; diff --git a/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs index 7a770b2..294755f 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs @@ -42,6 +42,8 @@ public void CleanUp() CoreCommand.ProcEntryCap = CoreCommand.ProcEntries; CoreCommand.ProcByteCap = CoreCommand.ProcBytes; CoreCommand.StatReadHook = null; + CoreCommand.ClockHook = null; + CoreCommand.SignalHook = null; foreach (string file in Directory.EnumerateFiles(root, "*.pid")) { Support.Kill(int.Parse(File.ReadAllText(file).Trim(), CultureInfo.InvariantCulture)); @@ -391,46 +393,123 @@ public void AProcRecordNeverReadsPastANonAlignedByteCap() Assert.That(reads, Is.EqualTo(new[] { 1024, 1 })); } + /// The state letter of a live process, from /proc/<pid>/stat. + private static char State(int pid) + { + string stat = File.ReadAllText("/proc/" + pid.ToString(CultureInfo.InvariantCulture) + "/stat"); + return stat[(stat.LastIndexOf(')') + 2)..][0]; + } + /// - /// A cancellation that lands during a record ends it at the next entry: - /// of 50 fake entries, the record stops after the third read, and the - /// command is stopped (the last record on the stop is bounded by the cap - /// instead). + /// On a stop without groups, the command is frozen with SIGSTOP before + /// the last record, the last record ends at its 250 ms budget, and only + /// then is the command killed. A cancellation during a record ends that + /// record at the next entry (the third of 50). The budget is measured on + /// a test clock that each read of the last record advances by 100 ms + /// (and each reading by 1 ms), so exactly three entries fit in it: the + /// fourth would start 304 ms after the stop. Without the budget, the + /// last record would read all 40 entries of its cap. /// [Test] - public void ACancelDuringAProcRecordEndsIt() + public void AStopFreezesTheCommandBeforeItsLastRecordAndBoundsIt() { CoreCommand.WithoutGroups = true; CoreCommand.ProcRoot = FakeProc("proc", 50, 1); CoreCommand.ProcEntryCap = 40; + long clock = 1_000_000; + long hundredMs = Stopwatch.Frequency / 10; + long oneMs = Stopwatch.Frequency / 1000; + CoreCommand.ClockHook = () => Interlocked.Add(ref clock, oneMs); + int command = 0; + CoreCommand.StartedHook = pid => command = pid; + var events = new List(); int stoppable = 0; - int final = 0; + char frozen = '?'; using var cancel = new CancellationTokenSource(); + CoreCommand.SignalHook = (pid, signal) => events.Add((pid == command ? "command " : "other ") + signal.ToString(CultureInfo.InvariantCulture)); CoreCommand.StatReadHook = (path, bytes, canStop) => { - if (!canStop) + if (canStop) { - final++; + if (++stoppable == 3) + { + cancel.Cancel(); + } + + return; } - else if (++stoppable == 3) + + if (!events.Contains("read")) { - cancel.Cancel(); + // SIGSTOP is delivered asynchronously; the command must reach the stopped state. + Support.Eventually(() => State(command) == 'T', TimeSpan.FromSeconds(10)); + frozen = State(command); } + + events.Add("read"); + Interlocked.Add(ref clock, hundredMs); }; CarrierException error = Support.Throws(() => Run(Waiting(Path.Combine(root, "never")), cancellationToken: cancel.Token))!; Assert.That(error.IsAborted, Is.True); Assert.That(stoppable, Is.EqualTo(3)); - Assert.That(final, Is.EqualTo(40)); + Assert.That(events, Is.EqualTo(new[] { "command 19", "read", "read", "read", "command 9" })); + Assert.That(frozen, Is.EqualTo('T')); + SdkSupport.Dies("the command", command); + } + + /// + /// The whole cleanup after a stop, on the real clock, ends within + /// (750 ms) plus scheduling slack, + /// even when the last record and the kills of 300 recorded descendants + /// would take 600 ms each (each fake stat read after the stop + /// takes at least 2 ms): the record and the kills are cut at their + /// budgets. The measured time is written to the test output. + /// + [Test] + public void TheCleanupAfterAStopEndsWithinItsBudget() + { + CoreCommand.WithoutGroups = true; + string proc = Path.Combine(root, "proc"); + CoreCommand.ProcRoot = proc; + CoreCommand.StartedHook = pid => FakeProc("proc", 300, pid); + int recorded = 0; + using var cancel = new CancellationTokenSource(); + long cancelled = 0; + CoreCommand.RecordedHook = pids => + { + if (recorded == 0) + { + recorded = pids.Count; + cancelled = Stopwatch.GetTimestamp(); + cancel.Cancel(); + } + }; + int after = 0; + CoreCommand.StatReadHook = (path, bytes, canStop) => + { + if (!canStop) + { + after++; + Thread.Sleep(2); + } + }; + Support.Throws(() => Run(Waiting(Path.Combine(root, "never")), cancellationToken: cancel.Token)); + double elapsed = Stopwatch.GetElapsedTime(cancelled).TotalMilliseconds; + TestContext.Out.WriteLine("cleanup after the stop: " + elapsed.ToString("F1", CultureInfo.InvariantCulture) + " ms; reads after the stop: " + after.ToString(CultureInfo.InvariantCulture)); + Assert.That(recorded, Is.EqualTo(300)); + Assert.That(after, Is.LessThan(600), "the record and the kills were not cut at their budgets"); + Assert.That(elapsed, Is.LessThan(CoreCommand.StopBudgetMs + 500)); } /// - /// The kills of recorded descendants when the command exits observe a - /// stop before each descendant: ten fake descendants are recorded, the - /// command exits, and a cancellation during the second kill check ends - /// the kills there and is reported. + /// A stop that lands during the kills of recorded descendants when the + /// command exits does not leave the rest running: ten fake descendants + /// are recorded, the command exits, a cancellation lands during the + /// second kill check, the other eight are still checked (within the kill + /// budget), and the stop is reported. /// [Test] - public void AStopDuringTheExitKillsEndsThem() + public void AStopDuringTheExitKillsFinishesThem() { CoreCommand.WithoutGroups = true; string release = Path.Combine(root, "release"); @@ -445,10 +524,15 @@ public void AStopDuringTheExitKillsEndsThem() File.WriteAllText(release, string.Empty); }; int checks = 0; + int afterStop = 0; using var cancel = new CancellationTokenSource(); CoreCommand.StatReadHook = (path, bytes, stoppable) => { - if (stoppable && !CoreCommand.RecordDescendants && ++checks == 2) + if (!stoppable) + { + afterStop++; + } + else if (!CoreCommand.RecordDescendants && ++checks == 2) { cancel.Cancel(); } @@ -457,6 +541,7 @@ public void AStopDuringTheExitKillsEndsThem() Assert.That(error.IsAborted, Is.True); Assert.That(recorded, Is.EqualTo(10)); Assert.That(checks, Is.EqualTo(2)); + Assert.That(afterStop, Is.EqualTo(8)); } [Test] From c6144316b046b506b6fed37dddc19c5bf68ff119 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:50:18 -0400 Subject: [PATCH 22/52] Refuse a cache root under a directory others may write, and recheck the root before each worker start The cache root's path is resolved once with realpath(3); every directory above it must be owned by the user or root and not group- or world-writable unless sticky. A prepared program checks the root's device and inode before each worker starts. --- dotnet/src/JacquardHost/Sdk/Core.cs | 46 +++++++++++ dotnet/src/JacquardHost/Sdk/Function.cs | 20 ++++- .../src/JacquardHost/Sdk/PreparedProgram.cs | 8 +- dotnet/src/JacquardHost/Sdk/Runtime.cs | 27 ++++--- dotnet/src/JacquardHost/Sdk/Unix.cs | 34 +++++++- .../test/JacquardHost.Tests/Sdk/FakeCore.cs | 3 + .../JacquardHost.Tests/Sdk/RuntimeTests.cs | 81 +++++++++++++++++++ 7 files changed, 201 insertions(+), 18 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Core.cs b/dotnet/src/JacquardHost/Sdk/Core.cs index 0c70344..3013fbc 100644 --- a/dotnet/src/JacquardHost/Sdk/Core.cs +++ b/dotnet/src/JacquardHost/Sdk/Core.cs @@ -581,4 +581,50 @@ internal static Unix.Status Check(string path, Access access) return status; } + + /// + /// Refuses the cache root , a real path with no + /// links in it, unless every directory above it, up to /, is owned + /// by this user or by root, and neither its group nor others may write it + /// unless it has the sticky bit (as OpenSSH checks the directories above + /// its configuration). Otherwise another user could rename the root away + /// and put another directory at its path at any time, and every later + /// use of the path, a prepared program's workers included, would use it. + /// In a sticky directory only the owner of an entry (or root) may rename + /// it, and the next directory down is checked in turn. + /// + /// A preparation failure naming the directory. + internal static void CheckAncestors(string root) + { + if (!Unix.Supported) + { + return; + } + + uint user = Unix.EffectiveUser(); + string? directory = Path.GetDirectoryName(root); + while (directory is not null) + { + Unix.Status status = StatOrFail(directory, follow: false, "cannot read "); + if (status.Kind != Unix.Kind.Directory) + { + throw JacquardException.Preparation("cache directory parent " + directory + " is not a directory"); + } + + if (status.Owner != user && status.Owner != 0) + { + throw JacquardException.Preparation("cache directory parent " + directory + " is owned by another user, who could replace the cache;" + + " keep the cache under directories owned by you or root"); + } + + // 0o022: group or other write; 0o1000: sticky. + if ((status.Mode & 0x12) != 0 && (status.Mode & 0x200) == 0) + { + throw JacquardException.Preparation("cache directory parent " + directory + " is group- or world-writable without the sticky bit," + + " so another user could replace the cache (make it, for example, chmod go-w " + directory + ")"); + } + + directory = Path.GetDirectoryName(directory); + } + } } diff --git a/dotnet/src/JacquardHost/Sdk/Function.cs b/dotnet/src/JacquardHost/Sdk/Function.cs index 71bbe09..bb14cfe 100644 --- a/dotnet/src/JacquardHost/Sdk/Function.cs +++ b/dotnet/src/JacquardHost/Sdk/Function.cs @@ -19,9 +19,11 @@ public sealed class Function private readonly Hash target; private readonly OperationRegistry registry; private readonly Host host; + private readonly Action? rootCheck; - internal Function(string name, CallInterface signature, Hash target, OperationRegistry registry, Host host, ProgramData data) + internal Function(string name, CallInterface signature, Hash target, OperationRegistry registry, Host host, ProgramData data, Action? rootCheck = null) { + this.rootCheck = rootCheck; Name = name; Signature = signature; this.target = target; @@ -123,8 +125,20 @@ private List Check(List values) return values; } - private Invocation Run(List values, CancellationToken cancellationToken) => - new(host.Invoke(target, Signature, values, registry, cancellationToken), this); + /// + /// Runs one worker on the prepared store, by its path, once the cache + /// root is checked to be the directory that was prepared (a cancelled + /// call starts no worker and is reported as aborted, as before). + /// + private Invocation Run(List values, CancellationToken cancellationToken) + { + if (!cancellationToken.IsCancellationRequested) + { + rootCheck?.Invoke(); + } + + return new(host.Invoke(target, Signature, values, registry, cancellationToken), this); + } /// /// Validates the arguments locally, then runs one worker. The exception diff --git a/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs b/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs index 5ab0147..53de80e 100644 --- a/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs +++ b/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs @@ -29,8 +29,9 @@ public sealed class PreparedProgram private Host host; [SuppressMessage("Design", "CA1031", Justification = "No exception from reading one signature may fail the preparation; the function is left unbound with the reason.")] - internal PreparedProgram(Runtime runtime, string directory, Runtime.Bindings bindings, string executable) + internal PreparedProgram(Runtime runtime, string directory, Runtime.Bindings bindings, string executable, Action rootCheck) { + RootCheck = rootCheck; Runtime = runtime; Directory = directory; var names = new Dictionary(StringComparer.Ordinal); @@ -101,6 +102,9 @@ internal PreparedProgram(Runtime runtime, string directory, Runtime.Bindings bin Data = new ProgramData(names, constructors, bindings.Groups, runtime.Limits); } + /// Refuses a call once the cache root is no longer the directory that was checked. + internal Action RootCheck { get; } + private sealed record EffectGroup(Hash Identity, IReadOnlyList Operations); /// The runtime that prepared this program. @@ -309,7 +313,7 @@ public Function Function(string name, IReadOnlyDictionary? b + ". Bind every operation in each declared effect before calling"); } - return new Function(name, signature, target, registry, host, Data); + return new Function(name, signature, target, registry, host, Data, RootCheck); } } diff --git a/dotnet/src/JacquardHost/Sdk/Runtime.cs b/dotnet/src/JacquardHost/Sdk/Runtime.cs index 70c0a0a..7117f3a 100644 --- a/dotnet/src/JacquardHost/Sdk/Runtime.cs +++ b/dotnet/src/JacquardHost/Sdk/Runtime.cs @@ -140,7 +140,8 @@ public Runtime(RuntimeOptions? options = null) /// /// Test seam: runs just before the cache root's identity is checked /// again, with the use that follows: stage, publish, - /// load, or loaded. Null outside tests. + /// load, loaded, or call (a prepared program's + /// worker start). Null outside tests. /// internal static volatile Action? RootHook; @@ -153,9 +154,10 @@ private string Command(string executable, CancellationToken cancellationToken, p } /// - /// Refuses the preparation unless the cache root at - /// is still the directory that was checked (the same device and inode), - /// so a root renamed away and replaced after the check is not used. + /// Refuses the preparation, or a prepared program's call, unless the + /// cache root at is still the directory that was + /// checked (the same device and inode), so a root renamed away and + /// replaced after the check is not used. /// private static void SameRoot(string cache, Unix.Status pinned, string use) { @@ -489,9 +491,10 @@ private PreparedProgram PrepareChecked(IReadOnlyList sources, PrepareOpt Directory.CreateDirectory(configured, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); } - // A symbolic link at the cache root is followed once, here; the - // directory it names is checked, and nothing inside is followed. - cache = new DirectoryInfo(configured).ResolveLinkTarget(returnFinalTarget: true)?.FullName ?? configured; + // Every symbolic link in the cache root's path is resolved once, + // here, and the real directory is checked and used from then on; + // nothing inside it is followed. + cache = Unix.Supported ? Unix.RealPath(configured) : new DirectoryInfo(configured).ResolveLinkTarget(returnFinalTarget: true)?.FullName ?? configured; } catch (Exception error) when (error is IOException or UnauthorizedAccessException or ArgumentException) { @@ -501,10 +504,12 @@ private PreparedProgram PrepareChecked(IReadOnlyList sources, PrepareOpt string directory = Path.Combine(cache, Core.MapDigest(identity)); // Only the cache's owner may write it; nothing inside it can resist - // someone else who may. The root is used by its path, so its - // identity is checked again before each use: another user who may - // write its parent could otherwise replace it after this check. + // someone else who may. The root is used by its path, so no other + // user may write a directory above it (one who could would rename it + // away and put another directory there), and its identity is checked + // again before each use, a prepared program's worker starts included. Unix.Status root = Core.Check(cache, Core.Access.OwnerOnly); + Core.CheckAncestors(cache); if (Unix.Stat(directory, follow: false).Kind != Unix.Kind.Missing) { Core.Unchanged(pins, cancellationToken); @@ -864,7 +869,7 @@ JacquardException Fail(string detail) => JacquardException.Preparation("prepared // What was read came from the root that was checked. SameRoot(cache, root, "loaded"); - var program = new PreparedProgram(this, directory, metadata.Bindings, executable); + var program = new PreparedProgram(this, directory, metadata.Bindings, executable, () => SameRoot(cache, root, "call")); LoadedHook?.Invoke(); return program; } diff --git a/dotnet/src/JacquardHost/Sdk/Unix.cs b/dotnet/src/JacquardHost/Sdk/Unix.cs index 850fe6b..4f2d4d0 100644 --- a/dotnet/src/JacquardHost/Sdk/Unix.cs +++ b/dotnet/src/JacquardHost/Sdk/Unix.cs @@ -12,8 +12,9 @@ namespace JacquardLang.Host.Sdk; /// and a status call (statx(2) on Linux, stat(2) and /// lstat(2) on macOS) for the cache's owner check, for file kinds and /// sizes, and for the identities that detect a replaced file, -/// access(2) for the execute permission, and mkdir(2) to -/// create a staging directory exclusively. +/// access(2) for the execute permission, mkdir(2) to +/// create a staging directory exclusively, and realpath(3) to resolve +/// the cache root's path once. /// /// /// The macOS calls use the 64-bit-inode struct stat layout; they are @@ -75,6 +76,12 @@ internal bool SameFile(Status other) => [LibraryImport("libc", EntryPoint = "access", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] private static partial int NativeAccess(string path, int mode); + [LibraryImport("libc", EntryPoint = "realpath", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static partial IntPtr NativeRealpath(string path, IntPtr resolved); + + [LibraryImport("libc", EntryPoint = "free")] + private static partial void NativeFree(IntPtr pointer); + [LibraryImport("libc", EntryPoint = "mkdir", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] private static partial int NativeMkdir(string path, uint mode); @@ -133,6 +140,29 @@ internal static void Signal(int pid, int signal) /// Whether this process may execute , as access(path, X_OK) reports. internal static bool MayExecute(string path) => NativeAccess(path, 1) == 0; + /// + /// The absolute path of with every symbolic link, + /// ., and .. resolved, as realpath(3) reports it. + /// + /// The path cannot be resolved. + internal static string RealPath(string path) + { + IntPtr resolved = NativeRealpath(path, IntPtr.Zero); + if (resolved == IntPtr.Zero) + { + throw new IOException("realpath failed with errno " + Marshal.GetLastPInvokeError().ToString(CultureInfo.InvariantCulture)); + } + + try + { + return Marshal.PtrToStringUTF8(resolved) ?? throw new IOException("realpath returned no path"); + } + finally + { + NativeFree(resolved); + } + } + /// Creates with ; false when it already exists. /// The directory cannot be created for another reason. internal static bool MakeDirectory(string path, int mode) diff --git a/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs b/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs index 88f915a..571fbbd 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/FakeCore.cs @@ -43,6 +43,9 @@ private FakeCore(string root, Runtime runtime) /// internal static FakeCore Create(string root, string version, string? versionCase, string cases) { + // Owner-only whatever the umask: the cache under it must have no parent others may write. + Directory.CreateDirectory(root); + File.SetUnixFileMode(root, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); Directory.CreateDirectory(Path.Combine(root, "prelude")); File.WriteAllText(Path.Combine(root, "prelude", "01-prim.jqd"), "prelude\n"); File.WriteAllText(Path.Combine(root, "rules.jac"), "rules\n"); diff --git a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs index 2b992b3..f818480 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs @@ -814,6 +814,87 @@ string Swap(string use, string source) fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); } + /// + /// No other user may write a directory above the cache root, so none can + /// rename the root away and put another directory at its path: a parent + /// that its group or others may write is refused, naming it, unless it + /// has the sticky bit (in which only an entry's owner may rename it). + /// Links in the root's path are resolved once, so the mode of a link + /// itself (always 0777) does not matter and its target is checked. + /// + [Test] + public void ACacheParentOthersMayWriteIsRefusedUnlessItIsSticky() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent"), UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute).FullName; + var options = new PrepareOptions { CacheDir = Path.Combine(parent, "cache") }; + string Refused(UnixFileMode mode) + { + File.SetUnixFileMode(parent, mode); + return Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options))!.Message; + } + + const UnixFileMode Owner = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute; + string expected = "cache directory parent " + parent + " is group- or world-writable without the sticky bit," + + " so another user could replace the cache (make it, for example, chmod go-w " + parent + ")"; + Assert.That(Refused(Owner | UnixFileMode.OtherWrite | UnixFileMode.OtherExecute), Is.EqualTo(expected), "others may write"); + Assert.That(Refused(Owner | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute), Is.EqualTo(expected), "the group may write"); + + // A grandparent is checked as well. + string nested = Directory.CreateDirectory(Path.Combine(parent, "nested"), Owner).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1FF); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = Path.Combine(nested, "cache") }))!.Message, + Is.EqualTo(expected)); + + // Sticky and world-writable, as /tmp is: accepted. + File.SetUnixFileMode(parent, (UnixFileMode)0x3FF); + Assert.That(fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options).Directory, Does.StartWith(Path.Combine(parent, "cache") + "/")); + File.SetUnixFileMode(parent, Owner | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute); + fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options); + + // Through a link (mode 0777) to the parent: the real path is used. + string link = At("link"); + File.CreateSymbolicLink(link, parent); + PreparedProgram linked = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = Path.Combine(link, "cache") }); + Assert.That(linked.Directory, Does.StartWith(Path.Combine(parent, "cache") + "/")); + } + + /// + /// A prepared program's workers run on its store by path, so the cache + /// root's device and inode are checked again before each worker starts. + /// After preparation the root is renamed away and back (the same + /// directory: calls still start a worker), then renamed away and + /// replaced by a new owner-only directory, as another user who could + /// write a parent could: the next call is refused before any worker + /// starts, for a function bound before the swap and one bound after it. + /// + [Test] + public void ACacheRootReplacedAfterPreparationIsRefusedAtTheNextCall() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string cache = Path.GetFullPath(fake.Options.CacheDir!); + PreparedProgram program = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + Function before = program.Function("first"); + long Workers() => File.ReadAllLines(Path.Combine(fake.Root, "log")).Length; + + long started = Workers(); + Assert.That(before.Invoke().Ok, Is.False, "the fake worker answers nothing"); + Assert.That(Workers(), Is.EqualTo(started + 1), "a worker started"); + + Directory.Move(cache, cache + "-away"); + Directory.Move(cache + "-away", cache); + before.Invoke(); + Assert.That(Workers(), Is.EqualTo(started + 2), "the same root renamed back is used"); + + Directory.Move(cache, cache + "-away"); + Directory.CreateDirectory(cache, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + string refused = "cache root " + cache + " was replaced after it was checked; prepare again"; + Assert.That(Support.Throws(() => before.Invoke())!.Message, Is.EqualTo(refused)); + Assert.That(Support.Throws(() => program.Function("second").Invoke())!.Message, Is.EqualTo(refused)); + Assert.That(Workers(), Is.EqualTo(started + 2), "no worker started from the replacement"); + } + /// /// The executable Core runs must be the one fingerprinted: a test seam /// replaces it (a new file renamed over it) after each kind of Core From 07bc7b50f7625ee426e7656844f7dbb5b0c46922 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:53:51 -0400 Subject: [PATCH 23/52] Make Dynamic equality's pair set only when it meets a shared sub-value --- dotnet/src/JacquardHost/Sdk/Dynamic.cs | 49 +++++++++++++++++-- .../test/JacquardHost.Tests/Sdk/DataTests.cs | 19 +++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Dynamic.cs b/dotnet/src/JacquardHost/Sdk/Dynamic.cs index 84984aa..9607b77 100644 --- a/dotnet/src/JacquardHost/Sdk/Dynamic.cs +++ b/dotnet/src/JacquardHost/Sdk/Dynamic.cs @@ -3,6 +3,7 @@ using System.Collections.ObjectModel; using System.Globalization; using System.Text; +using System.Threading; using JacquardLang.Host.Protocol; namespace JacquardLang.Host.Sdk; @@ -20,10 +21,29 @@ public abstract class Dynamic : IEquatable { private const int RenderLimit = 4096; + /// How many places hold this value as a child; more than one means it is shared. + private int uses; + private protected Dynamic() { } + /// Test seam: how many pair sets made on this thread. + [ThreadStatic] + internal static int PairSetsMade; + + /// Whether this value is a child in more than one place, so a walk can reach it twice. + private bool Shared => Volatile.Read(ref uses) > 1; + + /// Counts each of as held once more. + private protected static void Adopt(IReadOnlyList children) + { + foreach (Dynamic child in children) + { + Interlocked.Increment(ref child.uses); + } + } + /// Unit, the empty tuple. public static Tuple Unit { get; } = new(Array.Empty()); @@ -156,6 +176,7 @@ private static ReadOnlyCollection Limited(IEnumerable items, s copy.Add(item ?? throw new ArgumentNullException(nameof(items), what + " holds null")); } + Adopt(copy); return copy.AsReadOnly(); } @@ -178,18 +199,35 @@ public bool Equals(Dynamic? other) // Each pair of nodes is compared once: values that share sub-values // (as decoded values and caller-built graphs may) are compared in - // time proportional to their distinct pairs, not their paths. + // time proportional to their distinct pairs, not their paths. Only a + // pair with a shared side can be reached twice (an unshared node has + // one parent and one place in it), so only those pairs are kept, and + // the set is made at the first: unshared values need none. var stack = new Stack<(Dynamic Left, Dynamic Right)>(); - var compared = new HashSet<(Dynamic Left, Dynamic Right)>(PairComparer.Instance); + HashSet<(Dynamic Left, Dynamic Right)>? compared = null; stack.Push((this, other)); while (stack.Count > 0) { (Dynamic left, Dynamic right) = stack.Pop(); - if (ReferenceEquals(left, right) || !compared.Add((left, right))) + if (ReferenceEquals(left, right)) { continue; } + if (left.Shared || right.Shared) + { + if (compared is null) + { + compared = new HashSet<(Dynamic Left, Dynamic Right)>(PairComparer.Instance); + PairSetsMade++; + } + + if (!compared.Add((left, right))) + { + continue; + } + } + if (left.GetType() != right.GetType() || !left.SameNode(right)) { return false; @@ -393,6 +431,11 @@ public sealed class Option : Dynamic public Option(Dynamic? value) { Value = value; + if (value is not null) + { + Adopt(new[] { value }); + } + } /// The value of Some, or null for None. diff --git a/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs index 207f5eb..43a4385 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/DataTests.cs @@ -654,6 +654,25 @@ static Dynamic Layered(int depth, long leaf) Assert.That(comparison.Result, Is.EqualTo((true, false))); } + /// + /// Equality keeps a set of compared pairs only once it meets a shared + /// sub-value: two separately built lists of 1,000 tuples compare without + /// one (a test seam counts the sets made), and a value holding one leaf + /// twice makes one. + /// + [Test] + public void EqualityOfUnsharedValuesMakesNoPairSet() + { + static Dynamic Tree(long leaf) => new Dynamic.List(Enumerable.Range(0, 1000).Select(i => Dynamic.TupleOf(Dynamic.Of(i), new Dynamic.Option(Dynamic.Of(leaf))))); + Dynamic.PairSetsMade = 0; + Assert.That(Tree(1).Equals(Tree(1)), Is.True); + Assert.That(Tree(1).Equals(Tree(2)), Is.False); + Assert.That(Dynamic.PairSetsMade, Is.EqualTo(0)); + Dynamic leaf = Dynamic.Of(5); + Assert.That(Dynamic.TupleOf(leaf, leaf).Equals(Dynamic.TupleOf(Dynamic.Of(5), Dynamic.Of(5))), Is.True); + Assert.That(Dynamic.PairSetsMade, Is.EqualTo(1)); + } + /// /// Messages quote caller text as an excerpt, and no message is kept /// past 8,192 chars: a result codec naming a huge constructor, codecs, From b6ff6b44f1952ac219996bcde2763bd2de783eec Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:53:51 -0400 Subject: [PATCH 24/52] Wrap exceptions from the caller's source list and bindings with their cause --- .../src/JacquardHost/Sdk/JacquardException.cs | 3 + .../src/JacquardHost/Sdk/PreparedProgram.cs | 21 +++++-- dotnet/src/JacquardHost/Sdk/Runtime.cs | 18 +++++- .../JacquardHost.Tests/Sdk/RuntimeTests.cs | 61 +++++++++++++++++++ 4 files changed, 95 insertions(+), 8 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/JacquardException.cs b/dotnet/src/JacquardHost/Sdk/JacquardException.cs index 8d9b3ce..6791dfb 100644 --- a/dotnet/src/JacquardHost/Sdk/JacquardException.cs +++ b/dotnet/src/JacquardHost/Sdk/JacquardException.cs @@ -146,6 +146,9 @@ private static long NotedLength(string message) internal static JacquardException Preparation(string message) => new(JacquardErrorKind.Preparation, message, null, null, -1, Nothing); + /// A preparation failure caused by , such as a caller's source list that threw. + internal static JacquardException Preparation(string message, Exception? cause) => new(JacquardErrorKind.Preparation, message, null, cause, -1, Nothing); + internal static JacquardException Binding(string message) => new(JacquardErrorKind.Binding, message, null, null, -1, Nothing); /// A binding failure caused by , such as a caller's codec that threw. diff --git a/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs b/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs index 53de80e..2a2679b 100644 --- a/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs +++ b/dotnet/src/JacquardHost/Sdk/PreparedProgram.cs @@ -187,19 +187,28 @@ public Function Function(string name, IReadOnlyDictionary? b // taken, so its code cannot hold that lock: its count once, and its // entries only when they are no more than the program's effects, at // most one past them. The effects are fixed when the program is made. - int declared = bindings.Count; + // What it throws is wrapped with the cause, as for other caller code. + int declared; var given = new List>(); - if (declared <= effects.Count) + try { - foreach (KeyValuePair binding in bindings) + declared = bindings.Count; + if (declared <= effects.Count) { - given.Add(binding); - if (given.Count > effects.Count) + foreach (KeyValuePair binding in bindings) { - break; + given.Add(binding); + if (given.Count > effects.Count) + { + break; + } } } } + catch (Exception error) when (Caller.Wraps(error)) + { + throw JacquardException.Binding(Identities.Excerpt(name) + ": reading the bindings failed: " + Identities.Describe(error), error); + } lock (sync) { diff --git a/dotnet/src/JacquardHost/Sdk/Runtime.cs b/dotnet/src/JacquardHost/Sdk/Runtime.cs index 7117f3a..1e7e4bd 100644 --- a/dotnet/src/JacquardHost/Sdk/Runtime.cs +++ b/dotnet/src/JacquardHost/Sdk/Runtime.cs @@ -416,8 +416,21 @@ private PreparedProgram PrepareChecked(IReadOnlyList sources, PrepareOpt { JacquardException Usage() => JacquardException.Preparation("pass .jac files or .jqd files in dependency order"); + // The caller's list runs the caller's code; what it throws is wrapped with the cause. + static T Read(Func read) + { + try + { + return read(); + } + catch (Exception error) when (Caller.Wraps(error)) + { + throw JacquardException.Preparation("reading the source list failed: " + Identities.Describe(error), error); + } + } + // The count comes from Count alone, before any element is read. - int count = sources.Count; + int count = Read(() => sources.Count); if (count == 0) { throw Usage(); @@ -439,7 +452,8 @@ private PreparedProgram PrepareChecked(IReadOnlyList sources, PrepareOpt throw JacquardException.Preparation(Core.Aborted); } - string source = Core.CheckedPath(sources[i], "source"); + int index = i; + string source = Core.CheckedPath(Read(() => sources[index]), "source"); checkedSources.Add(source); string kind = Extension(source); syntax ??= kind; diff --git a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs index f818480..fce1091 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs @@ -995,6 +995,67 @@ public void AnExecutableTheUserMayNotRunIsRefused() Assert.That(fake.Calls("--version"), Is.EqualTo(0)); } + /// A source list whose count, or else whose indexer, throws Error. + private sealed class ThrowingSources(bool count) : IReadOnlyList + { + internal static readonly InvalidOperationException Error = new("list failed"); + + public int Count => count ? throw Error : 1; + + public string this[int index] => throw Error; + + public IEnumerator GetEnumerator() => throw Error; + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// A bindings dictionary whose count, or else whose enumerator, throws . + private sealed class ThrowingBindings(bool count) : IReadOnlyDictionary + { + public int Count => count ? throw ThrowingSources.Error : 1; + + public IEnumerable Keys => throw new NotSupportedException(); + + public IEnumerable Values => throw new NotSupportedException(); + + public Operations this[string key] => throw new NotSupportedException(); + + public bool ContainsKey(string key) => throw new NotSupportedException(); + + public bool TryGetValue(string key, out Operations value) => throw new NotSupportedException(); + + public IEnumerator> GetEnumerator() => throw ThrowingSources.Error; + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + /// + /// An exception from the caller's source list (its count or an element) + /// or bindings dictionary (its count or enumerator) is wrapped in the + /// SDK's exception, keeping the cause, as other caller code is. + /// + [Test] + public void ACallersFailingSourceListOrBindingsIsWrappedWithTheCause() + { + using FakeCore fake = FakeCore.Effects(At("throwing")); + foreach (bool count in new[] { true, false }) + { + JacquardException sources = Support.Throws(() => fake.Runtime.Prepare(new ThrowingSources(count), fake.Options))!; + Assert.That(sources.Kind, Is.EqualTo(JacquardErrorKind.Preparation)); + Assert.That(sources.Message, Is.EqualTo("reading the source list failed: System.InvalidOperationException: list failed")); + Assert.That(sources.InnerException, Is.SameAs(ThrowingSources.Error)); + } + + PreparedProgram program = fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, fake.Options); + foreach (bool count in new[] { true, false }) + { + JacquardException bindings = Support.Throws(() => program.Function("first", new ThrowingBindings(count)))!; + Assert.That(bindings.Kind, Is.EqualTo(JacquardErrorKind.Binding)); + Assert.That(bindings.Message, Is.EqualTo("first: reading the bindings failed: System.InvalidOperationException: list failed")); + Assert.That(bindings.InnerException, Is.SameAs(ThrowingSources.Error)); + } + } + /// A bindings dictionary that records whether the program's lock was held while it was read. private sealed class Watching : IReadOnlyDictionary { From ae9e2948d08f781e546f5668404b7f25863a8d22 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 08:59:23 -0400 Subject: [PATCH 25/52] Make each missing level of a cache root owner-only --- dotnet/src/JacquardHost/Sdk/Runtime.cs | 13 ++++++++++++- .../JacquardHost.Tests/Sdk/RuntimeTests.cs | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/dotnet/src/JacquardHost/Sdk/Runtime.cs b/dotnet/src/JacquardHost/Sdk/Runtime.cs index 1e7e4bd..a454186 100644 --- a/dotnet/src/JacquardHost/Sdk/Runtime.cs +++ b/dotnet/src/JacquardHost/Sdk/Runtime.cs @@ -502,7 +502,18 @@ static T Read(Func read) } else { - Directory.CreateDirectory(configured, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + // Each missing level is made owner-only, whatever the umask, + // since no directory above the root may be writable by others. + var missing = new Stack(); + for (string? level = configured; level is not null && !Directory.Exists(level); level = Path.GetDirectoryName(level)) + { + missing.Push(level); + } + + while (missing.Count > 0) + { + Directory.CreateDirectory(missing.Pop(), UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } } // Every symbolic link in the cache root's path is resolved once, diff --git a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs index fce1091..8e98b76 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs @@ -860,6 +860,24 @@ string Refused(UnixFileMode mode) Assert.That(linked.Directory, Does.StartWith(Path.Combine(parent, "cache") + "/")); } + /// + /// The directories a preparation makes for a missing cache root are each + /// owner-only (0700), whatever the umask, so a cache under new + /// directories is not refused for parents the SDK itself made writable. + /// + [Test] + public void TheDirectoriesMadeForACacheRootAreOwnerOnly() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string top = At("made"); + string cache = Path.Combine(top, "a", "b", "cache"); + fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = cache }); + foreach (string level in new[] { top, Path.Combine(top, "a"), Path.Combine(top, "a", "b"), cache }) + { + Assert.That(File.GetUnixFileMode(level), Is.EqualTo(UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute), level); + } + } + /// /// A prepared program's workers run on its store by path, so the cache /// root's device and inode are checked again before each worker starts. From 8050bef4a1af95f9ca773fbd47e0cf1d3c0e0b9a Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 09:10:48 -0400 Subject: [PATCH 26/52] Document the second review round's .NET SDK fixes --- CHANGELOG.md | 13 +++++--- docs/EVIDENCE.md | 7 +++-- docs/LIMITS.md | 4 +-- dotnet/README.md | 82 ++++++++++++++++++++++++++++++++---------------- 4 files changed, 70 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e0f4ec..3781db5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,10 +104,15 @@ Core's commands run on the caller's thread through `poll(2)`, bounded in time and output, and on Linux through `setsid`, whose process group is killed when a command exits or is stopped; without `setsid`, descendants - are recorded from `/proc` within entry and byte caps. Only the cache's - owner may write it: the root, each entry, and each file must be owned by - the current user and not group- or world-writable, and the root's identity - is checked again before each use (its parent directories are not checked). + are recorded from `/proc` within entry and byte caps, and on a stop Core + is frozen with `SIGSTOP` before the last record and the cleanup ends + within 750 ms. Only the cache's owner may write it: the root, each entry, + and each file must be owned by the current user and not group- or + world-writable, every directory above the root must be owned by the user + or root and not group- or world-writable unless it is sticky, and the + root's device and inode are checked again before each use, each worker a + prepared program starts included. Replacement by the same user is not + covered. Every binding is derived again from Core's cached output on load, and a preparation during which the Core executable or prelude changed is refused. `Invocation.Save` writes a new owner-only file and renames it over diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md index e646662..03b1ec2 100644 --- a/docs/EVIDENCE.md +++ b/docs/EVIDENCE.md @@ -149,18 +149,19 @@ Recorded against the same `jacquard-core-0.3.0` release asset (linux-x86_64), on Linux 7.0 with the .NET 8.0.427 and 10.0.403 SDKs (runtimes 8.0.33 and 10.0.14). The library still has no runtime dependencies beyond the base class library; the SDK calls the C library's `kill(2)`, `geteuid(2)`, `mkdir(2)`, -`access(2)`, and `statx(2)` (`stat(2)` on macOS, compiled but not run), and on Linux runs +`access(2)`, `realpath(3)`, and `statx(2)` (`stat(2)` on macOS, compiled but not run), and on Linux runs Core's commands through util-linux `setsid`, here 2.41.3. The test dependencies are unchanged. | Lane | Result | |---|---| -| .NET SDK offline (`dotnet test test/JacquardHost.Tests -f net8.0` with the .NET 8 SDK pinned as the CI 8.0 job pins it, and `-f net10.0`, without Core) | 195 tests pass on each runtime and 28 live tests are ignored: the adapter's 96 offline tests and 99 SDK ones. Core commands (20, Linux): exit status, output, and added environment, with stdin at end of file; output over its limits read and counted but not kept; a timeout and a cancellation each kill the command and its child; under `setsid` the command keeps its pid and its group and session ids equal it (with `setsid --fork` the test fails); `sh -c 'sleep 300 & echo $!'` succeeds and its child dies, and survives when the exit kill is disabled; a child that closed its output is killed too; a reaped leader's pid held by an unrelated group leader is not killed; a process outside the group holding the command's stdout is neither killed nor waited for; a stop kills a child and a grandchild; without groups a recorded descendant is killed at the exit (and survives with recording off) and a stop kills the tree; against a fake `/proc` (a seam, pids above Linux's largest), one record reads exactly its entry cap (20 of 50) and its byte cap (5 of 50 entries at 5 KiB), and 1,024 bytes of each 64 KiB `stat` file, a cancellation after the third read ends the record there (the stop's last record then reads its cap), and a cancellation during the second of ten exit kills ends them; a cancellation during the exit cleanup is reported; a missing executable, and one `setsid` cannot exec, are start failures; an escaped flooder holds neither the caller nor more than the kept output; 40 commands leave the descriptor count unchanged, counted at once. Preparation against a fake Core (37, Linux): option and path validation (empty, NUL, and over-long paths refused with excerpts before the file system sees them, a relative path measured absolute); one version check across four threads and two runtimes; prelude discovery once per Core and prelude; output over 8 MiB; the 30-second bound (shortened by a seam) and cancellation killing the command's child; a waiting preparation observing its own cancellation; FIFO and directory sources refused without blocking; sources over 16 MiB refused from their sizes before reading; cancellation between sources, during source validation, during fingerprinting (one 64 KiB piece), between empty prelude files, during the failure cleanup, and as a warm load finishes; prelude walks over 256 MiB, with a FIFO, over the entry cap (lowered by a seam), and 1,500 directories deep; a changed `program.json` binding, a changed Core output, a rewritten digest, and a `program.json` replaced by a link refused; group-writable roots, entries, files, and directories refused and new entries owner-only; a foreign-owned root refused; an executable over 512 MiB refused from its size; source lists over 1,024 refused from their count with no element read; the cleanup stopping at the cap; a symbolic link at the cache root followed to its directory; directories the preparation did not make, and one made at the vacated staging path, left alone; the cache root renamed away and replaced (a seam) before staging, publishing, and a cold or warm load, and after a load read the entry, each refused; the executable replaced by a new file after `--version`, `check`, `hash`, and `run`, a prelude file rewritten after `check`, and a file added to the prelude after `run`, each refused with nothing published; the version check run again for the same executable bytes with a new modification time; an executable only its group and others may run refused by `access(2)` before it runs; a bindings dictionary read without the program's lock held; function, effect, binding, and operation names over 4,096 chars refused and names of exactly 4,096 looked up. Data (25): signatures, hash groups, and display names; round trips and type conformance; the full Int range, Real bits, and `int` narrowing; records, enumerations, unions, nested `Option`, nullable codecs, tuples, custom codecs, and `Dynamic`; `ulong` (2^53 + 1), `nuint`, `char`, `float`, `Half`, `decimal`, `Int128`, `UInt128`, and `BigInteger` conversions and `Dynamic.Of` overloads obsolete as errors and, reached by reflection, refused, while the smaller integer types and `uint` still convert to `Int`; caller names over 4,096 chars refused before lookup (public, made, dynamic, record, and operation names); a lazy codec whose factory threw twice made on the third use and never again across 16 threads; two 64-level shared tuple graphs (2^64 paths) compared equal and unequal at once; recursive types through a lazy codec made once across eight threads, a chain of a million lazy codecs, and one standing for itself refused; the node, text, frame, depth, and count bounds (a list whose items throw is refused from its count, tuples and records before an item is read, a constructor of 20 million fields before it is copied); a raw value 100,000 levels deep refused by its depth; a `Dynamic` over the hard limit refused before it is copied, an endless enumeration after one item past the limit; deep `Dynamic` equality and text without recursion; 64-char excerpts and the 8,192-char message cap, caller exceptions kept as inner exceptions; 100,000 layers of nested unions flattened to one case and round-tripped; a recursive union through a lazy case from eight threads; signature depth 64 accepted, 65 and 5,000 parentheses refused, 100,000 refused by the token cap. Batches and operations (15): argument counts refused before encoding; a throwing codec an invalid argument; cancellation before, during, and after batch validation, each an exact encode count with no worker; every batch validated and the first encoded again; no field or return value that could keep an encoding; a shared graph from a custom codec measured before its type is checked; rows that shrink from 10 to 3 during validation refused after one batch's four encodings; `Save` over a 0666 file leaving a 0600 file, and over a symbolic link replacing the link and leaving its target unchanged; handlers' decoding, results, failures, context, and token, and no handler starting once the call stopped during decoding. Examples (2): usage errors exit 2; the store removes only the temporary file it created | +| .NET SDK offline (`dotnet test test/JacquardHost.Tests -f net8.0` with the .NET 8 SDK pinned as the CI 8.0 job pins it, and `-f net10.0`, without Core) | 202 tests pass on each runtime and 28 live tests are ignored: the adapter's 96 offline tests and 106 SDK ones. Core commands (22, Linux): exit status, output, and added environment, with stdin at end of file; output over its limits read and counted but not kept; a timeout and a cancellation each kill the command and its child; under `setsid` the command keeps its pid and its group and session ids equal it (with `setsid --fork` the test fails); `sh -c 'sleep 300 & echo $!'` succeeds and its child dies, and survives when the exit kill is disabled; a child that closed its output is killed too; a reaped leader's pid held by an unrelated group leader is not killed; a process outside the group holding the command's stdout is neither killed nor waited for; a stop kills a child and a grandchild; without groups a recorded descendant is killed at the exit (and survives with recording off) and a stop kills the tree; against a fake `/proc` (a seam, pids above Linux's largest), one record reads exactly its entry cap (20 of 50) and its byte cap (5 of 50 entries at 5 KiB), and 1,024 bytes of each 64 KiB `stat` file, and a 1,025-byte cap reads 1,024 bytes and then 1; a cancellation after the third read ends the record there, then Core is sent SIGSTOP and is seen stopped (state `T`) before the last record's first read, the last record ends at its 250 ms budget (exactly three reads on a test clock advancing 100 ms per read), and only then is Core sent SIGKILL; on the real clock, with 300 recorded fake descendants whose every read after the stop takes at least 2 ms, the whole cleanup after the cancellation took 501 ms on .NET 8 and on .NET 10 (bound 750 ms; the test allows 1,250) and stopped after about 240 of the 600 reads; a cancellation during the second of ten exit kills still checks the other eight; a cancellation during the exit cleanup is reported; a missing executable, and one `setsid` cannot exec, are start failures; an escaped flooder holds neither the caller nor more than the kept output; 40 commands leave the descriptor count unchanged, counted at once. Preparation against a fake Core (41, Linux): option and path validation (empty, NUL, and over-long paths refused with excerpts before the file system sees them, a relative path measured absolute); one version check across four threads and two runtimes; prelude discovery once per Core and prelude; output over 8 MiB; the 30-second bound (shortened by a seam) and cancellation killing the command's child; a waiting preparation observing its own cancellation; FIFO and directory sources refused without blocking; sources over 16 MiB refused from their sizes before reading; cancellation between sources, during source validation, during fingerprinting (one 64 KiB piece), between empty prelude files, during the failure cleanup, and as a warm load finishes; prelude walks over 256 MiB, with a FIFO, over the entry cap (lowered by a seam), and 1,500 directories deep; a changed `program.json` binding, a changed Core output, a rewritten digest, and a `program.json` replaced by a link refused; group-writable roots, entries, files, and directories refused and new entries owner-only; a foreign-owned root refused; an executable over 512 MiB refused from its size; source lists over 1,024 refused from their count with no element read; the cleanup stopping at the cap; a symbolic link at the cache root followed to its directory; directories the preparation did not make, and one made at the vacated staging path, left alone; the cache root renamed away and replaced (a seam) before staging, publishing, and a cold or warm load, and after a load read the entry, each refused; the executable replaced by a new file after `--version`, `check`, `hash`, and `run`, a prelude file rewritten after `check`, and a file added to the prelude after `run`, each refused with nothing published; the version check run again for the same executable bytes with a new modification time; an executable only its group and others may run refused by `access(2)` before it runs; a bindings dictionary read without the program's lock held; function, effect, binding, and operation names over 4,096 chars refused and names of exactly 4,096 looked up; a parent of the cache root writable by others, or by its group, and a grandparent writable by others, each refused naming it, a sticky world-writable parent and a 0755 one accepted, and a path through a link to the parent resolved to the real directory; the cache root renamed away and back after preparation still used, and renamed away and replaced by a new owner-only directory refused at the next call of a function bound before and one bound after, with no worker started; the missing levels a preparation makes for a cache root each 0700; a source list and a bindings dictionary whose count, element, or enumerator throws wrapped in a `Preparation` or `Binding` exception keeping the cause. Data (26): signatures, hash groups, and display names; round trips and type conformance; the full Int range, Real bits, and `int` narrowing; records, enumerations, unions, nested `Option`, nullable codecs, tuples, custom codecs, and `Dynamic`; `ulong` (2^53 + 1), `nuint`, `char`, `float`, `Half`, `decimal`, `Int128`, `UInt128`, and `BigInteger` conversions and `Dynamic.Of` overloads obsolete as errors and, reached by reflection, refused, while the smaller integer types and `uint` still convert to `Int`; caller names over 4,096 chars refused before lookup (public, made, dynamic, record, and operation names); a lazy codec whose factory threw twice made on the third use and never again across 16 threads; two 64-level shared tuple graphs (2^64 paths) compared equal and unequal at once, and two lists of 1,000 tuples with no shared sub-value compared with no pair set made (a seam counts them); recursive types through a lazy codec made once across eight threads, a chain of a million lazy codecs, and one standing for itself refused; the node, text, frame, depth, and count bounds (a list whose items throw is refused from its count, tuples and records before an item is read, a constructor of 20 million fields before it is copied); a raw value 100,000 levels deep refused by its depth; a `Dynamic` over the hard limit refused before it is copied, an endless enumeration after one item past the limit; deep `Dynamic` equality and text without recursion; 64-char excerpts and the 8,192-char message cap, caller exceptions kept as inner exceptions; 100,000 layers of nested unions flattened to one case and round-tripped; a recursive union through a lazy case from eight threads; signature depth 64 accepted, 65 and 5,000 parentheses refused, 100,000 refused by the token cap. Batches and operations (15): argument counts refused before encoding; a throwing codec an invalid argument; cancellation before, during, and after batch validation, each an exact encode count with no worker; every batch validated and the first encoded again; no field or return value that could keep an encoding; a shared graph from a custom codec measured before its type is checked; rows that shrink from 10 to 3 during validation refused after one batch's four encodings; `Save` over a 0666 file leaving a 0600 file, and over a symbolic link replacing the link and leaving its target unchanged; handlers' decoding, results, failures, context, and token, and no handler starting once the call stopped during decoding. Examples (2): usage errors exit 2; the store removes only the temporary file it created | | .NET SDK live (`JACQUARD_REQUIRE_LIVE=1`, `JACQUARD_BIN` and `JACQUARD_PRELUDE` set) | 20 tests pass on .NET 8 and on .NET 10 (unchanged in this review round): quickstart calls, `ToDynamic`, `Render`, and an owner-only saved inspection; the README examples; the 64-row triage oracle in batches of 8, with a codec failing on the second batch keeping the first and its exception; local refusals before any worker (a field over the Int range named by path, an over-deep call, a wrong type, a wrong count); missing handlers, a misspelled operation, and a 100-name service refused at binding with no handler run; unknown completion once with raw evidence; no handler starting once a real invocation stopped during decoding; a handler's token cancelled when the caller cancels; the 25-case planner oracle; cache reuse without Core, a byte-exact new entry, and corruption refused; a 2,500-`Option` signature left unbound with `safe` working cold and warm; a redirected name in `program.json` refused; the full Int range, Real bits, and nested options; spawn failure and cancellation; output overflow at batch 0; unit results kept after a later batch fails (E1602); too many bindings refused from the count; a throwing result codec keeping the result and its exception, a null mapping message, and an over-long save path; prelude contents keying the cache, and two concurrent preparations publishing one entry | | .NET projects live | 3 further tests pass on both runtimes: all four `JacquardExamples` commands, including duplicate fulfillment, `--inspect` (owner-only), explicit delivery, a failed delivery printing no count, and usage errors; the command in a fresh process exiting 1 with E1613; and the three fulfillment faults recovered explicitly across restart, each receipt delivered once. The CI step's four projects and recovery commands also pass locally | -| Totals | 223 tests on each runtime with Core required, none ignored; 195 pass and 28 are ignored without Core (the .NET 8 lane built with the 8.0.427 SDK through a temporary `global.json`, as CI pins it) | +| Totals | 230 tests on each runtime with Core required, none ignored; 202 pass and 28 are ignored without Core (the .NET 8 lane built with the 8.0.427 SDK through a temporary `global.json`, as CI pins it) | | Regressions checked by mutation (.NET 8) | Each guard was removed in turn and the named tests run; all 42 mutations failed them: the command's exit group kill, its stop kill, `setsid` without forking, the pid-reuse check, the recorded-descendant kill, the stop check after the exit cleanup, and closing the output streams; the source kind check (the FIFO test failed after its one-minute wait), source count, source validation stop, source ceiling, tree entry and byte caps, the executable's size check and per-piece stop, the derived-binding check, the owner, mode, and owner-only checks, the cleanup's stop and cap, leaving the vacated staging path, and the final stop check; the encoder's node budget, the decoder's measurement, the text charge and length check, the list, record, and constructor counts, measuring before the type check, counting arguments before encoding, the batch stop checks before and after validation, the handler's stop check (offline and live), the `Dynamic` limit, excerpts, the message cap, wrapping a field reader's and a record constructor's exceptions, the lazy codec's cycle check (the test then never returned and was stopped), and the store's exclusive temporary file. Separately, checking the cancellation in an exception filter, which .NET runs before the preparation's cleanup, failed the failure-cleanup test until the check moved into the handler | | Review-round regressions (.NET 8) | Each fix was undone in turn and its tests run; all 16 undoings failed them: `Invocation.cs`, `Builtins.cs`, and `Function.cs` as they were before the fixes (the old save left the existing file at 0666; the old lazy codec never ran its factory again; the old batch count made a batch of size -1, reported as an `OverflowException` from the codec), `Dynamic.cs` without the pair set (the equality test timed out after its one-minute wait), and guards turned off: the cache root recheck, the executable and prelude recheck, the identity in the version key, the record's entry cap, byte cap, and per-file read cap, the stop check in a record and in the exit kills, the name limit, the `ulong` refusal, `access(2)` (back to any execute bit), and reading the bindings outside the lock | +| Second review-round regressions (.NET 8) | Each new test was run against the source before its fix and failed: the 1,025-byte cap test against 96c0fc7's `CoreCommand.cs` (the second read took 1,024 bytes, not 1); the SIGSTOP-order, cleanup-budget, and finished-exit-kills tests against 96c0fc7's `CoreCommand.cs` with only the new seams declared (the last record read all 40 entries with no signal; the cleanup took 1,259 ms and all 600 reads; 0 kills after the stop, not 8); the parent and next-call root tests against the round-two `Core.cs`, `Function.cs`, `PreparedProgram.cs`, and `Runtime.cs` (a 0777 parent was accepted, and the call through a replaced root ran a worker); the owner-only levels test against the `Runtime.cs` before that fix (a level made with the umask, 0775 here, was refused as a group-writable parent); the pair-set test against 96c0fc7's `Dynamic.cs` with a counter at its allocation (2 sets for unshared values); and the wrapping test against the round-two `PreparedProgram.cs` and `Runtime.cs` (the `InvalidOperationException` escaped). Not tested: a parent owned by another user, which an unprivileged test cannot make | | Build | `TreatWarningsAsErrors`, nullable reference types, and the `8.0-recommended` analyzers are clean with both SDKs | | Unchanged lanes | The adapter's 96 offline and 5 live tests (the README-example check now covers all four examples), the 21-recording kit replay, `scripts/core_pin.py check`, `tests/test_core_pin.py`, and `scripts/check_distribution.py` (the source archive now carries the SDK and the example command) pass | diff --git a/docs/LIMITS.md b/docs/LIMITS.md index 4d25a81..35aaa61 100644 --- a/docs/LIMITS.md +++ b/docs/LIMITS.md @@ -29,8 +29,8 @@ also updates this document. | .NET worker process tree, not group | .NET cannot start the worker in its own process group, so the .NET host kills a process tree. At expiry, cancellation, or disposal of a running worker, it calls `Process.Kill(true)`, which stops the worker, lists its children from the process table, kills it, and does the same for each child. A descendant that was already reparented keeps running: one whose parent exited, or one that daemonized. So does one started after its parent was listed. Descendants are found and signalled by pid, with nothing pinning their identity: one that exits, and is reaped once the worker is killed, between the listing and the signal can have its pid reused by an unrelated process, which is then signalled (the Java host signals process handles, and the Rust and Go hosts a process group). Listing the process table takes time proportional to the number of processes on the machine. The runtime reaps the worker as soon as it exits. A descendant still holding stdout or stderr after the exit is detected and, after a one-second grace, reported as a carrier loss, as the Go and Rust hosts report it; it is not stopped. .NET reports a signal as exit status 128 + its number, so only the host's own SIGKILL is reported as a signal. | An OS-level containment mechanism (for example a cgroup or job object) with its own evidence, or a .NET process-group API. | | .NET pipe I/O goes through `poll(2)` | .NET's pipe streams have no read or write that a deadline or a cancellation can interrupt, so the .NET host never calls one that can block: the caller's thread asks the C library's `poll(2)` whether a pipe is ready, with a poll timeout of at most 5 ms, rechecking the deadline, the `CancellationToken`, and disposal between polls, and then reads only bytes already in the pipe or writes at most `PIPE_BUF` bytes (4096 on Linux, 512 on macOS), which a writable pipe accepts at once. No thread is left blocked on a worker pipe, even while an escaped descendant holds stdin without reading or floods stdout or stderr. The `pollfd` layout and event bits are the same on Linux and macOS; elsewhere, including Windows, the host does not start a worker and reports a spawn failure. .NET opens its own descriptors close-on-exec, but passes the worker any descriptor the host process itself inherited without that flag. | A .NET API for interruptible or non-blocking process pipes, and CI on each added platform. | | .NET handler cancellation is cooperative | Each .NET handler runs on its own background thread. At the deadline or on cancellation the host kills the worker's process tree, closes its input, and sends no late response. `HandlerContext.IsCancelled` turns true (it reads the deadline exactly), and stays true once the invocation ends. `HandlerContext.CancellationToken` is cancelled when the call stops or ends, never earlier (a lifetime of any length does not cancel it before its deadline). While a handler runs, the caller's thread rechecks the deadline and the caller's token between poll waits of at most 5 ms; that is the requested poll timeout, not a bound on the interval between checks, which reading pending stderr (up to 1 MiB per pass) and thread scheduling can lengthen. Whichever check sees the stop, the token's cancellation starts before the worker is killed; otherwise it starts shortly after the invocation ends. The cancellation runs on a short-lived thread the host starts for it, one per invocation that gave a handler a token, so it does not wait for the shared thread pool. If no thread can be created or started, the host queues it as a pool work item, which runs only when a pool thread is free, so a starved pool delays it. If the work item cannot be queued either, the host gives up: the token is never cancelled (`HandlerContext.IsCancelled` still turns true), `InvocationResult.HandlerTokenNotCancelled` is set, and a handler waiting only on the token keeps waiting; `Invoke` still returns its result and disposes the worker. Callbacks a handler registers on that token run on that thread, which the host never waits for; a callback that blocks holds that thread, and anything a callback throws is discarded. A callback registered after the token is cancelled runs at once on the handler's own thread and throws there, as with any `CancellationToken`. A handler that ignores both keeps running on its thread until it returns. No outside action is stopped or rolled back. A stack overflow in a handler ends the process, as it does any .NET code. | A separately enforced operation lifetime with explicit completion and reconciliation evidence. | -| .NET SDK Core command descendants | Each Core `--version`, `check`, `hash`, and `run` command must exit within 30 seconds and keeps at most 8 MiB of stdout and 4 KiB of stderr; cancelling the token stops it. Its output is read only after `poll(2)` reports it ready, on the caller's thread. On Linux with a `setsid` at `/usr/bin/setsid` or `/bin/setsid` (util-linux's, or a compatible one such as BusyBox's; any executable there is used), the command runs as `setsid -- `, which execs without forking, so Core keeps its pid and leads its own process group; when `setsid` cannot exec Core (exit status 126 or 127 with its own `setsid: ` message) the command is reported as a start failure. When the command exits, and on a timeout or cancellation, the host sends `kill(-pgid, SIGKILL)` to that group. Once .NET has reaped Core, a live process holding Core's pid means the id was reused and nothing is killed; a process that took Core's pid after a full pid wraparound, made its own group, and forked between that check and the kill could still have that group killed, the same window as the Rust SDK's `kill(-pgid)`. A descendant that leaves the group (calls `setsid`) is neither detected nor killed; the host does not wait for the end of the output after the exit, and closes its ends, so that descendant's next write fails. Without `setsid` (Linux without one at those paths), descendants are recorded from `/proc` about every 10 ms by pid and start time, and every recorded descendant still alive with that start time is killed when the command returns (the start time is read just before the kill, a short window in which a reused pid could be signalled, as with the JDK's process handles). Each record reads at most 16,384 `/proc` entries, 4 MiB of `stat` files, and 1 KiB of each, at most 16,384 descendants are kept, and a record ends at the deadline or a cancellation, checked before each entry; the kills at the exit check them before each descendant, so a stop that lands there leaves the descendants not yet killed running. On a timeout or cancellation, a last record bounded by the same caps (not ended by the stop) is made, then the command is killed, then every recorded descendant; .NET's `Process.Kill(true)` is not used on Linux. Descendants past the caps, and fast fork-and-exit chains between records, are missed. macOS has no `/proc`, so there only `Process.Kill(true)` on a stop applies. The .NET worker does not run in its own group (see the worker row above). | An OS-level containment mechanism, or a .NET process-group API, with its own evidence. | -| .NET SDK cache and source reads | A cache entry is verified against the file digests in its `program.json`, and every binding is derived again from Core's cached raw output, so an accidental or partial change is refused. No record inside the cache resists someone who can rewrite it: an output, its recorded digest, and the bindings changed together load. So the cache's owner is trusted and only the owner may write it: preparation refuses a cache root, an entry, or any file or directory in one that the current user does not own or that its group or others may write (read with `statx(2)` on Linux and `stat(2)` on macOS, without following links), on create and on load, and new entries are owner-only. A symbolic link at the cache root is followed once and the directory it names checked; nothing inside it is followed. The root is then used by its path: its device and inode, read at the owner check, are compared again before the staging directory is made, before publishing, and before and after each load, and a replaced root is refused. Its parent directories are not checked, so another user who may write a parent can still replace the root in the few system calls between one of those comparisons and the step that follows it; a cache under directories only its owner can write is not exposed. The Core executable and every prelude file and directory are fingerprinted by path, and Core opens them by path later: each one's identity (device, inode, size, mode, owner, and modification and status-change times) is read as it is fingerprinted and compared after Core's commands finish (inside the version check and prelude discovery, before a new entry is published, and before an existing one is loaded), and any change refuses the preparation. Not detected: a file swapped and then restored with the same inode and times, which an unprivileged user can do only within the file system's timestamp granularity, since writing or renaming a file updates its status-change time (on Linux from a coarse clock, a few milliseconds); files Core loads besides the executable and prelude (its shared libraries and loader); and any change after the last comparison, since a prepared program's workers later run the executable by path without checking it again. The macOS status calls are compiled but not exercised by any test. A failed preparation's cleanup removes at most 100,000 entries of its staging directory, created exclusively, and ends at a cancellation, so a stopped preparation leaves an unpublished `prepare-*` directory in the cache root; it is never loaded, and nothing removes it later (a name and an age cannot show that the SDK made a directory or that no preparation is using it), so remove leftovers yourself when no preparation is running. At most 1,024 sources are taken, refused from the list's count before any element is read, and they must be regular files totalling at most 16 MiB, the Core executable at most 512 MiB, and the prelude and cache walks read at most 100,000 entries and 256 MiB of regular files under one root, each file checked from its size before it is read and again while it is read; a source or cache file replaced by a FIFO (or, in the cache, by a link) between that check and its open can still block or redirect the open, which a cancellation does not end. | An open that cannot block on a FIFO or follow a link; directory-relative (`openat`) access pinned to the checked root; Core reading descriptors the host opened and verified. | +| .NET SDK Core command descendants | Each Core `--version`, `check`, `hash`, and `run` command must exit within 30 seconds and keeps at most 8 MiB of stdout and 4 KiB of stderr; cancelling the token stops it. Its output is read only after `poll(2)` reports it ready, on the caller's thread. On Linux with a `setsid` at `/usr/bin/setsid` or `/bin/setsid` (util-linux's, or a compatible one such as BusyBox's; any executable there is used), the command runs as `setsid -- `, which execs without forking, so Core keeps its pid and leads its own process group; when `setsid` cannot exec Core (exit status 126 or 127 with its own `setsid: ` message) the command is reported as a start failure. When the command exits, and on a timeout or cancellation, the host sends `kill(-pgid, SIGKILL)` to that group. Once .NET has reaped Core, a live process holding Core's pid means the id was reused and nothing is killed; a process that took Core's pid after a full pid wraparound, made its own group, and forked between that check and the kill could still have that group killed, the same window as the Rust SDK's `kill(-pgid)`. A descendant that leaves the group (calls `setsid`) is neither detected nor killed; the host does not wait for the end of the output after the exit, and closes its ends, so that descendant's next write fails. With `setsid`, the wait for a killed Core to exit after a stop ends 750 ms after the stop. Without `setsid` (Linux without one at those paths), descendants are recorded from `/proc` about every 10 ms by pid and start time, and every recorded descendant still alive with that start time is killed when the command returns (the start time is read just before the kill, a short window in which a reused pid could be signalled, as with the JDK's process handles). Each record reads at most 16,384 `/proc` entries and 4 MiB of `stat` files in all, each read limited to 1 KiB and to the bytes left, at most 16,384 descendants are kept, and a record ends at the deadline or a cancellation, checked before each entry. The kills when Core exits by itself run to the end; a stop that lands during them lets them go on for at most 250 ms more. On a timeout or cancellation, Core is first sent `SIGSTOP` (only while its start time still matches the one read when it started), so it starts no more children and its children stay its children; then a last record is made, within the same caps, until 250 ms after the stop; then Core is killed, and every recorded descendant until 500 ms after the stop; then the host waits for Core to exit until 750 ms after the stop. Each budget is checked before each entry or descendant, so the cleanup returns within 750 ms of the stop plus one `stat` read and one signal. Only Core is frozen: its descendants run until they are killed, so one whose parent exits before the last record reaches it is reparented and missed. Descendants past the caps or the budgets, and fast fork-and-exit chains between records, are missed too. .NET's `Process.Kill(true)` is not used on Linux. macOS has no `/proc`, so there only `Process.Kill(true)` on a stop applies. The .NET worker does not run in its own group (see the worker row above). | An OS-level containment mechanism, or a .NET process-group API, with its own evidence. | +| .NET SDK cache and source reads | A cache entry is verified against the file digests in its `program.json`, and every binding is derived again from Core's cached raw output, so an accidental or partial change is refused. No record inside the cache resists someone who can rewrite it: an output, its recorded digest, and the bindings changed together load. So the cache's owner is trusted and only the owner may write it: preparation refuses a cache root, an entry, or any file or directory in one that the current user does not own or that its group or others may write (read with `statx(2)` on Linux and `stat(2)` on macOS, without following links), on create and on load, and new entries are owner-only. Every symbolic link in the cache root's path is resolved once (`realpath(3)`), and the real directory is checked and used from then on; nothing inside it is followed. Every directory above the root, up to `/`, must be owned by the current user or root and must not be writable by its group or others unless it has the sticky bit (as OpenSSH checks the directories above its configuration); otherwise preparation is refused with a message naming the directory. So no other user can rename the root away and put another directory at its path. The directories a preparation makes for a missing root are each made owner-only, whatever the `umask`, but an existing parent that is group-writable only because of a `umask` of 002 (user-private groups) is refused too: run `chmod go-w` on it, or put the cache under owner-only directories. The root is used by its path, also by every worker a prepared program starts (on `//store`): its device and inode, read at the owner check, are compared again before the staging directory is made, before publishing, before and after each load, and before each worker starts, and a changed root is refused (a call then fails with a `Preparation` exception and starts no worker). Not covered: replacement by the same user or by root, whom the owner check trusts; a root renamed away and back between two comparisons, which keeps its inode (once the parents are checked, only its owner or root can rename it); and a directory above the root made writable by others after preparation, which the next preparation refuses but a prepared program does not notice (its comparisons still refuse a replaced root, except in the few system calls between the last one and a worker's start). The Core executable and every prelude file and directory are fingerprinted by path, and Core opens them by path later: each one's identity (device, inode, size, mode, owner, and modification and status-change times) is read as it is fingerprinted and compared after Core's commands finish (inside the version check and prelude discovery, before a new entry is published, and before an existing one is loaded), and any change refuses the preparation. Not detected: a file swapped and then restored with the same inode and times, which an unprivileged user can do only within the file system's timestamp granularity, since writing or renaming a file updates its status-change time (on Linux from a coarse clock, a few milliseconds); files Core loads besides the executable and prelude (its shared libraries and loader); and any change after the last comparison, since a prepared program's workers later run the executable and read the prelude (`JACQUARD_PRELUDE`) by path without checking them again. The macOS status calls are compiled but not exercised by any test. A failed preparation's cleanup removes at most 100,000 entries of its staging directory, created exclusively, and ends at a cancellation, so a stopped preparation leaves an unpublished `prepare-*` directory in the cache root; it is never loaded, and nothing removes it later (a name and an age cannot show that the SDK made a directory or that no preparation is using it), so remove leftovers yourself when no preparation is running. At most 1,024 sources are taken, refused from the list's count before any element is read, and they must be regular files totalling at most 16 MiB, the Core executable at most 512 MiB, and the prelude and cache walks read at most 100,000 entries and 256 MiB of regular files under one root, each file checked from its size before it is read and again while it is read; a source or cache file replaced by a FIFO (or, in the cache, by a link) between that check and its open can still block or redirect the open, which a cancellation does not end. | An open that cannot block on a FIFO or follow a link; directory-relative (`openat`) access pinned to the checked root; Core reading descriptors the host opened and verified. | | .NET SDK cancellation is by token only | Preparation, Core's commands, `Call`, `Invoke`, and `MapBatches` observe the `CancellationToken` they take. `Thread.Interrupt` is not a stop signal: the SDK's own waits are `poll(2)` sleeps, which neither end at an interrupt nor consume it. The API is synchronous; there are no `Task`-returning variants. | `Task`-returning variants with the same bounds, if callers need them. | | .NET SDK signature depth | A checked function whose signature nests types deeper than 64 levels (`max_json_depth`, the protocol's value-depth limit), names more than 4,096 types, or has more than 16,384 tokens is left unbound, like a polymorphic one: preparation succeeds, the other functions bind, and binding it fails with a `Binding` exception naming the limit. The SDK parses no operation signatures (Core's `check --print-sigs` prints none). | None needed for depth, since a deeper value cannot cross the protocol; Core's structured host interface exporter for the rest. | | .NET SDK record fields are declared, not exported | A record codec's fields, in the order given, are a constructor's fields; the pinned CLI exports no field types, so a wrong order is not detected locally, and Core refuses it only where the swapped fields' types differ. | Core's structured host interface exporter, so bindings can be generated or checked. | diff --git a/dotnet/README.md b/dotnet/README.md index f11db61..5989d05 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -130,9 +130,9 @@ parses none; an operation's arguments are checked by the protocol's limits when they arrive. The cache defaults to `.scratch/jacquard-host-dotnet` under the working -directory; `PrepareOptions.CacheDir` changes it. A symbolic link at the cache -root is followed once, and the directory it names is checked and used; -nothing inside it is followed. A cache entry keeps Core's raw +directory; `PrepareOptions.CacheDir` changes it. Every symbolic link in the +cache root's path is resolved once (`realpath(3)`), and the real directory is +checked and used; nothing inside it is followed. A cache entry keeps Core's raw `check --print-sigs` output and the `hash` output for the source and each prelude file beside the store, and `program.json` records every file's digest. Every binding (names, constructors, signatures, and declaration groups) is @@ -147,13 +147,22 @@ entry, or any file or directory in one that the current user does not own or that its group or others may write, both when it creates an entry and when it loads one, and it makes each new entry's directories and files owner-only (Core's store included). The owner and mode come from `statx(2)` on Linux and -`stat(2)` on macOS, without following links. The cache root is then used by -its path, so its device and inode, read at that check, are compared again -before the staging directory is made, before an entry is published, and -before and after each load; a root renamed away and replaced is refused. Its -parent directories are not checked, so a root replaced between one of those -checks and the step that follows it is not detected: keep the cache under -directories only you can write. +`stat(2)` on macOS, without following links. Every directory above the +root, up to `/`, must be owned by you or root and must not be writable by its +group or others unless it has the sticky bit (as `/tmp` has), as OpenSSH +checks the directories above its configuration; otherwise preparation is +refused, naming the directory. So no other user can rename the root away and +put another directory at its path. The directories a preparation makes for a +missing root are each made owner-only, whatever the `umask`; an existing one +left group-writable by a `umask` of 002 is refused: run `chmod go-w` on it, or +choose a cache under owner-only directories. The root is used by its path, also by every +worker a prepared program starts, so its device and inode, read at the owner +check, are compared again before the staging directory is made, before an +entry is published, before and after each load, and before each worker +starts; a changed root is refused (a call fails with a `Preparation` +exception and starts no worker). The owner check trusts you and root: a +replacement by the same user is not detected, nor is a root renamed away and +back between two comparisons (it keeps its inode). The executable and every prelude file and directory are fingerprinted by path, and Core opens them by path afterwards. Each one's identity (device, @@ -190,7 +199,12 @@ a `Dynamic` to itself; anything else, lists and records included, is `Int128`, `UInt128`, or `BigInteger` does not compile as an argument or in `Dynamic.Of`: those conversions are marked obsolete as errors, with the reason, because C# would otherwise convert a `ulong` through `double` (2^53 + 1 -would arrive as 2^53) and a `char` to an `Int` (see [data](#data)). `Call` +would arrive as 2^53) and a `char` to an `Int` (see [data](#data)). +`Codecs.Real` is a codec of `double`, so a `long` given to it +(`Codecs.Real.Arg(total)`, or a record field reader returning a `long` for a +`Codecs.Real` field) is converted by C#'s implicit numeric conversion before +the SDK sees it: that is your explicit choice of a Real, as in Java, and it is +exact only up to 2^53. `Call` decodes the result with the codec you pass; `Invoke` keeps the whole `Invocation`. Overloads that take an argument list and a `CancellationToken` kill the worker when it is cancelled. `MapBatches` reads the number of rows @@ -411,10 +425,12 @@ keep their usual .NET types: passing `null` where an object is required throws `ArgumentNullException`; building a codec that cannot work (an enumeration map missing a member, a union case that is not a subtype) or a `Dynamic` over the hard limit throws `ArgumentException` at construction; -`Invocation.Save` throws `IOException` or `UnauthorizedAccessException`; and -an exception from a collection you pass (its `Count`, indexer, or -enumerator) propagates unchanged, except for `MapBatches` rows, which are read -while a batch is encoded and so are reported as a codec failure. A +`Invocation.Save` throws `IOException` or `UnauthorizedAccessException`; an +exception from the source list `Prepare` takes or the bindings dictionary +`Function` takes (its `Count`, indexer, or enumerator) is wrapped, with the +cause, in a `Preparation` or `Binding` exception; and one from any other +collection you pass propagates unchanged, except for `MapBatches` rows, which +are read while a batch is encoded and so are reported as a codec failure. A `JacquardException`'s `Kind` is: - `InvalidArgument`: a local refusal, such as a value of the wrong type or one too large or too deep for a frame. No worker starts. `Invoke` throws @@ -468,7 +484,10 @@ The command reads the rules and synthetic inputs from the repository's [`examples/`](../examples/README.md), from the repository root or `dotnet/` (or `JACQUARD_EXAMPLES=DIR`), so it runs from a clone. Custom input JSON may follow the project name. `--inspect PATH` saves frames; triage adds a -numbered suffix per batch. State defaults to `.scratch/dotnet-examples/`. The +numbered suffix per batch. State defaults to `.scratch/dotnet-examples/`; +its program cache is refused while any directory above it is writable by its +group or others (as a clone made with a `umask` of 002 is), so run +`chmod go-w` on those or pass a `--workdir` under owner-only directories. The exit status is 1 for a Jacquard or SDK failure and 2 for usage or other errors. @@ -736,8 +755,8 @@ forking: Core keeps its pid and leads a new process group whose id is that pid (a test checks both). Every descendant that does not leave the group shares it. The host kills that group with `kill(-pgid, SIGKILL)` when the command exits, and on a timeout or cancellation (then also killing Core itself, -waiting up to one second for its exit, and killing the group again for -members forked meanwhile). So `sh -c 'sleep 300 & echo $!'` succeeds, as in +waiting for its exit until 750 ms after the stop, and killing the group again +for members forked meanwhile). So `sh -c 'sleep 300 & echo $!'` succeeds, as in the Rust SDK, and its child has been sent SIGKILL by the time it returns. Once .NET has reaped Core, a live process holding Core's pid means the id was reused, so the old group is empty and nothing is killed; a process that took Core's pid after a @@ -755,15 +774,24 @@ descendants are recorded from `/proc` about every 10 ms, by pid and start time, and every recorded descendant still alive with that start time is killed when the command returns (the start time is read just before the kill, as the JDK's process handles do). Each record reads at most 16,384 -`/proc` entries, 4 MiB of `stat` files, and 1 KiB of each, at most 16,384 -descendants are kept, and a record ends at the deadline or a cancellation, -checked before each entry; the kills at the exit check them before each -descendant, so a stop there leaves the rest running. On a timeout or -cancellation, a last record bounded by the same caps (but not ended by the -stop) is made, then the command is killed, then every recorded descendant; -.NET's `Process.Kill(true)`, which reads every process on the machine, is not -used. A descendant past the caps, or one started and reparented between two -records, keeps running. macOS has no `/proc`, so nothing is recorded there: +`/proc` entries and 4 MiB of `stat` files in all, each read limited to 1 KiB +and to the bytes left, at most 16,384 descendants are kept, and a record ends +at the deadline or a cancellation, checked before each entry. The kills when +Core exits by itself run to the end; a stop that lands during them lets them +go on for at most 250 ms more. On a timeout or cancellation, Core is first +sent `SIGSTOP` (only while its start time matches the one read when it +started), so it starts no more children and its children stay its children; +then a last record is made, within the same caps, until 250 ms after the +stop; then Core is killed, and every recorded descendant until 500 ms after +the stop; then the host waits for Core's exit until 750 ms after the stop. +Each budget is checked before each entry or descendant, so the cleanup +returns within 750 ms of the stop plus one `stat` read and one signal (tests +check the order and the budgets on a test clock, and the whole bound on the +real clock). .NET's `Process.Kill(true)`, which reads every process on the +machine, is not used. Only Core is frozen, so a descendant whose parent exits +before the last record reaches it is reparented and keeps running, as do +one past the caps or the budgets and one started and reparented between two +records. macOS has no `/proc`, so nothing is recorded there: only .NET's tree kill on a stop applies. ## Conformance kit From 16ab90c6734bd43b85e5fd2ceab7148b793dfc65 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 09:41:20 -0400 Subject: [PATCH 27/52] Measure the cleanup budgets after a stop on the test clock instead of the real one --- dotnet/src/JacquardHost/Sdk/CoreCommand.cs | 4 ++ .../Sdk/CoreCommandTests.cs | 53 +++++++++++++------ 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs index 07aaaa7..3d8e71c 100644 --- a/dotnet/src/JacquardHost/Sdk/CoreCommand.cs +++ b/dotnet/src/JacquardHost/Sdk/CoreCommand.cs @@ -141,6 +141,9 @@ internal static class CoreCommand /// Test seam: runs before the host sends the command itself a signal on a stop without groups, with its pid and the signal. Null outside tests. internal static volatile Action? SignalHook; + /// Test seam: receives the end of each wait for a killed command to exit, on the budgets' clock. Null outside tests. + internal static volatile Action? ExitWaitHook; + /// Test seam: receives the pid of each command once it has started. Null outside tests. internal static volatile Action? StartedHook; @@ -858,6 +861,7 @@ private static void KillQuietly(Process process, bool entireTree) /// private static void AwaitExit(Process process, long end) { + ExitWaitHook?.Invoke(end); while (!HasExited(process) && Now() < end) { Posix.Poll(Span.Empty, 1); diff --git a/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs index 294755f..5afc236 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/CoreCommandTests.cs @@ -44,6 +44,7 @@ public void CleanUp() CoreCommand.StatReadHook = null; CoreCommand.ClockHook = null; CoreCommand.SignalHook = null; + CoreCommand.ExitWaitHook = null; foreach (string file in Directory.EnumerateFiles(root, "*.pid")) { Support.Kill(int.Parse(File.ReadAllText(file).Trim(), CultureInfo.InvariantCulture)); @@ -458,47 +459,67 @@ public void AStopFreezesTheCommandBeforeItsLastRecordAndBoundsIt() } /// - /// The whole cleanup after a stop, on the real clock, ends within - /// (750 ms) plus scheduling slack, - /// even when the last record and the kills of 300 recorded descendants - /// would take 600 ms each (each fake stat read after the stop - /// takes at least 2 ms): the record and the kills are cut at their - /// budgets. The measured time is written to the test output. + /// The cleanup after a stop ends at its budgets on the test clock, even + /// when the last record and the kills of 300 recorded descendants would + /// take 600 ms each: the clock stands still except that each + /// stat read after the stop advances it by 2 ms. The last record + /// reads at 0, 2, ..., 248 ms after the stop (125 reads) and ends at + /// 250 ms, when the command is killed; the kills read at 250, ..., 498 ms + /// (125 reads) and end at 500 ms; the wait for the command to exit ends + /// at 750 ms. Without the budgets, all 300 entries would be read in each. /// [Test] - public void TheCleanupAfterAStopEndsWithinItsBudget() + public void TheCleanupAfterAStopEndsAtItsBudgets() { CoreCommand.WithoutGroups = true; string proc = Path.Combine(root, "proc"); CoreCommand.ProcRoot = proc; CoreCommand.StartedHook = pid => FakeProc("proc", 300, pid); + long start = 1_000_000_000; + long clock = start; + long ms = Stopwatch.Frequency / 1000; + CoreCommand.ClockHook = () => Interlocked.Read(ref clock); int recorded = 0; using var cancel = new CancellationTokenSource(); - long cancelled = 0; CoreCommand.RecordedHook = pids => { if (recorded == 0) { recorded = pids.Count; - cancelled = Stopwatch.GetTimestamp(); cancel.Cancel(); } }; - int after = 0; + var reads = new List(); + long killed = -1; + CoreCommand.SignalHook = (pid, signal) => + { + if (signal == Unix.Sigkill) + { + killed = Interlocked.Read(ref clock) - start; + reads.Add(-1); + } + }; CoreCommand.StatReadHook = (path, bytes, canStop) => { if (!canStop) { - after++; - Thread.Sleep(2); + reads.Add(Interlocked.Read(ref clock) - start); + Interlocked.Add(ref clock, 2 * ms); } }; + var waits = new List(); + CoreCommand.ExitWaitHook = end => waits.Add(end - start); Support.Throws(() => Run(Waiting(Path.Combine(root, "never")), cancellationToken: cancel.Token)); - double elapsed = Stopwatch.GetElapsedTime(cancelled).TotalMilliseconds; - TestContext.Out.WriteLine("cleanup after the stop: " + elapsed.ToString("F1", CultureInfo.InvariantCulture) + " ms; reads after the stop: " + after.ToString(CultureInfo.InvariantCulture)); Assert.That(recorded, Is.EqualTo(300)); - Assert.That(after, Is.LessThan(600), "the record and the kills were not cut at their budgets"); - Assert.That(elapsed, Is.LessThan(CoreCommand.StopBudgetMs + 500)); + int split = reads.IndexOf(-1); + Assert.That(split, Is.GreaterThanOrEqualTo(0), "the command was not killed"); + List record = reads.Take(split).ToList(); + List kills = reads.Skip(split + 1).ToList(); + Assert.That(record, Is.EqualTo(Enumerable.Range(0, 125).Select(i => 2L * i * ms)), "the last record ends at 250 ms"); + Assert.That(killed, Is.EqualTo(250 * ms), "the command is killed when the record ends"); + Assert.That(kills, Is.EqualTo(Enumerable.Range(125, 125).Select(i => 2L * i * ms)), "the kills end at 500 ms"); + Assert.That(reads.Count - 1, Is.LessThan(600), "the record and the kills were not cut at their budgets"); + Assert.That(waits, Is.EqualTo(new[] { (long)CoreCommand.StopBudgetMs * ms }), "the exit wait ends at 750 ms"); } /// From 0540b7884f6ba32a0b0dd307ee9f8eaee5e4cad7 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 09:41:28 -0400 Subject: [PATCH 28/52] Accept a cache parent writable only by the user's private group, and name every unsafe parent --- dotnet/src/JacquardHost/Sdk/Core.cs | 142 +++++++++-- dotnet/src/JacquardHost/Sdk/Unix.cs | 234 ++++++++++++++++-- .../JacquardHost.Tests/Sdk/RuntimeTests.cs | 148 ++++++++++- 3 files changed, 472 insertions(+), 52 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Core.cs b/dotnet/src/JacquardHost/Sdk/Core.cs index 3013fbc..4a3f70b 100644 --- a/dotnet/src/JacquardHost/Sdk/Core.cs +++ b/dotnet/src/JacquardHost/Sdk/Core.cs @@ -584,47 +584,145 @@ internal static Unix.Status Check(string path, Access access) /// /// Refuses the cache root , a real path with no - /// links in it, unless every directory above it, up to /, is owned - /// by this user or by root, and neither its group nor others may write it - /// unless it has the sticky bit (as OpenSSH checks the directories above - /// its configuration). Otherwise another user could rename the root away - /// and put another directory at its path at any time, and every later - /// use of the path, a prepared program's workers included, would use it. - /// In a sticky directory only the owner of an entry (or root) may rename - /// it, and the next directory down is checked in turn. + /// links in it, unless every directory above it, up to /, is safe + /// (see ). Otherwise another user could + /// rename the root away and put another directory at its path at any + /// time, and every later use of the path, a prepared program's workers + /// included, would use it. /// - /// A preparation failure naming the directory. + /// A preparation failure naming every unsafe directory. internal static void CheckAncestors(string root) + { + if (Path.GetDirectoryName(root) is { } parent && UnsafeDirectories(parent, "cache directory " + root) is { } problem) + { + throw JacquardException.Preparation(problem); + } + } + + /// + /// Refuses , a real path with no links in it, + /// and the directories above it before any missing level below it is + /// made, so a refused preparation makes nothing. + /// + /// A preparation failure naming every unsafe directory. + internal static void CheckDirectories(string directory, string root) + { + if (UnsafeDirectories(directory, "cache directory " + root) is { } problem) + { + throw JacquardException.Preparation(problem); + } + } + + /// + /// Test seam: answers the password database lookups of the private-group + /// rule in place of . Null outside tests. + /// + internal static volatile Func? UserHook; + + /// + /// Test seam: answers the group database lookups of the private-group + /// rule in place of . Null outside tests. + /// + internal static volatile Func? GroupHook; + + /// The most unsafe directories one refusal names; the rest are counted. + internal const int UnsafeListed = 16; + + /// + /// Checks , a real path with no links in it, and + /// every directory above it up to /, with lstat: each must + /// be a directory owned by this user or by root, and neither its group + /// nor others may write it, unless it has the sticky bit (as OpenSSH + /// checks the directories above its configuration) or the group is this + /// user's private group. In a sticky directory only the owner of an entry + /// (or the directory's owner, or root) may rename or remove it, and the + /// next directory down is checked in turn. The private group is Debian + /// OpenSSH's rule: the directory's group is the user's primary group, + /// that group's name is the user's name, and it lists no members or only + /// the user. Every unsafe directory is named, nearest first, up to + /// . + /// + /// The nearest directory to check. + /// What the directories hold, which starts the message. + /// The refusal naming every unsafe directory, or null when all are safe or there are no POSIX permissions. + /// A directory's status cannot be read. + internal static string? UnsafeDirectories(string first, string subject) { if (!Unix.Supported) { - return; + return null; } uint user = Unix.EffectiveUser(); - string? directory = Path.GetDirectoryName(root); - while (directory is not null) + bool looked = false; + Unix.Account? account = null; + Unix.GroupEntry? group = null; + var unsafeOnes = new List(); + int count = 0; + for (string? directory = first; directory is not null; directory = Path.GetDirectoryName(directory)) { Unix.Status status = StatOrFail(directory, follow: false, "cannot read "); + string? reason = null; if (status.Kind != Unix.Kind.Directory) { - throw JacquardException.Preparation("cache directory parent " + directory + " is not a directory"); + reason = "not a directory"; } - - if (status.Owner != user && status.Owner != 0) + else if (status.Owner != user && status.Owner != 0) { - throw JacquardException.Preparation("cache directory parent " + directory + " is owned by another user, who could replace the cache;" - + " keep the cache under directories owned by you or root"); + reason = "owned by another user"; } + else if ((status.Mode & 0x200) == 0) + { + // 0o002: other write; 0o020: group write; 0o1000: sticky. + if ((status.Mode & 0x2) != 0) + { + reason = "writable by others"; + } + else if ((status.Mode & 0x10) != 0) + { + if (!looked) + { + looked = true; + account = UserHook is { } users ? users(user) : Unix.User(user); + if (account is { } known) + { + group = GroupHook is { } groups ? groups(known.Group) : Unix.GroupOf(known.Group); + } + } - // 0o022: group or other write; 0o1000: sticky. - if ((status.Mode & 0x12) != 0 && (status.Mode & 0x200) == 0) + if (!PrivateGroup(status, account, group)) + { + reason = "writable by its group, which is not your private group"; + } + } + } + + if (reason is not null) { - throw JacquardException.Preparation("cache directory parent " + directory + " is group- or world-writable without the sticky bit," - + " so another user could replace the cache (make it, for example, chmod go-w " + directory + ")"); + if (++count <= UnsafeListed) + { + unsafeOnes.Add(directory + " (" + reason + ")"); + } } + } - directory = Path.GetDirectoryName(directory); + if (count == 0) + { + return null; } + + string more = count > UnsafeListed ? "; and " + (count - UnsafeListed).ToString(CultureInfo.InvariantCulture) + " more" : string.Empty; + + // The remedy comes before the list, so a message cut at its limit keeps it. + return subject + " could be replaced by another user through the directories above it." + + " Make each of these writable only by you (for example chmod go-w DIR), or use a location under directories only you or root may write: " + + string.Join("; ", unsafeOnes) + more; } + + /// Debian OpenSSH's private-group rule for a group-writable . + private static bool PrivateGroup(Unix.Status status, Unix.Account? account, Unix.GroupEntry? group) => + account is { } user && group is not null + && status.Group == user.Group + && string.Equals(group.Name, user.Name, StringComparison.Ordinal) + && (group.Members.Count == 0 || (group.Members.Count == 1 && string.Equals(group.Members[0], user.Name, StringComparison.Ordinal))); } diff --git a/dotnet/src/JacquardHost/Sdk/Unix.cs b/dotnet/src/JacquardHost/Sdk/Unix.cs index 4f2d4d0..bd10f10 100644 --- a/dotnet/src/JacquardHost/Sdk/Unix.cs +++ b/dotnet/src/JacquardHost/Sdk/Unix.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Globalization; using System.IO; using System.Runtime.InteropServices; @@ -13,8 +14,9 @@ namespace JacquardLang.Host.Sdk; /// lstat(2) on macOS) for the cache's owner check, for file kinds and /// sizes, and for the identities that detect a replaced file, /// access(2) for the execute permission, mkdir(2) to -/// create a staging directory exclusively, and realpath(3) to resolve -/// the cache root's path once. +/// create a staging directory exclusively, realpath(3) to resolve +/// the cache root's path once, and getpwuid_r(3) and +/// getgrgid_r(3) to recognize the user's private group. /// /// /// The macOS calls use the 64-bit-inode struct stat layout; they are @@ -33,6 +35,11 @@ internal static partial class Unix private const int Eexist = 17; private const int AtFdcwd = -100; private const int AtSymlinkNofollow = 0x100; + private const int AtEmptyPath = 0x1000; + private const int Erange = 34; + + /// The most bytes a user or group lookup may use; a larger entry is treated as not found. + internal const int AccountBytes = 64 * 1024; private const uint StatxBasicStats = 0x7ff; private const int FileTypeMask = 0xF000; private const int RegularType = 0x8000; @@ -51,11 +58,14 @@ internal enum Kind /// /// What the status call reports: the kind, size, owner, and permission - /// bits, and the file's identity: its device and inode, and its - /// modification and status-change times in nanoseconds. + /// bits, the file's identity: its device and inode, and its + /// modification and status-change times in nanoseconds; and its group. /// - internal readonly record struct Status(Kind Kind, long Size, uint Owner, int Mode, ulong Device = 0, ulong Inode = 0, long Modified = 0, long Changed = 0) + internal readonly record struct Status(Kind Kind, long Size, uint Owner, int Mode, ulong Device = 0, ulong Inode = 0, long Modified = 0, long Changed = 0, uint Group = 0) { + /// Whether is the same file: kind, device, and inode. + internal bool SameInode(Status other) => Kind == other.Kind && Device == other.Device && Inode == other.Inode; + /// /// Whether describes the same file in the /// same state: kind, device, inode, size, mode, owner, and both times. @@ -88,6 +98,21 @@ internal bool SameFile(Status other) => [LibraryImport("libc", EntryPoint = "statx", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] private static unsafe partial int NativeStatx(int dirfd, string path, int flags, uint mask, byte* buffer); + [LibraryImport("libc", EntryPoint = "statx", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static unsafe partial int NativeFstatx(int dirfd, string path, int flags, uint mask, byte* buffer); + + [LibraryImport("libc", EntryPoint = "fstat", SetLastError = true)] + private static unsafe partial int MacFstatArm(int descriptor, byte* buffer); + + [LibraryImport("libc", EntryPoint = "fstat$INODE64", SetLastError = true)] + private static unsafe partial int MacFstatIntel(int descriptor, byte* buffer); + + [LibraryImport("libc", EntryPoint = "getpwuid_r")] + private static unsafe partial int NativeGetpwuid(uint uid, byte* entry, byte* buffer, nuint length, byte** result); + + [LibraryImport("libc", EntryPoint = "getgrgid_r")] + private static unsafe partial int NativeGetgrgid(uint gid, byte* entry, byte* buffer, nuint length, byte** result); + [LibraryImport("libc", EntryPoint = "stat", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] private static unsafe partial int MacStatArm(string path, byte* buffer); @@ -192,13 +217,6 @@ internal static unsafe Status Stat(string path, bool follow) byte* buffer = stackalloc byte[256]; new Span(buffer, 256).Clear(); int result; - int modeOffset; - int ownerOffset; - int sizeOffset; - ulong device; - ulong inode; - long modified; - long changed; if (OperatingSystem.IsMacOS()) { bool arm = RuntimeInformation.ProcessArchitecture == Architecture.Arm64; @@ -209,8 +227,78 @@ internal static unsafe Status Stat(string path, bool follow) (true, false) => MacStatIntel(path, buffer), _ => MacLstatIntel(path, buffer), }; + } + else + { + result = NativeStatx(AtFdcwd, path, follow ? 0 : AtSymlinkNofollow, StatxBasicStats, buffer); + } + + if (result != 0) + { + int error = Marshal.GetLastPInvokeError(); + if (error == Enoent || error == Enotdir) + { + return new Status(Kind.Missing, 0, 0, 0); + } + + throw new IOException("cannot read its status (errno " + error.ToString(CultureInfo.InvariantCulture) + ")"); + } + + return Decode(buffer); + } + + /// The status of the open file (fstat(2)). + /// The status cannot be read. + internal static unsafe Status Stat(SafeHandle handle) + { + byte* buffer = stackalloc byte[256]; + new Span(buffer, 256).Clear(); + bool added = false; + int result; + try + { + handle.DangerousAddRef(ref added); + int descriptor = (int)handle.DangerousGetHandle(); + if (OperatingSystem.IsMacOS()) + { + result = RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? MacFstatArm(descriptor, buffer) : MacFstatIntel(descriptor, buffer); + } + else + { + result = NativeFstatx(descriptor, string.Empty, AtEmptyPath, StatxBasicStats, buffer); + } + + if (result != 0) + { + throw new IOException("cannot read its status (errno " + Marshal.GetLastPInvokeError().ToString(CultureInfo.InvariantCulture) + ")"); + } + } + finally + { + if (added) + { + handle.DangerousRelease(); + } + } + + return Decode(buffer); + } + + private static unsafe Status Decode(byte* buffer) + { + int modeOffset; + int ownerOffset; + int groupOffset; + int sizeOffset; + ulong device; + ulong inode; + long modified; + long changed; + if (OperatingSystem.IsMacOS()) + { modeOffset = 4; ownerOffset = 16; + groupOffset = 20; sizeOffset = 96; // st_dev (int32) at 0, st_ino at 8, st_mtimespec at 48, st_ctimespec at 64. @@ -221,9 +309,9 @@ internal static unsafe Status Stat(string path, bool follow) } else { - result = NativeStatx(AtFdcwd, path, follow ? 0 : AtSymlinkNofollow, StatxBasicStats, buffer); modeOffset = 28; ownerOffset = 20; + groupOffset = 24; sizeOffset = 40; // stx_ino at 32, stx_ctime at 96, stx_mtime at 112 (each a @@ -235,19 +323,9 @@ internal static unsafe Status Stat(string path, bool follow) device = ((ulong)*(uint*)(buffer + 136) << 32) | *(uint*)(buffer + 140); } - if (result != 0) - { - int error = Marshal.GetLastPInvokeError(); - if (error == Enoent || error == Enotdir) - { - return new Status(Kind.Missing, 0, 0, 0); - } - - throw new IOException("cannot read its status (errno " + error.ToString(CultureInfo.InvariantCulture) + ")"); - } - int mode = *(ushort*)(buffer + modeOffset); uint owner = *(uint*)(buffer + ownerOffset); + uint group = *(uint*)(buffer + groupOffset); long size = *(long*)(buffer + sizeOffset); Kind kind = (mode & FileTypeMask) switch { @@ -256,7 +334,113 @@ internal static unsafe Status Stat(string path, bool follow) LinkType => Kind.Link, _ => Kind.Other, }; - return new Status(kind, size, owner, mode & 0xFFF, device, inode, modified, changed); + return new Status(kind, size, owner, mode & 0xFFF, device, inode, modified, changed, group); + } + + /// A user's name and primary group, from the password database. + internal readonly record struct Account(string Name, uint Group); + + /// A group's name and its listed members (gr_mem), from the group database. + internal sealed record GroupEntry(string Name, IReadOnlyList Members); + + /// + /// The password database entry of + /// (getpwuid_r(3)); null when there is none, it cannot be read, + /// or it needs more than . + /// + internal static unsafe Account? User(uint uid) + { + if (IntPtr.Size != 8) + { + return null; + } + + // struct passwd on Linux and macOS (LP64): pw_name at 0, pw_uid at 16, pw_gid at 20. + byte* entry = stackalloc byte[256]; + for (int length = 1024; length <= AccountBytes; length *= 2) + { + byte[] buffer = new byte[length]; + byte* result = null; + int error; + fixed (byte* data = buffer) + { + error = NativeGetpwuid(uid, entry, data, (nuint)length, &result); + if (error == 0) + { + if (result == null) + { + return null; + } + + string? name = Marshal.PtrToStringUTF8(*(IntPtr*)entry); + return name is null ? null : new Account(name, *(uint*)(entry + 20)); + } + } + + if (error != Erange) + { + return null; + } + } + + return null; + } + + /// + /// The group database entry of + /// (getgrgid_r(3)); null when there is none, it cannot be read, + /// or it needs more than . + /// + internal static unsafe GroupEntry? GroupOf(uint gid) + { + if (IntPtr.Size != 8) + { + return null; + } + + // struct group on Linux and macOS (LP64): gr_name at 0, gr_gid at 16, gr_mem at 24. + byte* entry = stackalloc byte[256]; + for (int length = 1024; length <= AccountBytes; length *= 2) + { + byte[] buffer = new byte[length]; + byte* result = null; + int error; + fixed (byte* data = buffer) + { + error = NativeGetgrgid(gid, entry, data, (nuint)length, &result); + if (error == 0) + { + if (result == null) + { + return null; + } + + string? name = Marshal.PtrToStringUTF8(*(IntPtr*)entry); + IntPtr* members = *(IntPtr**)(entry + 24); + var list = new List(); + + // Each member's pointer and name lie in the bounded buffer, so the list is bounded by it. + for (int i = 0; members != null && members[i] != IntPtr.Zero; i++) + { + if (i >= length / IntPtr.Size) + { + return null; + } + + list.Add(Marshal.PtrToStringUTF8(members[i]) ?? string.Empty); + } + + return name is null ? null : new GroupEntry(name, list); + } + } + + if (error != Erange) + { + return null; + } + } + + return null; } private static long Nanoseconds(long seconds, long nanoseconds) => unchecked((seconds * 1_000_000_000L) + nanoseconds); diff --git a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs index 8e98b76..202ed22 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs @@ -45,6 +45,8 @@ public void CleanUp() Core.DigestChunkHook = null; Core.TreeFileHook = null; Core.TreeEntryCap = Core.TreeEntries; + Core.UserHook = null; + Core.GroupHook = null; SdkSupport.RemoveScratch(scratch); } @@ -827,6 +829,7 @@ public void ACacheParentOthersMayWriteIsRefusedUnlessItIsSticky() { using FakeCore fake = FakeCore.Programs(At("root")); string parent = Directory.CreateDirectory(At("parent"), UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute).FullName; + NotPrivate(parent); var options = new PrepareOptions { CacheDir = Path.Combine(parent, "cache") }; string Refused(UnixFileMode mode) { @@ -835,17 +838,16 @@ string Refused(UnixFileMode mode) } const UnixFileMode Owner = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute; - string expected = "cache directory parent " + parent + " is group- or world-writable without the sticky bit," - + " so another user could replace the cache (make it, for example, chmod go-w " + parent + ")"; - Assert.That(Refused(Owner | UnixFileMode.OtherWrite | UnixFileMode.OtherExecute), Is.EqualTo(expected), "others may write"); - Assert.That(Refused(Owner | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute), Is.EqualTo(expected), "the group may write"); + string cache = Path.Combine(parent, "cache"); + Assert.That(Refused(Owner | UnixFileMode.OtherWrite | UnixFileMode.OtherExecute), Is.EqualTo(Unsafe(cache, parent + " (writable by others)")), "others may write"); + Assert.That(Refused(Owner | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute), Is.EqualTo(Unsafe(cache, parent + " (" + NotPrivateGroup + ")")), "the group may write"); // A grandparent is checked as well. string nested = Directory.CreateDirectory(Path.Combine(parent, "nested"), Owner).FullName; File.SetUnixFileMode(parent, (UnixFileMode)0x1FF); Assert.That( Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = Path.Combine(nested, "cache") }))!.Message, - Is.EqualTo(expected)); + Is.EqualTo(Unsafe(Path.Combine(nested, "cache"), parent + " (writable by others)"))); // Sticky and world-writable, as /tmp is: accepted. File.SetUnixFileMode(parent, (UnixFileMode)0x3FF); @@ -860,6 +862,142 @@ string Refused(UnixFileMode mode) Assert.That(linked.Directory, Does.StartWith(Path.Combine(parent, "cache") + "/")); } + private const string NotPrivateGroup = "writable by its group, which is not your private group"; + + /// The refusal of a cache at listing . + private static string Unsafe(string cache, params string[] levels) => + "cache directory " + cache + " could be replaced by another user through the directories above it." + + " Make each of these writable only by you (for example chmod go-w DIR), or use a location under directories only you or root may write: " + + string.Join("; ", levels); + + /// Answers the private-group lookups as a user "me" whose primary group 's group is, named with . + private static void Accounts(string directory, string name, params string[] members) + { + uint group = Unix.Stat(directory, follow: false).Group; + Core.UserHook = uid => uid == Unix.EffectiveUser() ? new Unix.Account("me", group) : null; + Core.GroupHook = gid => gid == group ? new Unix.GroupEntry(name, members) : null; + } + + /// Answers the lookups so that 's group is not the user's private group. + private static void NotPrivate(string directory) => Accounts(directory, "staff"); + + /// + /// A group-writable directory above the cache is accepted when its group + /// is the user's private group (Debian OpenSSH's rule): the directory's + /// group is the user's primary group, the group's name is the user's + /// name, and it lists no members or only the user. The password and + /// group answers come from a seam; the directory is 0770. + /// + [Test] + public void ACacheParentWritableByThePrivateGroupIsAccepted() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent"), (UnixFileMode)0x1F8).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1F8); + var options = new PrepareOptions { CacheDir = Path.Combine(parent, "cache") }; + Accounts(parent, "me"); + Assert.That(fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options).Directory, Does.StartWith(Path.Combine(parent, "cache") + "/"), "no members"); + Accounts(parent, "me", "me"); + Assert.That(fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options).Directory, Does.StartWith(Path.Combine(parent, "cache") + "/"), "the user alone"); + + // The private group does not excuse others' write. + File.SetUnixFileMode(parent, (UnixFileMode)0x1FA); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options))!.Message, + Is.EqualTo(Unsafe(Path.Combine(parent, "cache"), parent + " (writable by others)"))); + } + + /// + /// Any other group-writable directory above the cache is refused: a + /// group that is not the user's primary group, a group named otherwise, + /// a group with another member (alone or beside the user), and a user or + /// group the databases do not know. + /// + [Test] + public void ACacheParentWritableByAnyOtherGroupIsRefused() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent"), (UnixFileMode)0x1F8).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1F8); + var options = new PrepareOptions { CacheDir = Path.Combine(parent, "cache") }; + uint group = Unix.Stat(parent, follow: false).Group; + string expected = Unsafe(Path.Combine(parent, "cache"), parent + " (" + NotPrivateGroup + ")"); + void Refused(string what) => + Assert.That(Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, options))!.Message, Is.EqualTo(expected), what); + + Core.UserHook = uid => new Unix.Account("me", group + 1); + Core.GroupHook = gid => new Unix.GroupEntry("me", Array.Empty()); + Refused("another primary group"); + Accounts(parent, "staff"); + Refused("a group named otherwise"); + Accounts(parent, "me", "other"); + Refused("another member"); + Accounts(parent, "me", "me", "other"); + Refused("another member beside the user"); + Core.UserHook = uid => null; + Refused("an unknown user"); + Accounts(parent, "me"); + Core.GroupHook = gid => null; + Refused("an unknown group"); + Assert.That(Directory.Exists(Path.Combine(parent, "cache")), Is.False, "a refused preparation made the root"); + } + + /// + /// The real password and group databases answer for this user: its name + /// is the one .NET reports, and its primary group has an entry. + /// + [Test] + public void TheAccountLookupsReadTheRealDatabases() + { + Unix.Account account = Unix.User(Unix.EffectiveUser()) ?? throw new AssertionException("no passwd entry for this user"); + Assert.That(account.Name, Is.EqualTo(Environment.UserName)); + Unix.GroupEntry group = Unix.GroupOf(account.Group) ?? throw new AssertionException("no group entry for the primary group"); + Assert.That(group.Name, Is.Not.Empty); + Assert.That(group.Members.Count, Is.LessThan(Unix.AccountBytes)); + } + + /// + /// A refusal names every unsafe directory above the cache, nearest + /// first, with its reason, not only the first found; beyond 16 it counts + /// the rest. + /// + [Test] + public void ARefusalListsEveryUnsafeDirectoryAboveTheCache() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string a = Directory.CreateDirectory(At("a")).FullName; + string b = Directory.CreateDirectory(Path.Combine(a, "b")).FullName; + string c = Directory.CreateDirectory(Path.Combine(b, "c")).FullName; + File.SetUnixFileMode(a, (UnixFileMode)0x1FF); + File.SetUnixFileMode(b, (UnixFileMode)0x1F8); + File.SetUnixFileMode(c, (UnixFileMode)0x1C0); + NotPrivate(b); + string cache = Path.Combine(c, "cache"); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = cache }))!.Message, + Is.EqualTo(Unsafe(cache, b + " (" + NotPrivateGroup + ")", a + " (writable by others)"))); + + // Eighteen unsafe levels: sixteen are named and two counted. + string deep = Directory.CreateDirectory(At("deep"), UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute).FullName; + var levels = new List(); + for (int i = 0; i < 18; i++) + { + deep = Directory.CreateDirectory(Path.Combine(deep, "d")).FullName; + levels.Add(deep); + } + + foreach (string level in levels) + { + File.SetUnixFileMode(level, (UnixFileMode)0x1FF); + } + + string deepCache = Path.Combine(deep, "cache"); + levels.Reverse(); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = deepCache }))!.Message, + Is.EqualTo(Unsafe(deepCache, levels.Take(16).Select(level => level + " (writable by others)").ToArray()) + "; and 2 more")); + } + /// /// The directories a preparation makes for a missing cache root are each /// owner-only (0700), whatever the umask, so a cache under new From 8f4881198b16030c679c9b836c7fc0c0129445c1 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 09:41:28 -0400 Subject: [PATCH 29/52] Check a cache root's existing parents before making its missing levels --- dotnet/src/JacquardHost/Sdk/Runtime.cs | 10 +++++++++- .../JacquardHost.Tests/Sdk/RuntimeTests.cs | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/dotnet/src/JacquardHost/Sdk/Runtime.cs b/dotnet/src/JacquardHost/Sdk/Runtime.cs index a454186..c522143 100644 --- a/dotnet/src/JacquardHost/Sdk/Runtime.cs +++ b/dotnet/src/JacquardHost/Sdk/Runtime.cs @@ -505,11 +505,19 @@ static T Read(Func read) // Each missing level is made owner-only, whatever the umask, // since no directory above the root may be writable by others. var missing = new Stack(); - for (string? level = configured; level is not null && !Directory.Exists(level); level = Path.GetDirectoryName(level)) + string? level = configured; + for (; level is not null && !Directory.Exists(level); level = Path.GetDirectoryName(level)) { missing.Push(level); } + // The directories that exist are checked before any level + // is made, so a refused preparation makes nothing. + if (missing.Count > 0 && level is not null && Unix.Supported) + { + Core.CheckDirectories(Unix.RealPath(level), configured); + } + while (missing.Count > 0) { Directory.CreateDirectory(missing.Pop(), UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); diff --git a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs index 202ed22..961ebf0 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs @@ -998,6 +998,24 @@ public void ARefusalListsEveryUnsafeDirectoryAboveTheCache() Is.EqualTo(Unsafe(deepCache, levels.Take(16).Select(level => level + " (writable by others)").ToArray()) + "; and 2 more")); } + /// + /// The directories that exist above a missing cache root are checked + /// before any missing level is made, so a refused preparation leaves + /// nothing behind. + /// + [Test] + public void ARefusedPreparationMakesNoDirectory() + { + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent")).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1FF); + string cache = Path.Combine(parent, "x", "y", "cache"); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = cache }))!.Message, + Is.EqualTo(Unsafe(cache, parent + " (writable by others)"))); + Assert.That(Directory.EnumerateFileSystemEntries(parent), Is.Empty); + } + /// /// The directories a preparation makes for a missing cache root are each /// owner-only (0700), whatever the umask, so a cache under new From baa841eba5784b2ee319cc305b1807b1bb0cd386 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 09:41:28 -0400 Subject: [PATCH 30/52] Refuse a save directory others may write, and check the temporary file is the one written before its rename --- dotnet/src/JacquardHost/Sdk/Invocation.cs | 129 ++++++++++++++---- .../JacquardHost.Tests/Sdk/BatchesTests.cs | 60 ++++++++ 2 files changed, 165 insertions(+), 24 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Invocation.cs b/dotnet/src/JacquardHost/Sdk/Invocation.cs index 3b2d779..ca35e8c 100644 --- a/dotnet/src/JacquardHost/Sdk/Invocation.cs +++ b/dotnet/src/JacquardHost/Sdk/Invocation.cs @@ -79,20 +79,33 @@ public string Render() return string.Join("\n", lines); } + /// + /// Test seam: runs with the temporary file's path after it is written + /// and flushed, before it is checked and renamed over the target. Null + /// outside tests. + /// + internal static volatile Action? BeforeRenameHook; + /// /// Writes the invocation's frames and host evidence. The record contains - /// argument and result values; it is written only when called. The - /// record is written to a new file, created exclusively in the target's - /// directory, owned by this process's user and owner-only (read and write - /// for its owner) where the file system has POSIX permissions; that file - /// is then renamed over . Whatever was at the path - /// is replaced, never written through: an existing file's permissions and - /// owner are not kept, and a symbolic link there is replaced, not - /// followed. The target's directory must be writable; where it is - /// sticky (such as /tmp), a file another user owns cannot be - /// replaced and the save fails. + /// argument and result values; it is written only when called. Where the + /// file system has POSIX permissions, the target's directory and every + /// directory above it must be owned by this user or root and not writable + /// by its group or others, unless it is sticky (such as /tmp) or + /// the group is this user's private group, as for the cache's + /// directories; otherwise the save is refused, since another user could + /// replace the file before or after it is renamed. The record is written + /// through one handle to a new file, created exclusively (never through + /// a link) in the target's directory, owned by this process's user and + /// owner-only (read and write for its owner), and flushed to disk; the + /// file at its path is checked to be the one written (the same device + /// and inode), and only then renamed over . + /// Whatever was at the path is replaced, never written through: an + /// existing file's permissions and owner are not kept, and a symbolic + /// link there is replaced, not followed. Where the directory is sticky, + /// a file another user owns cannot be replaced and the save fails. /// - /// The file cannot be written, or the path is longer than any path. + /// The file cannot be written, the target's directory is not safe from other users, the temporary file was replaced before its rename, or the path is longer than any path. /// The file may not be written. public void Save(string path) { @@ -126,16 +139,55 @@ public void Save(string path) byte[] bytes = Encoding.UTF8.GetBytes(text + "\n"); string full = Path.GetFullPath(path); string directory = Path.GetDirectoryName(full) ?? throw new IOException(Identities.Excerpt(path) + " names no file"); + string name = Path.GetFileName(full); + if (name.Length == 0) + { + throw new IOException(Identities.Excerpt(path) + " names no file"); + } + + if (Unix.Supported) + { + // The directory is resolved once and used by its real path, so + // the directories checked are the ones written. + directory = Unix.RealPath(directory); + full = Path.Combine(directory, name); + string? problem; + try + { + problem = Core.UnsafeDirectories(directory, "save target " + full); + } + catch (JacquardException error) + { + throw new IOException(error.Message, error); + } + + if (problem is not null) + { + throw new IOException(JacquardException.Bounded(problem)); + } + } + string temporary = Path.Combine(directory, string.Format( CultureInfo.InvariantCulture, ".jacquard-save-{0}-{1}", Environment.ProcessId, Convert.ToHexString(System.Security.Cryptography.RandomNumberGenerator.GetBytes(8)).ToLowerInvariant())); bool renamed = false; + Unix.Status written = default; try { - using (var stream = new FileStream(temporary, Core.OwnerOnly(FileMode.CreateNew))) + // FileMode.CreateNew is O_CREAT|O_EXCL, which never follows a + // link at the path. The one handle is written, flushed, and, + // where there are POSIX permissions, held open until the rename + // (elsewhere an open file cannot be renamed). + var stream = new FileStream(temporary, Core.OwnerOnly(FileMode.CreateNew)); + try { + if (Unix.Supported) + { + written = Unix.Stat(stream.SafeFileHandle); + } + if (!OperatingSystem.IsWindows()) { // Exactly owner-only, whatever the umask left. @@ -143,26 +195,55 @@ public void Save(string path) } stream.Write(bytes); - } + stream.Flush(flushToDisk: true); + BeforeRenameHook?.Invoke(temporary); + if (Unix.Supported && !Unix.Stat(temporary, follow: false).SameInode(written)) + { + // Not this call's file: it is neither renamed nor removed. + written = default; + throw new IOException("the temporary file for " + Identities.Excerpt(full) + " was replaced before it was renamed; nothing was saved"); + } - File.Move(temporary, full, overwrite: true); - renamed = true; + if (!Unix.Supported) + { + stream.Dispose(); + } + + File.Move(temporary, full, overwrite: true); + renamed = true; + } + finally + { + stream.Dispose(); + } } finally { if (!renamed) { - // Only the file this call created exclusively. - try - { - File.Delete(temporary); - } - catch (Exception error) when (error is IOException or UnauthorizedAccessException) - { - // Left behind under its distinctive name. - } + RemoveTemporary(temporary, written); + } + } + } + + /// + /// Removes the temporary file this call created exclusively, and only + /// that: where it was replaced, or its identity is unknown, whatever is + /// at the path is left behind under its distinctive name. + /// + private static void RemoveTemporary(string temporary, Unix.Status written) + { + try + { + if (!Unix.Supported || (written.Kind == Unix.Kind.Regular && Unix.Stat(temporary, follow: false).SameInode(written))) + { + File.Delete(temporary); } } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) + { + // Left behind under its distinctive name. + } } /// diff --git a/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs index 62ec785..3f18db1 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/BatchesTests.cs @@ -34,6 +34,7 @@ public sealed class BatchesTests public void CleanUp() { Function.ConformHook = null; + Invocation.BeforeRenameHook = null; SdkSupport.RemoveScratch(root); } @@ -321,4 +322,63 @@ public void SaveReplacesTheTargetWithANewOwnerOnlyFile() Assert.That(File.GetUnixFileMode(victim), Is.EqualTo(Open)); Assert.That(Directory.EnumerateFiles(root).Select(Path.GetFileName), Is.EquivalentTo(new[] { "existing.json", "victim.txt", "link.json" })); } + + /// + /// Save refuses a target directory another user could write (0777, not + /// sticky), as for the cache's directories, before it writes anything + /// there; a sticky one (0777 with the sticky bit, as /tmp is) is used. + /// + [Test] + [Platform("Linux")] + public void SaveRefusesATargetDirectoryOthersMayWrite() + { + Invocation invocation = Ints().Invoke(Argument.Of(Codecs.List(Codecs.Int), new long[] { 1 })); + string shared = Directory.CreateDirectory(Path.Combine(root, "shared")).FullName; + File.SetUnixFileMode(shared, (UnixFileMode)0x1FF); + string target = Path.Combine(shared, "record.json"); + IOException error = Support.Throws(() => invocation.Save(target))!; + Assert.That(error.Message, Is.EqualTo( + "save target " + target + " could be replaced by another user through the directories above it." + + " Make each of these writable only by you (for example chmod go-w DIR), or use a location under directories only you or root may write: " + + shared + " (writable by others)")); + Assert.That(Directory.EnumerateFileSystemEntries(shared), Is.Empty); + + File.SetUnixFileMode(shared, (UnixFileMode)0x3FF); + invocation.Save(target); + Assert.That(Directory.EnumerateFileSystemEntries(shared).Select(Path.GetFileName), Is.EqualTo(new[] { "record.json" })); + } + + /// + /// The temporary file is written through the one handle that created it, + /// and the file at its path is checked to be that file (the same device + /// and inode) just before the rename. A seam replaces the temporary path + /// with another file before the rename, as someone who may write the + /// directory could: the replacement is detected, it is neither renamed + /// over the target nor removed, and nothing is saved. + /// + [Test] + [Platform("Linux")] + public void SaveDetectsATemporaryFileReplacedBeforeItsRename() + { + Invocation invocation = Ints().Invoke(Argument.Of(Codecs.List(Codecs.Int), new long[] { 1 })); + string target = Path.Combine(root, "record.json"); + string? temporary = null; + Invocation.BeforeRenameHook = path => + { + temporary = path; + File.Delete(path); + File.WriteAllText(path, "planted"); + }; + IOException error = Support.Throws(() => invocation.Save(target))!; + Assert.That(error.Message, Is.EqualTo("the temporary file for " + target + " was replaced before it was renamed; nothing was saved")); + Assert.That(File.Exists(target), Is.False, "the replacement was installed"); + Assert.That(File.ReadAllText(temporary!), Is.EqualTo("planted"), "the replacement was removed"); + + // Without the replacement, the record is saved and nothing else is left. + File.Delete(temporary!); + Invocation.BeforeRenameHook = null; + invocation.Save(target); + Assert.That(File.ReadAllText(target), Does.StartWith("{").And.EndWith("}\n")); + Assert.That(Directory.EnumerateFiles(root).Select(Path.GetFileName), Is.EquivalentTo(new[] { "record.json" })); + } } From fc19966fa4e9f9ccc3b2eb6b96febd9073a74ea1 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 09:46:04 -0400 Subject: [PATCH 31/52] Check RUNNER_TEMP's ancestry before the .NET example projects run in CI --- .github/workflows/ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3d9024..d241fbe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -224,6 +224,19 @@ jobs: working-directory: dotnet run: | set -euo pipefail + # The SDK refuses a program cache under a directory another user + # could write; check $RUNNER_TEMP's ancestry here, so a runner set up + # otherwise fails clearly rather than inside the SDK. + d=$(realpath "$RUNNER_TEMP") + while :; do + mode=$(stat -c %a "$d"); owner=$(stat -c %u "$d") + if { [ "$owner" != "$(id -u)" ] && [ "$owner" != 0 ]; } || { [ $((0$mode & 022)) != 0 ] && [ $((0$mode & 01000)) = 0 ]; }; then + echo "::error::$d (mode $mode, owner $owner) may be written by another user; the .NET SDK refuses a cache under it" >&2 + exit 1 + fi + [ "$d" = / ] && break + d=$(dirname "$d") + done dotnet build examples/JacquardExamples run() { dotnet examples/JacquardExamples/bin/Debug/net8.0/JacquardExamples.dll --workdir "$RUNNER_TEMP/dotnet-examples" "$@"; } run quickstart From 5d493dc003f30baeee2fbb4a1d2f72664712412a Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 09:48:06 -0400 Subject: [PATCH 32/52] Refuse a private-group directory that has an access ACL --- dotnet/src/JacquardHost/Sdk/Core.cs | 8 ++++- dotnet/src/JacquardHost/Sdk/Unix.cs | 32 +++++++++++++++++-- .../JacquardHost.Tests/Sdk/RuntimeTests.cs | 31 ++++++++++++++++++ 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Core.cs b/dotnet/src/JacquardHost/Sdk/Core.cs index 4a3f70b..94fdff6 100644 --- a/dotnet/src/JacquardHost/Sdk/Core.cs +++ b/dotnet/src/JacquardHost/Sdk/Core.cs @@ -639,7 +639,8 @@ internal static void CheckDirectories(string directory, string root) /// next directory down is checked in turn. The private group is Debian /// OpenSSH's rule: the directory's group is the user's primary group, /// that group's name is the user's name, and it lists no members or only - /// the user. Every unsafe directory is named, nearest first, up to + /// the user, and the directory has no access ACL (whose mask, on Linux, + /// is what the group bits show). Every unsafe directory is named, nearest first, up to /// . /// /// The nearest directory to check. @@ -694,6 +695,11 @@ internal static void CheckDirectories(string directory, string root) { reason = "writable by its group, which is not your private group"; } + else if (Unix.HasAccessAcl(directory)) + { + // On Linux an ACL's mask shows in the group bits, and its entries may grant others write. + reason = "writable by its group, with an access control list"; + } } } diff --git a/dotnet/src/JacquardHost/Sdk/Unix.cs b/dotnet/src/JacquardHost/Sdk/Unix.cs index bd10f10..39fc85d 100644 --- a/dotnet/src/JacquardHost/Sdk/Unix.cs +++ b/dotnet/src/JacquardHost/Sdk/Unix.cs @@ -15,8 +15,9 @@ namespace JacquardLang.Host.Sdk; /// sizes, and for the identities that detect a replaced file, /// access(2) for the execute permission, mkdir(2) to /// create a staging directory exclusively, realpath(3) to resolve -/// the cache root's path once, and getpwuid_r(3) and -/// getgrgid_r(3) to recognize the user's private group. +/// the cache root's path once, and getpwuid_r(3), +/// getgrgid_r(3), and on Linux lgetxattr(2) (for an access +/// ACL) to recognize a directory writable only by the user's private group. /// /// /// The macOS calls use the 64-bit-inode struct stat layout; they are @@ -37,6 +38,8 @@ internal static partial class Unix private const int AtSymlinkNofollow = 0x100; private const int AtEmptyPath = 0x1000; private const int Erange = 34; + private const int Enodata = 61; + private const int Eopnotsupp = 95; /// The most bytes a user or group lookup may use; a larger entry is treated as not found. internal const int AccountBytes = 64 * 1024; @@ -107,6 +110,9 @@ internal bool SameFile(Status other) => [LibraryImport("libc", EntryPoint = "fstat$INODE64", SetLastError = true)] private static unsafe partial int MacFstatIntel(int descriptor, byte* buffer); + [LibraryImport("libc", EntryPoint = "lgetxattr", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] + private static partial nint NativeLgetxattr(string path, string name, IntPtr value, nuint size); + [LibraryImport("libc", EntryPoint = "getpwuid_r")] private static unsafe partial int NativeGetpwuid(uint uid, byte* entry, byte* buffer, nuint length, byte** result); @@ -337,6 +343,28 @@ private static unsafe Status Decode(byte* buffer) return new Status(kind, size, owner, mode & 0xFFF, device, inode, modified, changed, group); } + /// + /// Whether , not followed if it is a link, may + /// have a POSIX access ACL beyond its mode bits: on Linux, whether it + /// has the system.posix_acl_access attribute (true when that + /// cannot be read for another reason than its absence); false elsewhere. + /// + internal static bool HasAccessAcl(string path) + { + if (!OperatingSystem.IsLinux()) + { + return false; + } + + if (NativeLgetxattr(path, "system.posix_acl_access", IntPtr.Zero, 0) >= 0) + { + return true; + } + + int error = Marshal.GetLastPInvokeError(); + return error != Enodata && error != Eopnotsupp; + } + /// A user's name and primary group, from the password database. internal readonly record struct Account(string Name, uint Group); diff --git a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs index 961ebf0..65bb48f 100644 --- a/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs +++ b/dotnet/test/JacquardHost.Tests/Sdk/RuntimeTests.cs @@ -942,6 +942,37 @@ void Refused(string what) => Assert.That(Directory.Exists(Path.Combine(parent, "cache")), Is.False, "a refused preparation made the root"); } + /// + /// A directory writable by the private group is still refused when it + /// has an access ACL: on Linux the group bits show the ACL's mask, and + /// an entry may grant another user write (here nobody, set with + /// setfacl). + /// + [Test] + public void APrivateGroupParentWithAnAccessListIsRefused() + { + if (!File.Exists("/usr/bin/setfacl")) + { + Assert.Ignore("setfacl is not installed"); + } + + using FakeCore fake = FakeCore.Programs(At("root")); + string parent = Directory.CreateDirectory(At("parent")).FullName; + File.SetUnixFileMode(parent, (UnixFileMode)0x1C0); + using (var setfacl = System.Diagnostics.Process.Start("/usr/bin/setfacl", new[] { "-m", "u:nobody:rwx", parent })) + { + Assert.That(setfacl.WaitForExit(30_000), Is.True); + Assert.That(setfacl.ExitCode, Is.EqualTo(0), "setfacl"); + } + + Assert.That(File.GetUnixFileMode(parent), Is.EqualTo((UnixFileMode)0x1F8), "the mask shows in the group bits"); + Accounts(parent, "me"); + string cache = Path.Combine(parent, "cache"); + Assert.That( + Support.Throws(() => fake.Runtime.Prepare(new[] { fake.Source("rules.jac") }, new PrepareOptions { CacheDir = cache }))!.Message, + Is.EqualTo(Unsafe(cache, parent + " (writable by its group, with an access control list)"))); + } + /// /// The real password and group databases answer for this user: its name /// is the one .NET reports, and its primary group has an entry. From be6a7ee4590e5d2ebfa2349d4fb6899b164a33f3 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 10:00:56 -0400 Subject: [PATCH 33/52] Document the private-group rule, the save directory rule, and the quickstart remedy; drop review chronology from the .NET evidence --- CHANGELOG.md | 16 ++++++----- CONTRIBUTING.md | 10 ++++++- docs/EVIDENCE.md | 14 +++++----- docs/LIMITS.md | 3 ++- dotnet/README.md | 69 ++++++++++++++++++++++++++++++++++-------------- 5 files changed, 77 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3781db5..eea6321 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,14 +109,18 @@ within 750 ms. Only the cache's owner may write it: the root, each entry, and each file must be owned by the current user and not group- or world-writable, every directory above the root must be owned by the user - or root and not group- or world-writable unless it is sticky, and the - root's device and inode are checked again before each use, each worker a - prepared program starts included. Replacement by the same user is not - covered. + or root and not group- or world-writable unless it is sticky or the group + is the user's private group (Debian OpenSSH's rule, so a clone made with a + `umask` of 002 under user-private groups is accepted), a refusal names + every such directory and creates nothing, and the root's device and inode + are checked again before each use, each worker a prepared program starts + included. Replacement by the same user is not covered. Every binding is derived again from Core's cached output on load, and a preparation during which the Core executable or prelude changed is - refused. `Invocation.Save` writes a new owner-only file and renames it over - the target. + refused. `Invocation.Save` refuses a target directory another user could + write, by the same directory rule, writes a new owner-only file through one + exclusively created handle, flushes it, checks the file at its path is the + one written, and renames it over the target. The four example projects run through a `JacquardExamples` command in `dotnet/examples/`, and CI runs them and the SDK's live tests against the pinned Core. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index adc2861..e14c7db 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -151,7 +151,7 @@ The independent .NET adapter and SDK live in `dotnet/` as package directory). It targets .NET 8, has no runtime dependencies beyond the .NET base class library, and does not call Python, Node.js, Go, Rust, or Java; it calls the C library (`poll(2)`, and for the SDK `kill(2)`, `geteuid(2)`, `mkdir(2)`, -and `statx(2)` or `stat(2)`) and runs on Linux and macOS only. On Linux the +`statx(2)` or `stat(2)`, `getpwuid_r(3)`, `getgrgid_r(3)`, and `lgetxattr(2)`) and runs on Linux and macOS only. On Linux the SDK runs Core's commands through util-linux `setsid` when one is installed. NUnit, its adapter, and the test SDK are its only (test) dependencies. The .NET 8 or 10 SDK is required. @@ -166,6 +166,14 @@ dotnet build examples/JacquardExamples dotnet examples/JacquardExamples/bin/Debug/net8.0/JacquardExamples.dll quickstart ``` +The example command's program cache is under `dotnet/.scratch/`, and the SDK +refuses it when another user could write a directory above it: one writable +by others, or by its group unless that group is your private group (named +after you, with no other members; a clone made with a `umask` of 002 under +user-private groups passes). The refusal names every such directory; run +`chmod go-w` on each, or pass `--workdir` under directories only you or root +may write. + CI runs the offline tests on .NET 8 and 10, and the live tests on both and the four example projects in the shared pinned-Core job. The build treats warnings as errors, with nullable reference types and the `8.0-recommended` analyzers. diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md index 03b1ec2..958db66 100644 --- a/docs/EVIDENCE.md +++ b/docs/EVIDENCE.md @@ -149,19 +149,19 @@ Recorded against the same `jacquard-core-0.3.0` release asset (linux-x86_64), on Linux 7.0 with the .NET 8.0.427 and 10.0.403 SDKs (runtimes 8.0.33 and 10.0.14). The library still has no runtime dependencies beyond the base class library; the SDK calls the C library's `kill(2)`, `geteuid(2)`, `mkdir(2)`, -`access(2)`, `realpath(3)`, and `statx(2)` (`stat(2)` on macOS, compiled but not run), and on Linux runs +`access(2)`, `realpath(3)`, `getpwuid_r(3)`, `getgrgid_r(3)`, `lgetxattr(2)`, and `statx(2)` (`stat(2)` and `fstat(2)` on macOS, compiled but not run), and on Linux runs Core's commands through util-linux `setsid`, here 2.41.3. The test dependencies are unchanged. | Lane | Result | |---|---| -| .NET SDK offline (`dotnet test test/JacquardHost.Tests -f net8.0` with the .NET 8 SDK pinned as the CI 8.0 job pins it, and `-f net10.0`, without Core) | 202 tests pass on each runtime and 28 live tests are ignored: the adapter's 96 offline tests and 106 SDK ones. Core commands (22, Linux): exit status, output, and added environment, with stdin at end of file; output over its limits read and counted but not kept; a timeout and a cancellation each kill the command and its child; under `setsid` the command keeps its pid and its group and session ids equal it (with `setsid --fork` the test fails); `sh -c 'sleep 300 & echo $!'` succeeds and its child dies, and survives when the exit kill is disabled; a child that closed its output is killed too; a reaped leader's pid held by an unrelated group leader is not killed; a process outside the group holding the command's stdout is neither killed nor waited for; a stop kills a child and a grandchild; without groups a recorded descendant is killed at the exit (and survives with recording off) and a stop kills the tree; against a fake `/proc` (a seam, pids above Linux's largest), one record reads exactly its entry cap (20 of 50) and its byte cap (5 of 50 entries at 5 KiB), and 1,024 bytes of each 64 KiB `stat` file, and a 1,025-byte cap reads 1,024 bytes and then 1; a cancellation after the third read ends the record there, then Core is sent SIGSTOP and is seen stopped (state `T`) before the last record's first read, the last record ends at its 250 ms budget (exactly three reads on a test clock advancing 100 ms per read), and only then is Core sent SIGKILL; on the real clock, with 300 recorded fake descendants whose every read after the stop takes at least 2 ms, the whole cleanup after the cancellation took 501 ms on .NET 8 and on .NET 10 (bound 750 ms; the test allows 1,250) and stopped after about 240 of the 600 reads; a cancellation during the second of ten exit kills still checks the other eight; a cancellation during the exit cleanup is reported; a missing executable, and one `setsid` cannot exec, are start failures; an escaped flooder holds neither the caller nor more than the kept output; 40 commands leave the descriptor count unchanged, counted at once. Preparation against a fake Core (41, Linux): option and path validation (empty, NUL, and over-long paths refused with excerpts before the file system sees them, a relative path measured absolute); one version check across four threads and two runtimes; prelude discovery once per Core and prelude; output over 8 MiB; the 30-second bound (shortened by a seam) and cancellation killing the command's child; a waiting preparation observing its own cancellation; FIFO and directory sources refused without blocking; sources over 16 MiB refused from their sizes before reading; cancellation between sources, during source validation, during fingerprinting (one 64 KiB piece), between empty prelude files, during the failure cleanup, and as a warm load finishes; prelude walks over 256 MiB, with a FIFO, over the entry cap (lowered by a seam), and 1,500 directories deep; a changed `program.json` binding, a changed Core output, a rewritten digest, and a `program.json` replaced by a link refused; group-writable roots, entries, files, and directories refused and new entries owner-only; a foreign-owned root refused; an executable over 512 MiB refused from its size; source lists over 1,024 refused from their count with no element read; the cleanup stopping at the cap; a symbolic link at the cache root followed to its directory; directories the preparation did not make, and one made at the vacated staging path, left alone; the cache root renamed away and replaced (a seam) before staging, publishing, and a cold or warm load, and after a load read the entry, each refused; the executable replaced by a new file after `--version`, `check`, `hash`, and `run`, a prelude file rewritten after `check`, and a file added to the prelude after `run`, each refused with nothing published; the version check run again for the same executable bytes with a new modification time; an executable only its group and others may run refused by `access(2)` before it runs; a bindings dictionary read without the program's lock held; function, effect, binding, and operation names over 4,096 chars refused and names of exactly 4,096 looked up; a parent of the cache root writable by others, or by its group, and a grandparent writable by others, each refused naming it, a sticky world-writable parent and a 0755 one accepted, and a path through a link to the parent resolved to the real directory; the cache root renamed away and back after preparation still used, and renamed away and replaced by a new owner-only directory refused at the next call of a function bound before and one bound after, with no worker started; the missing levels a preparation makes for a cache root each 0700; a source list and a bindings dictionary whose count, element, or enumerator throws wrapped in a `Preparation` or `Binding` exception keeping the cause. Data (26): signatures, hash groups, and display names; round trips and type conformance; the full Int range, Real bits, and `int` narrowing; records, enumerations, unions, nested `Option`, nullable codecs, tuples, custom codecs, and `Dynamic`; `ulong` (2^53 + 1), `nuint`, `char`, `float`, `Half`, `decimal`, `Int128`, `UInt128`, and `BigInteger` conversions and `Dynamic.Of` overloads obsolete as errors and, reached by reflection, refused, while the smaller integer types and `uint` still convert to `Int`; caller names over 4,096 chars refused before lookup (public, made, dynamic, record, and operation names); a lazy codec whose factory threw twice made on the third use and never again across 16 threads; two 64-level shared tuple graphs (2^64 paths) compared equal and unequal at once, and two lists of 1,000 tuples with no shared sub-value compared with no pair set made (a seam counts them); recursive types through a lazy codec made once across eight threads, a chain of a million lazy codecs, and one standing for itself refused; the node, text, frame, depth, and count bounds (a list whose items throw is refused from its count, tuples and records before an item is read, a constructor of 20 million fields before it is copied); a raw value 100,000 levels deep refused by its depth; a `Dynamic` over the hard limit refused before it is copied, an endless enumeration after one item past the limit; deep `Dynamic` equality and text without recursion; 64-char excerpts and the 8,192-char message cap, caller exceptions kept as inner exceptions; 100,000 layers of nested unions flattened to one case and round-tripped; a recursive union through a lazy case from eight threads; signature depth 64 accepted, 65 and 5,000 parentheses refused, 100,000 refused by the token cap. Batches and operations (15): argument counts refused before encoding; a throwing codec an invalid argument; cancellation before, during, and after batch validation, each an exact encode count with no worker; every batch validated and the first encoded again; no field or return value that could keep an encoding; a shared graph from a custom codec measured before its type is checked; rows that shrink from 10 to 3 during validation refused after one batch's four encodings; `Save` over a 0666 file leaving a 0600 file, and over a symbolic link replacing the link and leaving its target unchanged; handlers' decoding, results, failures, context, and token, and no handler starting once the call stopped during decoding. Examples (2): usage errors exit 2; the store removes only the temporary file it created | -| .NET SDK live (`JACQUARD_REQUIRE_LIVE=1`, `JACQUARD_BIN` and `JACQUARD_PRELUDE` set) | 20 tests pass on .NET 8 and on .NET 10 (unchanged in this review round): quickstart calls, `ToDynamic`, `Render`, and an owner-only saved inspection; the README examples; the 64-row triage oracle in batches of 8, with a codec failing on the second batch keeping the first and its exception; local refusals before any worker (a field over the Int range named by path, an over-deep call, a wrong type, a wrong count); missing handlers, a misspelled operation, and a 100-name service refused at binding with no handler run; unknown completion once with raw evidence; no handler starting once a real invocation stopped during decoding; a handler's token cancelled when the caller cancels; the 25-case planner oracle; cache reuse without Core, a byte-exact new entry, and corruption refused; a 2,500-`Option` signature left unbound with `safe` working cold and warm; a redirected name in `program.json` refused; the full Int range, Real bits, and nested options; spawn failure and cancellation; output overflow at batch 0; unit results kept after a later batch fails (E1602); too many bindings refused from the count; a throwing result codec keeping the result and its exception, a null mapping message, and an over-long save path; prelude contents keying the cache, and two concurrent preparations publishing one entry | -| .NET projects live | 3 further tests pass on both runtimes: all four `JacquardExamples` commands, including duplicate fulfillment, `--inspect` (owner-only), explicit delivery, a failed delivery printing no count, and usage errors; the command in a fresh process exiting 1 with E1613; and the three fulfillment faults recovered explicitly across restart, each receipt delivered once. The CI step's four projects and recovery commands also pass locally | -| Totals | 230 tests on each runtime with Core required, none ignored; 202 pass and 28 are ignored without Core (the .NET 8 lane built with the 8.0.427 SDK through a temporary `global.json`, as CI pins it) | +| .NET SDK offline (`dotnet test test/JacquardHost.Tests -f net8.0` with the .NET 8 SDK pinned as the CI 8.0 job pins it, and `-f net10.0`, without Core) | 210 tests pass on each runtime and 28 live tests are ignored: the adapter's 96 offline tests and 114 SDK ones. Core commands (22, Linux): exit status, output, and added environment, with stdin at end of file; output over its limits read and counted but not kept; a timeout and a cancellation each kill the command and its child; under `setsid` the command keeps its pid and its group and session ids equal it (with `setsid --fork` the test fails); `sh -c 'sleep 300 & echo $!'` succeeds and its child dies, and survives when the exit kill is disabled; a child that closed its output is killed too; a reaped leader's pid held by an unrelated group leader is not killed; a process outside the group holding the command's stdout is neither killed nor waited for; a stop kills a child and a grandchild; without groups a recorded descendant is killed at the exit (and survives with recording off) and a stop kills the tree; against a fake `/proc` (a seam, pids above Linux's largest), one record reads exactly its entry cap (20 of 50) and its byte cap (5 of 50 entries at 5 KiB), and 1,024 bytes of each 64 KiB `stat` file, and a 1,025-byte cap reads 1,024 bytes and then 1; a cancellation after the third read ends the record there, then Core is sent SIGSTOP and is seen stopped (state `T`) before the last record's first read, the last record ends at its 250 ms budget (exactly three reads on a test clock advancing 100 ms per read), and only then is Core sent SIGKILL; with 300 recorded fake descendants and a test clock that stands still except for 2 ms per read after the stop, the last record reads at 0 to 248 ms (125 reads) and Core is killed at 250 ms, the kills read at 250 to 498 ms (125 reads) and end at 500 ms, and the exit wait ends at 750 ms, so 250 of the 600 reads happen; a cancellation during the second of ten exit kills still checks the other eight; a cancellation during the exit cleanup is reported; a missing executable, and one `setsid` cannot exec, are start failures; an escaped flooder holds neither the caller nor more than the kept output; 40 commands leave the descriptor count unchanged, counted at once. Preparation against a fake Core (47, Linux): option and path validation (empty, NUL, and over-long paths refused with excerpts before the file system sees them, a relative path measured absolute); one version check across four threads and two runtimes; prelude discovery once per Core and prelude; output over 8 MiB; the 30-second bound (shortened by a seam) and cancellation killing the command's child; a waiting preparation observing its own cancellation; FIFO and directory sources refused without blocking; sources over 16 MiB refused from their sizes before reading; cancellation between sources, during source validation, during fingerprinting (one 64 KiB piece), between empty prelude files, during the failure cleanup, and as a warm load finishes; prelude walks over 256 MiB, with a FIFO, over the entry cap (lowered by a seam), and 1,500 directories deep; a changed `program.json` binding, a changed Core output, a rewritten digest, and a `program.json` replaced by a link refused; group-writable roots, entries, files, and directories refused and new entries owner-only; a foreign-owned root refused; an executable over 512 MiB refused from its size; source lists over 1,024 refused from their count with no element read; the cleanup stopping at the cap; a symbolic link at the cache root followed to its directory; directories the preparation did not make, and one made at the vacated staging path, left alone; the cache root renamed away and replaced (a seam) before staging, publishing, and a cold or warm load, and after a load read the entry, each refused; the executable replaced by a new file after `--version`, `check`, `hash`, and `run`, a prelude file rewritten after `check`, and a file added to the prelude after `run`, each refused with nothing published; the version check run again for the same executable bytes with a new modification time; an executable only its group and others may run refused by `access(2)` before it runs; a bindings dictionary read without the program's lock held; function, effect, binding, and operation names over 4,096 chars refused and names of exactly 4,096 looked up; a parent of the cache root writable by others, or by a group that is not the user's private group, and a grandparent writable by others, each refused naming it, a sticky world-writable parent and a 0755 one accepted, and a path through a link to the parent resolved to the real directory; with the password and group answers from a seam, a 0770 parent whose group is the user's private group (no members, or only the user) accepted, and refused when others may write it, when the group is not the user's primary group, is named otherwise, lists another member (alone or beside the user), or is unknown, or the user is unknown; a private-group parent with an access ACL (`setfacl -m u:nobody:rwx`, mode 0770 from its mask) refused (the test is ignored where `/usr/bin/setfacl` is missing); the real `getpwuid_r` and `getgrgid_r` answering for this user; one refusal naming every unsafe level nearest first with its reason (a group-writable one and an other-writable one above it), and 16 of 18 with the other 2 counted; a cache under missing levels below an unsafe parent refused with nothing made; the cache root renamed away and back after preparation still used, and renamed away and replaced by a new owner-only directory refused at the next call of a function bound before and one bound after, with no worker started; the missing levels a preparation makes for a cache root each 0700; a source list and a bindings dictionary whose count, element, or enumerator throws wrapped in a `Preparation` or `Binding` exception keeping the cause. Data (26): signatures, hash groups, and display names; round trips and type conformance; the full Int range, Real bits, and `int` narrowing; records, enumerations, unions, nested `Option`, nullable codecs, tuples, custom codecs, and `Dynamic`; `ulong` (2^53 + 1), `nuint`, `char`, `float`, `Half`, `decimal`, `Int128`, `UInt128`, and `BigInteger` conversions and `Dynamic.Of` overloads obsolete as errors and, reached by reflection, refused, while the smaller integer types and `uint` still convert to `Int`; caller names over 4,096 chars refused before lookup (public, made, dynamic, record, and operation names); a lazy codec whose factory threw twice made on the third use and never again across 16 threads; two 64-level shared tuple graphs (2^64 paths) compared equal and unequal at once, and two lists of 1,000 tuples with no shared sub-value compared with no pair set made (a seam counts them); recursive types through a lazy codec made once across eight threads, a chain of a million lazy codecs, and one standing for itself refused; the node, text, frame, depth, and count bounds (a list whose items throw is refused from its count, tuples and records before an item is read, a constructor of 20 million fields before it is copied); a raw value 100,000 levels deep refused by its depth; a `Dynamic` over the hard limit refused before it is copied, an endless enumeration after one item past the limit; deep `Dynamic` equality and text without recursion; 64-char excerpts and the 8,192-char message cap, caller exceptions kept as inner exceptions; 100,000 layers of nested unions flattened to one case and round-tripped; a recursive union through a lazy case from eight threads; signature depth 64 accepted, 65 and 5,000 parentheses refused, 100,000 refused by the token cap. Batches and operations (17): argument counts refused before encoding; a throwing codec an invalid argument; cancellation before, during, and after batch validation, each an exact encode count with no worker; every batch validated and the first encoded again; no field or return value that could keep an encoding; a shared graph from a custom codec measured before its type is checked; rows that shrink from 10 to 3 during validation refused after one batch's four encodings; `Save` over a 0666 file leaving a 0600 file, and over a symbolic link replacing the link and leaving its target unchanged; `Save` into a 0777 directory refused with nothing written there, and into a sticky 0777 one saved; a temporary file replaced (a seam) after it was written and before its rename detected, the replacement neither renamed over the target nor removed, and nothing saved; handlers' decoding, results, failures, context, and token, and no handler starting once the call stopped during decoding. Examples (2): usage errors exit 2; the store removes only the temporary file it created | +| .NET SDK live (`JACQUARD_REQUIRE_LIVE=1`, `JACQUARD_BIN` and `JACQUARD_PRELUDE` set) | 20 tests pass on .NET 8 and on .NET 10: quickstart calls, `ToDynamic`, `Render`, and an owner-only saved inspection; the README examples; the 64-row triage oracle in batches of 8, with a codec failing on the second batch keeping the first and its exception; local refusals before any worker (a field over the Int range named by path, an over-deep call, a wrong type, a wrong count); missing handlers, a misspelled operation, and a 100-name service refused at binding with no handler run; unknown completion once with raw evidence; no handler starting once a real invocation stopped during decoding; a handler's token cancelled when the caller cancels; the 25-case planner oracle; cache reuse without Core, a byte-exact new entry, and corruption refused; a 2,500-`Option` signature left unbound with `safe` working cold and warm; a redirected name in `program.json` refused; the full Int range, Real bits, and nested options; spawn failure and cancellation; output overflow at batch 0; unit results kept after a later batch fails (E1602); too many bindings refused from the count; a throwing result codec keeping the result and its exception, a null mapping message, and an over-long save path; prelude contents keying the cache, and two concurrent preparations publishing one entry | +| .NET projects live | 3 further tests pass on both runtimes: all four `JacquardExamples` commands, including duplicate fulfillment, `--inspect` (owner-only), explicit delivery, a failed delivery printing no count, and usage errors; the command in a fresh process exiting 1 with E1613; and the three fulfillment faults recovered explicitly across restart, each receipt delivered once. The CI step's four projects and recovery commands also pass locally, as do the `dotnet/README.md` quickstart and its "Run the projects" and recovery commands with their default work directories, run from `dotnet/` in a checkout whose directories are 0775 (a `umask` of 002, and the user's private group), including an `--inspect` file saved into a 0775 directory | +| Totals | 238 tests on each runtime with Core required, none ignored; 210 pass and 28 are ignored without Core (the .NET 8 lane built with the 8.0.427 SDK through a temporary `global.json`, as CI pins it) | | Regressions checked by mutation (.NET 8) | Each guard was removed in turn and the named tests run; all 42 mutations failed them: the command's exit group kill, its stop kill, `setsid` without forking, the pid-reuse check, the recorded-descendant kill, the stop check after the exit cleanup, and closing the output streams; the source kind check (the FIFO test failed after its one-minute wait), source count, source validation stop, source ceiling, tree entry and byte caps, the executable's size check and per-piece stop, the derived-binding check, the owner, mode, and owner-only checks, the cleanup's stop and cap, leaving the vacated staging path, and the final stop check; the encoder's node budget, the decoder's measurement, the text charge and length check, the list, record, and constructor counts, measuring before the type check, counting arguments before encoding, the batch stop checks before and after validation, the handler's stop check (offline and live), the `Dynamic` limit, excerpts, the message cap, wrapping a field reader's and a record constructor's exceptions, the lazy codec's cycle check (the test then never returned and was stopped), and the store's exclusive temporary file. Separately, checking the cancellation in an exception filter, which .NET runs before the preparation's cleanup, failed the failure-cleanup test until the check moved into the handler | -| Review-round regressions (.NET 8) | Each fix was undone in turn and its tests run; all 16 undoings failed them: `Invocation.cs`, `Builtins.cs`, and `Function.cs` as they were before the fixes (the old save left the existing file at 0666; the old lazy codec never ran its factory again; the old batch count made a batch of size -1, reported as an `OverflowException` from the codec), `Dynamic.cs` without the pair set (the equality test timed out after its one-minute wait), and guards turned off: the cache root recheck, the executable and prelude recheck, the identity in the version key, the record's entry cap, byte cap, and per-file read cap, the stop check in a record and in the exit kills, the name limit, the `ulong` refusal, `access(2)` (back to any execute bit), and reading the bindings outside the lock | -| Second review-round regressions (.NET 8) | Each new test was run against the source before its fix and failed: the 1,025-byte cap test against 96c0fc7's `CoreCommand.cs` (the second read took 1,024 bytes, not 1); the SIGSTOP-order, cleanup-budget, and finished-exit-kills tests against 96c0fc7's `CoreCommand.cs` with only the new seams declared (the last record read all 40 entries with no signal; the cleanup took 1,259 ms and all 600 reads; 0 kills after the stop, not 8); the parent and next-call root tests against the round-two `Core.cs`, `Function.cs`, `PreparedProgram.cs`, and `Runtime.cs` (a 0777 parent was accepted, and the call through a replaced root ran a worker); the owner-only levels test against the `Runtime.cs` before that fix (a level made with the umask, 0775 here, was refused as a group-writable parent); the pair-set test against 96c0fc7's `Dynamic.cs` with a counter at its allocation (2 sets for unshared values); and the wrapping test against the round-two `PreparedProgram.cs` and `Runtime.cs` (the `InvalidOperationException` escaped). Not tested: a parent owned by another user, which an unprivileged test cannot make | +| Regressions checked against the code before each fix (.NET 8) | Each fix was undone in turn and its tests run; all 16 undoings failed them: `Invocation.cs`, `Builtins.cs`, and `Function.cs` as they were before the fixes (the old save left the existing file at 0666; the old lazy codec never ran its factory again; the old batch count made a batch of size -1, reported as an `OverflowException` from the codec), `Dynamic.cs` without the pair set (the equality test timed out after its one-minute wait), and guards turned off: the cache root recheck, the executable and prelude recheck, the identity in the version key, the record's entry cap, byte cap, and per-file read cap, the stop check in a record and in the exit kills, the name limit, the `ulong` refusal, `access(2)` (back to any execute bit), and reading the bindings outside the lock | +| Regressions checked against the code before each fix, continued (.NET 8) | Each test below fails on the code without its fix (the file as it was before the fix, with only the test seams declared, or the guard turned off): the 1,025-byte cap test (the second read takes 1,024 bytes, not 1); the SIGSTOP-order test (the last record reads all 40 entries with no signal); the cleanup-budget test with the last record's, the kills', or the exit wait's budget turned off in turn (300 record reads, not 125; 300 kill reads, not 125; the exit wait not ending 750 ms after the stop); the finished-exit-kills test (0 kills after the stop, not 8); the parent test (a 0777 parent accepted) and the next-call root test (a call through a replaced root runs a worker); the owner-only levels test (a level made with the umask, 0775 here, refused as a group-writable parent); the private-group test (a 0770 parent of the user's private group refused), the other-group test with the member check turned off (a group listing another member accepted), and the access-ACL test with the ACL check turned off (accepted); the every-level test with the walk stopped at the first unsafe level (one named, not two); the nothing-made test with the early check turned off (the missing level `x` made before the refusal); the save-directory test (a 0777 directory accepted) and the replaced-temporary test with the identity check turned off (the replacement renamed over the target); the pair-set test (2 sets for unshared values); and the wrapping test (the `InvalidOperationException` escapes). Not tested: a parent owned by another user, which an unprivileged test cannot make | | Build | `TreatWarningsAsErrors`, nullable reference types, and the `8.0-recommended` analyzers are clean with both SDKs | | Unchanged lanes | The adapter's 96 offline and 5 live tests (the README-example check now covers all four examples), the 21-recording kit replay, `scripts/core_pin.py check`, `tests/test_core_pin.py`, and `scripts/check_distribution.py` (the source archive now carries the SDK and the example command) pass | diff --git a/docs/LIMITS.md b/docs/LIMITS.md index 35aaa61..008fcea 100644 --- a/docs/LIMITS.md +++ b/docs/LIMITS.md @@ -30,10 +30,11 @@ also updates this document. | .NET pipe I/O goes through `poll(2)` | .NET's pipe streams have no read or write that a deadline or a cancellation can interrupt, so the .NET host never calls one that can block: the caller's thread asks the C library's `poll(2)` whether a pipe is ready, with a poll timeout of at most 5 ms, rechecking the deadline, the `CancellationToken`, and disposal between polls, and then reads only bytes already in the pipe or writes at most `PIPE_BUF` bytes (4096 on Linux, 512 on macOS), which a writable pipe accepts at once. No thread is left blocked on a worker pipe, even while an escaped descendant holds stdin without reading or floods stdout or stderr. The `pollfd` layout and event bits are the same on Linux and macOS; elsewhere, including Windows, the host does not start a worker and reports a spawn failure. .NET opens its own descriptors close-on-exec, but passes the worker any descriptor the host process itself inherited without that flag. | A .NET API for interruptible or non-blocking process pipes, and CI on each added platform. | | .NET handler cancellation is cooperative | Each .NET handler runs on its own background thread. At the deadline or on cancellation the host kills the worker's process tree, closes its input, and sends no late response. `HandlerContext.IsCancelled` turns true (it reads the deadline exactly), and stays true once the invocation ends. `HandlerContext.CancellationToken` is cancelled when the call stops or ends, never earlier (a lifetime of any length does not cancel it before its deadline). While a handler runs, the caller's thread rechecks the deadline and the caller's token between poll waits of at most 5 ms; that is the requested poll timeout, not a bound on the interval between checks, which reading pending stderr (up to 1 MiB per pass) and thread scheduling can lengthen. Whichever check sees the stop, the token's cancellation starts before the worker is killed; otherwise it starts shortly after the invocation ends. The cancellation runs on a short-lived thread the host starts for it, one per invocation that gave a handler a token, so it does not wait for the shared thread pool. If no thread can be created or started, the host queues it as a pool work item, which runs only when a pool thread is free, so a starved pool delays it. If the work item cannot be queued either, the host gives up: the token is never cancelled (`HandlerContext.IsCancelled` still turns true), `InvocationResult.HandlerTokenNotCancelled` is set, and a handler waiting only on the token keeps waiting; `Invoke` still returns its result and disposes the worker. Callbacks a handler registers on that token run on that thread, which the host never waits for; a callback that blocks holds that thread, and anything a callback throws is discarded. A callback registered after the token is cancelled runs at once on the handler's own thread and throws there, as with any `CancellationToken`. A handler that ignores both keeps running on its thread until it returns. No outside action is stopped or rolled back. A stack overflow in a handler ends the process, as it does any .NET code. | A separately enforced operation lifetime with explicit completion and reconciliation evidence. | | .NET SDK Core command descendants | Each Core `--version`, `check`, `hash`, and `run` command must exit within 30 seconds and keeps at most 8 MiB of stdout and 4 KiB of stderr; cancelling the token stops it. Its output is read only after `poll(2)` reports it ready, on the caller's thread. On Linux with a `setsid` at `/usr/bin/setsid` or `/bin/setsid` (util-linux's, or a compatible one such as BusyBox's; any executable there is used), the command runs as `setsid -- `, which execs without forking, so Core keeps its pid and leads its own process group; when `setsid` cannot exec Core (exit status 126 or 127 with its own `setsid: ` message) the command is reported as a start failure. When the command exits, and on a timeout or cancellation, the host sends `kill(-pgid, SIGKILL)` to that group. Once .NET has reaped Core, a live process holding Core's pid means the id was reused and nothing is killed; a process that took Core's pid after a full pid wraparound, made its own group, and forked between that check and the kill could still have that group killed, the same window as the Rust SDK's `kill(-pgid)`. A descendant that leaves the group (calls `setsid`) is neither detected nor killed; the host does not wait for the end of the output after the exit, and closes its ends, so that descendant's next write fails. With `setsid`, the wait for a killed Core to exit after a stop ends 750 ms after the stop. Without `setsid` (Linux without one at those paths), descendants are recorded from `/proc` about every 10 ms by pid and start time, and every recorded descendant still alive with that start time is killed when the command returns (the start time is read just before the kill, a short window in which a reused pid could be signalled, as with the JDK's process handles). Each record reads at most 16,384 `/proc` entries and 4 MiB of `stat` files in all, each read limited to 1 KiB and to the bytes left, at most 16,384 descendants are kept, and a record ends at the deadline or a cancellation, checked before each entry. The kills when Core exits by itself run to the end; a stop that lands during them lets them go on for at most 250 ms more. On a timeout or cancellation, Core is first sent `SIGSTOP` (only while its start time still matches the one read when it started), so it starts no more children and its children stay its children; then a last record is made, within the same caps, until 250 ms after the stop; then Core is killed, and every recorded descendant until 500 ms after the stop; then the host waits for Core to exit until 750 ms after the stop. Each budget is checked before each entry or descendant, so the cleanup returns within 750 ms of the stop plus one `stat` read and one signal. Only Core is frozen: its descendants run until they are killed, so one whose parent exits before the last record reaches it is reparented and missed. Descendants past the caps or the budgets, and fast fork-and-exit chains between records, are missed too. .NET's `Process.Kill(true)` is not used on Linux. macOS has no `/proc`, so there only `Process.Kill(true)` on a stop applies. The .NET worker does not run in its own group (see the worker row above). | An OS-level containment mechanism, or a .NET process-group API, with its own evidence. | -| .NET SDK cache and source reads | A cache entry is verified against the file digests in its `program.json`, and every binding is derived again from Core's cached raw output, so an accidental or partial change is refused. No record inside the cache resists someone who can rewrite it: an output, its recorded digest, and the bindings changed together load. So the cache's owner is trusted and only the owner may write it: preparation refuses a cache root, an entry, or any file or directory in one that the current user does not own or that its group or others may write (read with `statx(2)` on Linux and `stat(2)` on macOS, without following links), on create and on load, and new entries are owner-only. Every symbolic link in the cache root's path is resolved once (`realpath(3)`), and the real directory is checked and used from then on; nothing inside it is followed. Every directory above the root, up to `/`, must be owned by the current user or root and must not be writable by its group or others unless it has the sticky bit (as OpenSSH checks the directories above its configuration); otherwise preparation is refused with a message naming the directory. So no other user can rename the root away and put another directory at its path. The directories a preparation makes for a missing root are each made owner-only, whatever the `umask`, but an existing parent that is group-writable only because of a `umask` of 002 (user-private groups) is refused too: run `chmod go-w` on it, or put the cache under owner-only directories. The root is used by its path, also by every worker a prepared program starts (on `//store`): its device and inode, read at the owner check, are compared again before the staging directory is made, before publishing, before and after each load, and before each worker starts, and a changed root is refused (a call then fails with a `Preparation` exception and starts no worker). Not covered: replacement by the same user or by root, whom the owner check trusts; a root renamed away and back between two comparisons, which keeps its inode (once the parents are checked, only its owner or root can rename it); and a directory above the root made writable by others after preparation, which the next preparation refuses but a prepared program does not notice (its comparisons still refuse a replaced root, except in the few system calls between the last one and a worker's start). The Core executable and every prelude file and directory are fingerprinted by path, and Core opens them by path later: each one's identity (device, inode, size, mode, owner, and modification and status-change times) is read as it is fingerprinted and compared after Core's commands finish (inside the version check and prelude discovery, before a new entry is published, and before an existing one is loaded), and any change refuses the preparation. Not detected: a file swapped and then restored with the same inode and times, which an unprivileged user can do only within the file system's timestamp granularity, since writing or renaming a file updates its status-change time (on Linux from a coarse clock, a few milliseconds); files Core loads besides the executable and prelude (its shared libraries and loader); and any change after the last comparison, since a prepared program's workers later run the executable and read the prelude (`JACQUARD_PRELUDE`) by path without checking them again. The macOS status calls are compiled but not exercised by any test. A failed preparation's cleanup removes at most 100,000 entries of its staging directory, created exclusively, and ends at a cancellation, so a stopped preparation leaves an unpublished `prepare-*` directory in the cache root; it is never loaded, and nothing removes it later (a name and an age cannot show that the SDK made a directory or that no preparation is using it), so remove leftovers yourself when no preparation is running. At most 1,024 sources are taken, refused from the list's count before any element is read, and they must be regular files totalling at most 16 MiB, the Core executable at most 512 MiB, and the prelude and cache walks read at most 100,000 entries and 256 MiB of regular files under one root, each file checked from its size before it is read and again while it is read; a source or cache file replaced by a FIFO (or, in the cache, by a link) between that check and its open can still block or redirect the open, which a cancellation does not end. | An open that cannot block on a FIFO or follow a link; directory-relative (`openat`) access pinned to the checked root; Core reading descriptors the host opened and verified. | +| .NET SDK cache and source reads | A cache entry is verified against the file digests in its `program.json`, and every binding is derived again from Core's cached raw output, so an accidental or partial change is refused. No record inside the cache resists someone who can rewrite it: an output, its recorded digest, and the bindings changed together load. So the cache's owner is trusted and only the owner may write it: preparation refuses a cache root, an entry, or any file or directory in one that the current user does not own or that its group or others may write (read with `statx(2)` on Linux and `stat(2)` on macOS, without following links), on create and on load, and new entries are owner-only. Every symbolic link in the cache root's path is resolved once (`realpath(3)`), and the real directory is checked and used from then on; nothing inside it is followed. Every directory above the root, up to `/`, must be owned by the current user or root and must not be writable by others, nor by its group, unless it has the sticky bit (as OpenSSH checks the directories above its configuration) or the group is the user's private group by Debian OpenSSH's rule: the directory's group is the user's primary group, the group's name is the user's name, and the group lists no members (`gr_mem`) or only the user (`getpwuid_r(3)` and `getgrgid_r(3)`, each within 64 KiB; an entry that needs more counts as not private), and on Linux the directory has no access ACL (`lgetxattr(2)`). Otherwise preparation is refused with a message naming every such directory, nearest first (at most 16, the rest counted). So no other user can rename the root away and put another directory at its path. The existing directories are checked before a missing level is made, so a refused preparation creates nothing, and each level it makes is owner-only, whatever the `umask`. The private-group rule trusts the group database: unlike Debian OpenSSH, it does not scan the password database for another account whose primary group is the same group, so such an account could write a directory the rule accepts. The check reads mode bits only: an ACL that grants another user write without showing in the mode bits is not seen (macOS ACLs, `chmod +a`; on Linux a POSIX ACL's mask shows in the group bits, so such a directory is refused, and one with an access ACL never passes as the private group's), and on WSL drive mounts (`/mnt/c`, DrvFs without the `metadata` option) every directory shows as 0777 and `chmod go-w` has no effect, so the cache must live in the Linux file system there. The root is used by its path, also by every worker a prepared program starts (on `//store`): its device and inode, read at the owner check, are compared again before the staging directory is made, before publishing, before and after each load, and before each worker starts, and a changed root is refused (a call then fails with a `Preparation` exception and starts no worker). Not covered: replacement by the same user or by root, whom the owner check trusts; a root renamed away and back between two comparisons, which keeps its inode (once the parents are checked, only its owner or root can rename it); and a directory above the root made writable by others after preparation, which the next preparation refuses but a prepared program does not notice (its comparisons still refuse a replaced root, except in the few system calls between the last one and a worker's start). The Core executable and every prelude file and directory are fingerprinted by path, and Core opens them by path later: each one's identity (device, inode, size, mode, owner, and modification and status-change times) is read as it is fingerprinted and compared after Core's commands finish (inside the version check and prelude discovery, before a new entry is published, and before an existing one is loaded), and any change refuses the preparation. Not detected: a file swapped and then restored with the same inode and times, which an unprivileged user can do only within the file system's timestamp granularity, since writing or renaming a file updates its status-change time (on Linux from a coarse clock, a few milliseconds); files Core loads besides the executable and prelude (its shared libraries and loader); and any change after the last comparison, since a prepared program's workers later run the executable and read the prelude (`JACQUARD_PRELUDE`) by path without checking them again. The macOS status calls are compiled but not exercised by any test. A failed preparation's cleanup removes at most 100,000 entries of its staging directory, created exclusively, and ends at a cancellation, so a stopped preparation leaves an unpublished `prepare-*` directory in the cache root; it is never loaded, and nothing removes it later (a name and an age cannot show that the SDK made a directory or that no preparation is using it), so remove leftovers yourself when no preparation is running. At most 1,024 sources are taken, refused from the list's count before any element is read, and they must be regular files totalling at most 16 MiB, the Core executable at most 512 MiB, and the prelude and cache walks read at most 100,000 entries and 256 MiB of regular files under one root, each file checked from its size before it is read and again while it is read; a source or cache file replaced by a FIFO (or, in the cache, by a link) between that check and its open can still block or redirect the open, which a cancellation does not end. | An open that cannot block on a FIFO or follow a link; directory-relative (`openat`) access pinned to the checked root; Core reading descriptors the host opened and verified. | | .NET SDK cancellation is by token only | Preparation, Core's commands, `Call`, `Invoke`, and `MapBatches` observe the `CancellationToken` they take. `Thread.Interrupt` is not a stop signal: the SDK's own waits are `poll(2)` sleeps, which neither end at an interrupt nor consume it. The API is synchronous; there are no `Task`-returning variants. | `Task`-returning variants with the same bounds, if callers need them. | | .NET SDK signature depth | A checked function whose signature nests types deeper than 64 levels (`max_json_depth`, the protocol's value-depth limit), names more than 4,096 types, or has more than 16,384 tokens is left unbound, like a polymorphic one: preparation succeeds, the other functions bind, and binding it fails with a `Binding` exception naming the limit. The SDK parses no operation signatures (Core's `check --print-sigs` prints none). | None needed for depth, since a deeper value cannot cross the protocol; Core's structured host interface exporter for the rest. | | .NET SDK record fields are declared, not exported | A record codec's fields, in the order given, are a constructor's fields; the pinned CLI exports no field types, so a wrong order is not detected locally, and Core refuses it only where the swapped fields' types differ. | Core's structured host interface exporter, so bindings can be generated or checked. | +| .NET SDK saved inspections | `Invocation.Save` resolves the target's directory once and refuses it unless it and every directory above it pass the cache's directory rule (owned by the user or root; not writable by others, nor by its group unless sticky or the user's private group), since another user who could write it could replace the file before or after the rename. It writes the record through one handle to a new file it created exclusively (`O_CREAT|O_EXCL`, so no link is followed), owner-only and flushed to disk, checks that the file at the temporary path has that handle's device and inode, and only then renames it over the target; a replacement found there is refused and left in place, and nothing is saved. Not covered: a replacement by the same user or by root between that check and the rename (no other user can, given the directory rule); the directory itself is not synced, so the rename may not survive a power loss; and the mode-bit limits of the cache's rule (ACLs not shown in the mode bits, WSL drive mounts). | A rename relative to a directory descriptor pinned to the checked directory (`renameat(2)`, `RENAME_NOREPLACE` where the target is new), and an `fsync(2)` of that directory. | | .NET fulfillment is a single-process snapshot example | Atomic JSON replacement demonstrates booking/outbox recovery and deduplication, without cross-process locking or power-loss durability evidence. | A concrete production storage contract and crash/concurrent-writer evidence. | | .NET platforms | Linux runs in CI on .NET 8 and 10. macOS has no CI; the offline descendant tests need `sh`, `sleep`, and `yes`, and the descriptor, Core command, and preparation tests need `/proc` and run on Linux only. Windows is not supported: the worker reports a spawn failure and `Prepare` refuses. | CI, lifecycle, and cleanup evidence on each added platform. | | Node handler cancellation is cooperative | The Node adapter can expire while awaiting a Promise, signal the handler, and kill the worker; it cannot stop an outside action or preempt JavaScript that blocks the event loop. No late response or automatic retry follows expiration. | A separately enforced operation lifetime with explicit completion and reconciliation evidence. | diff --git a/dotnet/README.md b/dotnet/README.md index 5989d05..3fd62ff 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -19,7 +19,8 @@ The assembly has two layers: Linux is exercised. macOS should work but has no CI: the carrier and the SDK call the C library (`poll(2)`, and for the SDK `kill(2)`, `geteuid(2)`, -`mkdir(2)`, and `stat(2)`; see [process carrier](#process-carrier) and +`mkdir(2)`, `stat(2)`, `getpwuid_r(3)`, `getgrgid_r(3)`, and on Linux +`lgetxattr(2)`; see [process carrier](#process-carrier) and [Core's commands](#cores-commands)), and the macOS status calls are compiled but not run by any test. Windows is not supported: the worker reports a spawn failure and `Prepare` refuses. **Experimental:** the protocol is Core's pinned @@ -50,6 +51,15 @@ Order total 35: shipping 5 Order total 75: shipping 0 ``` +The command keeps its program cache under `.scratch/dotnet-examples/` in the +working directory, and the SDK refuses a cache when another user could write +any directory above it, up to `/`. A clone made with a `umask` of 002 passes +when its directories belong to your private group (a group named after you, +with no other members, as Debian, Ubuntu, and Fedora create for each user). +If preparation is refused, the message names every such directory: run +`chmod go-w` on each, or pass `--workdir` under directories only you or root +may write (see [call a checked function](#call-a-checked-function)). + The same command runs the other example projects; see [run the projects](#run-the-projects). @@ -148,14 +158,22 @@ that its group or others may write, both when it creates an entry and when it loads one, and it makes each new entry's directories and files owner-only (Core's store included). The owner and mode come from `statx(2)` on Linux and `stat(2)` on macOS, without following links. Every directory above the -root, up to `/`, must be owned by you or root and must not be writable by its -group or others unless it has the sticky bit (as `/tmp` has), as OpenSSH -checks the directories above its configuration; otherwise preparation is -refused, naming the directory. So no other user can rename the root away and -put another directory at its path. The directories a preparation makes for a -missing root are each made owner-only, whatever the `umask`; an existing one -left group-writable by a `umask` of 002 is refused: run `chmod go-w` on it, or -choose a cache under owner-only directories. The root is used by its path, also by every +root, up to `/`, must be owned by you or root and must not be writable by +others, nor by its group, unless it has the sticky bit (as `/tmp` has), as +OpenSSH checks the directories above its configuration. A directory writable +by its group is accepted when that group is your private group, by Debian +OpenSSH's rule: the directory's group is your primary group, the group's name +is your user name, the group lists no members or only you (read with +`getpwuid_r(3)` and `getgrgid_r(3)`), and, on Linux, the directory has no +access ACL, whose mask is what its group bits show. So a clone made with a `umask` of 002 +under user-private groups is accepted, and no other user can rename the root +away and put another directory at its path. Otherwise preparation is refused +with a message naming every such directory, nearest first, with its reason +(at most 16 named, the rest counted): run `chmod go-w` on each, or choose a +cache under directories only you or root may write. The existing directories +are checked before any missing level is made, so a refused preparation +creates nothing, and the levels it makes are each owner-only, whatever the +`umask`. The root is used by its path, also by every worker a prepared program starts, so its device and inode, read at the owner check, are compared again before the staging directory is made, before an entry is published, before and after each load, and before each worker @@ -210,13 +228,23 @@ decodes the result with the codec you pass; `Invoke` keeps the whole kill the worker when it is cancelled. `MapBatches` reads the number of rows once and refuses a list whose count differs before a later batch. `Invocation.Save` writes the frames and evidence, including argument and -result values, to a new file created exclusively in the target's directory, -made owner-only (0600) and owned by you, and then renames it over the path: -whatever was there is replaced, never written through, so an existing file's +result values. The target's directory, resolved once, and every directory +above it must pass the cache's directory rule (owned by you or root; not +writable by others or by its group unless sticky or your private group), or +the save is refused with an `IOException` naming them: otherwise another +user could replace the file before or after the rename. The record is written +through one handle to a new file created exclusively (`O_CREAT|O_EXCL`, never +through a link) in that directory, owner-only (0600) and owned by you, and +flushed to disk (`fsync`); the file at its path is then checked to be the one +written (same device and inode), and only then renamed over the target. A +replacement is refused and left in place; nothing is saved. Whatever was at +the target is replaced, never written through, so an existing file's permissions and owner are not kept and a symbolic link there is replaced -rather than followed. The directory must be writable; in a sticky directory -such as `/tmp`, a file another user owns cannot be replaced and the save -fails. The API is synchronous; there are no `Task`-returning variants. +rather than followed. In a sticky directory such as `/tmp`, a file another +user owns cannot be replaced and the save fails. Not covered: a replacement +by you or root between that check and the rename, and the directory's own +entry, which is not synced, so the rename itself may not survive a power +loss. The API is synchronous; there are no `Task`-returning variants. ## Data @@ -485,9 +513,11 @@ The command reads the rules and synthetic inputs from the repository's (or `JACQUARD_EXAMPLES=DIR`), so it runs from a clone. Custom input JSON may follow the project name. `--inspect PATH` saves frames; triage adds a numbered suffix per batch. State defaults to `.scratch/dotnet-examples/`; -its program cache is refused while any directory above it is writable by its -group or others (as a clone made with a `umask` of 002 is), so run -`chmod go-w` on those or pass a `--workdir` under owner-only directories. The +its program cache, and a `--inspect` file, are refused while another user +could write a directory above them (one writable by others, or by a group +other than your private group), so run `chmod go-w` on the directories the +message names, or pass `--workdir` and `--inspect` paths under directories +only you or root may write. The exit status is 1 for a Jacquard or SDK failure and 2 for usage or other errors. @@ -786,8 +816,7 @@ stop; then Core is killed, and every recorded descendant until 500 ms after the stop; then the host waits for Core's exit until 750 ms after the stop. Each budget is checked before each entry or descendant, so the cleanup returns within 750 ms of the stop plus one `stat` read and one signal (tests -check the order and the budgets on a test clock, and the whole bound on the -real clock). .NET's `Process.Kill(true)`, which reads every process on the +check the order and each budget's end on a test clock). .NET's `Process.Kill(true)`, which reads every process on the machine, is not used. Only Core is frozen, so a descendant whose parent exits before the last record reaches it is reparented and keeps running, as do one past the caps or the budgets and one started and reparented between two From 901a3fcd040ba2d3a7e899a81c90874a2c318f01 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 29 Oct 2026 10:29:31 -0400 Subject: [PATCH 34/52] Refuse a private group that another account has as its primary group, and read libc's account records only within their buffer The private-group rule now enumerates the password database (getpwent_r, twice, under a process-wide lock, at most 100,000 entries of 64 KiB) and refuses the group when another account has it as its primary group, which gr_mem does not show. A cap, a lookup error, a repeated name, a missing user, or passes that differ count as not private. The pointers getpwuid_r, getgrgid_r, and getpwent_r return are checked against the buffer given to them before they are read. --- dotnet/src/JacquardHost/Sdk/Core.cs | 94 +++++++- dotnet/src/JacquardHost/Sdk/Unix.cs | 223 +++++++++++++++--- .../JacquardHost.Tests.csproj | 2 + .../JacquardHost.Tests/Sdk/BatchesTests.cs | 30 +++ .../JacquardHost.Tests/Sdk/RuntimeTests.cs | 171 +++++++++++++- .../test/JacquardHost.Tests/Sdk/SdkSupport.cs | 26 ++ 6 files changed, 497 insertions(+), 49 deletions(-) diff --git a/dotnet/src/JacquardHost/Sdk/Core.cs b/dotnet/src/JacquardHost/Sdk/Core.cs index 94fdff6..535b293 100644 --- a/dotnet/src/JacquardHost/Sdk/Core.cs +++ b/dotnet/src/JacquardHost/Sdk/Core.cs @@ -625,6 +625,16 @@ internal static void CheckDirectories(string directory, string root) /// internal static volatile Func? GroupHook; + /// + /// Test seam: supplies the password database enumeration of the + /// private-group rule in place of , once per + /// pass (a null element is a lookup error). Null outside tests. + /// + internal static volatile Func>? PasswdHook; + + /// The most password database entries one pass of the private-group rule reads; more is refused. + internal const int PasswdEntries = 100_000; + /// The most unsafe directories one refusal names; the rest are counted. internal const int UnsafeListed = 16; @@ -637,11 +647,17 @@ internal static void CheckDirectories(string directory, string root) /// user's private group. In a sticky directory only the owner of an entry /// (or the directory's owner, or root) may rename or remove it, and the /// next directory down is checked in turn. The private group is Debian - /// OpenSSH's rule: the directory's group is the user's primary group, - /// that group's name is the user's name, and it lists no members or only - /// the user, and the directory has no access ACL (whose mask, on Linux, - /// is what the group bits show). Every unsafe directory is named, nearest first, up to - /// . + /// OpenSSH's rule (no account but the user has the group as its primary + /// group, and the group lists no member but the user) with stricter + /// conditions: the directory's group is the user's primary group, that + /// group's name is the user's name, it lists no members or only the + /// user, the password database as enumerated lists the user with that + /// primary group and no other account with it (see + /// ), and the directory has no + /// access ACL (whose mask, on Linux, is what the group bits show). Every + /// unsafe directory is named, nearest first, up to + /// . The enumeration runs at most once per + /// call, and its verdict is not kept after the call. /// /// The nearest directory to check. /// What the directories hold, which starts the message. @@ -658,6 +674,7 @@ internal static void CheckDirectories(string directory, string root) bool looked = false; Unix.Account? account = null; Unix.GroupEntry? group = null; + bool? sole = null; var unsafeOnes = new List(); int count = 0; for (string? directory = first; directory is not null; directory = Path.GetDirectoryName(directory)) @@ -691,7 +708,7 @@ internal static void CheckDirectories(string directory, string root) } } - if (!PrivateGroup(status, account, group)) + if (!PrivateGroup(status, account, group) || !(sole ??= OnlyUserHasPrimaryGroup(user, status.Group))) { reason = "writable by its group, which is not your private group"; } @@ -725,10 +742,73 @@ internal static void CheckDirectories(string directory, string root) + string.Join("; ", unsafeOnes) + more; } - /// Debian OpenSSH's private-group rule for a group-writable . + /// + /// The group-database half of the private-group rule for a + /// group-writable ; the password database half + /// is . + /// private static bool PrivateGroup(Unix.Status status, Unix.Account? account, Unix.GroupEntry? group) => account is { } user && group is not null && status.Group == user.Group && string.Equals(group.Name, user.Name, StringComparison.Ordinal) && (group.Members.Count == 0 || (group.Members.Count == 1 && string.Equals(group.Members[0], user.Name, StringComparison.Ordinal))); + + /// + /// Whether the password database, enumerated twice under + /// , lists with + /// primary group and no other account with that + /// primary group (gr_mem omits primary-group members, so this is + /// what shows them), as Debian OpenSSH's secure_permissions scans + /// with getpwent. False, so the directory is refused, when a pass + /// reads more than entries, meets a lookup + /// error, lists a name twice (as a pass restarted by another + /// setpwent or endpwent in the process would), or does not + /// list the user with the group, or when the two passes differ (as they + /// would if another enumeration in the process moved the position during + /// either). Accounts a source does not enumerate cannot be seen. + /// + internal static bool OnlyUserHasPrimaryGroup(uint uid, uint gid) + { + Func> source = PasswdHook ?? Unix.Passwd; + lock (Unix.PasswdLock) + { + return OnePass(source(), uid, gid) is { } first + && OnePass(source(), uid, gid) is { } second + && first.AsSpan().SequenceEqual(second); + } + } + + /// One pass of : a digest of every entry read when it holds, null otherwise. + private static byte[]? OnePass(IEnumerable entries, uint uid, uint gid) + { + using var digest = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var names = new HashSet(StringComparer.Ordinal); + int read = 0; + bool listed = false; + Span numbers = stackalloc byte[8]; + foreach (Unix.PasswdEntry? item in entries) + { + if (item is not { } entry || ++read > PasswdEntries || !names.Add(entry.Name)) + { + return null; + } + + if (entry.Group == gid) + { + if (entry.Uid != uid) + { + return null; + } + + listed = true; + } + + digest.AppendData(Encoding.UTF8.GetBytes(entry.Name + "\0")); + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(numbers, entry.Uid); + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(numbers[4..], entry.Group); + digest.AppendData(numbers); + } + + return listed ? digest.GetHashAndReset() : null; + } } diff --git a/dotnet/src/JacquardHost/Sdk/Unix.cs b/dotnet/src/JacquardHost/Sdk/Unix.cs index 39fc85d..f22c3c4 100644 --- a/dotnet/src/JacquardHost/Sdk/Unix.cs +++ b/dotnet/src/JacquardHost/Sdk/Unix.cs @@ -3,6 +3,7 @@ using System.Globalization; using System.IO; using System.Runtime.InteropServices; +using System.Text; namespace JacquardLang.Host.Sdk; @@ -16,8 +17,10 @@ namespace JacquardLang.Host.Sdk; /// access(2) for the execute permission, mkdir(2) to /// create a staging directory exclusively, realpath(3) to resolve /// the cache root's path once, and getpwuid_r(3), -/// getgrgid_r(3), and on Linux lgetxattr(2) (for an access -/// ACL) to recognize a directory writable only by the user's private group. +/// getgrgid_r(3), and on Linux setpwent(3), +/// getpwent_r(3), endpwent(3), and lgetxattr(2) (for an +/// access ACL) to recognize a directory writable only by the user's private +/// group. /// /// /// The macOS calls use the 64-bit-inode struct stat layout; they are @@ -119,6 +122,15 @@ internal bool SameFile(Status other) => [LibraryImport("libc", EntryPoint = "getgrgid_r")] private static unsafe partial int NativeGetgrgid(uint gid, byte* entry, byte* buffer, nuint length, byte** result); + [LibraryImport("libc", EntryPoint = "setpwent")] + private static partial void NativeSetpwent(); + + [LibraryImport("libc", EntryPoint = "getpwent_r")] + private static unsafe partial int NativeGetpwent(byte* entry, byte* buffer, nuint length, byte** result); + + [LibraryImport("libc", EntryPoint = "endpwent")] + private static partial void NativeEndpwent(); + [LibraryImport("libc", EntryPoint = "stat", SetLastError = true, StringMarshalling = StringMarshalling.Utf8)] private static unsafe partial int MacStatArm(string path, byte* buffer); @@ -371,10 +383,20 @@ internal static bool HasAccessAcl(string path) /// A group's name and its listed members (gr_mem), from the group database. internal sealed record GroupEntry(string Name, IReadOnlyList Members); + /// One password database entry: the account's name, user id, and primary group. + internal readonly record struct PasswdEntry(string Name, uint Uid, uint Group); + + /// + /// Serializes every enumeration of the password database in this + /// process: getpwent_r(3)'s position is process-global. + /// + internal static readonly object PasswdLock = new(); + /// /// The password database entry of /// (getpwuid_r(3)); null when there is none, it cannot be read, - /// or it needs more than . + /// it needs more than , or libc returns a + /// pointer outside the buffer it was given. /// internal static unsafe Account? User(uint uid) { @@ -383,7 +405,6 @@ internal sealed record GroupEntry(string Name, IReadOnlyList Members); return null; } - // struct passwd on Linux and macOS (LP64): pw_name at 0, pw_uid at 16, pw_gid at 20. byte* entry = stackalloc byte[256]; for (int length = 1024; length <= AccountBytes; length *= 2) { @@ -395,13 +416,7 @@ internal sealed record GroupEntry(string Name, IReadOnlyList Members); error = NativeGetpwuid(uid, entry, data, (nuint)length, &result); if (error == 0) { - if (result == null) - { - return null; - } - - string? name = Marshal.PtrToStringUTF8(*(IntPtr*)entry); - return name is null ? null : new Account(name, *(uint*)(entry + 20)); + return result == null ? null : ReadAccount(entry, data, length) is { } read ? new Account(read.Name, read.Group) : null; } } @@ -417,7 +432,8 @@ internal sealed record GroupEntry(string Name, IReadOnlyList Members); /// /// The group database entry of /// (getgrgid_r(3)); null when there is none, it cannot be read, - /// or it needs more than . + /// it needs more than , or libc returns a + /// pointer outside the buffer it was given. /// internal static unsafe GroupEntry? GroupOf(uint gid) { @@ -426,7 +442,6 @@ internal sealed record GroupEntry(string Name, IReadOnlyList Members); return null; } - // struct group on Linux and macOS (LP64): gr_name at 0, gr_gid at 16, gr_mem at 24. byte* entry = stackalloc byte[256]; for (int length = 1024; length <= AccountBytes; length *= 2) { @@ -438,37 +453,179 @@ internal sealed record GroupEntry(string Name, IReadOnlyList Members); error = NativeGetgrgid(gid, entry, data, (nuint)length, &result); if (error == 0) { - if (result == null) - { - return null; - } + return result == null ? null : ReadGroup(entry, data, length); + } + } - string? name = Marshal.PtrToStringUTF8(*(IntPtr*)entry); - IntPtr* members = *(IntPtr**)(entry + 24); - var list = new List(); + if (error != Erange) + { + return null; + } + } - // Each member's pointer and name lie in the bounded buffer, so the list is bounded by it. - for (int i = 0; members != null && members[i] != IntPtr.Zero; i++) - { - if (i >= length / IntPtr.Size) - { - return null; - } + return null; + } - list.Add(Marshal.PtrToStringUTF8(members[i]) ?? string.Empty); - } + /// + /// Reads a struct passwd at (LP64 on + /// Linux and macOS: pw_name at 0, pw_uid at 16, + /// pw_gid at 20) whose strings libc put in the + /// bytes at ; null when + /// the name's pointer or its terminating NUL is outside them. + /// + internal static unsafe PasswdEntry? ReadAccount(byte* entry, byte* data, int length) => + Text(*(IntPtr*)entry, data, length) is { } name ? new PasswdEntry(name, *(uint*)(entry + 16), *(uint*)(entry + 20)) : null; - return name is null ? null : new GroupEntry(name, list); - } + /// + /// Reads a struct group at (LP64 on + /// Linux and macOS: gr_name at 0, gr_gid at 16, + /// gr_mem at 24) whose strings and member array libc put in the + /// bytes at ; null when + /// the name, the member array, any slot of it up to its terminating null, + /// or any member's name lies even partly outside them. + /// + internal static unsafe GroupEntry? ReadGroup(byte* entry, byte* data, int length) + { + if (Text(*(IntPtr*)entry, data, length) is not { } name) + { + return null; + } + + var list = new List(); + byte* members = *(byte**)(entry + 24); + if (members == null) + { + return new GroupEntry(name, list); + } + + byte* end = data + length; + for (byte* slot = members; ; slot += IntPtr.Size) + { + // The slot itself is checked before it is read. + if (slot < data || slot > end - IntPtr.Size) + { + return null; } - if (error != Erange) + IntPtr member = *(IntPtr*)slot; + if (member == IntPtr.Zero) + { + return new GroupEntry(name, list); + } + + if (Text(member, data, length) is not { } text) { return null; } + + list.Add(text); } + } - return null; + /// + /// The NUL-terminated UTF-8 string at , which + /// must lie with its NUL inside the bytes at + /// ; null otherwise. + /// + private static unsafe string? Text(IntPtr pointer, byte* data, int length) + { + byte* start = (byte*)pointer; + if (start < data || start >= data + length) + { + return null; + } + + var bytes = new ReadOnlySpan(start, (int)(data + length - start)); + int nul = bytes.IndexOf((byte)0); + return nul < 0 ? null : Encoding.UTF8.GetString(bytes[..nul]); + } + + /// + /// Every entry of the password database in turn, as + /// setpwent(3), getpwent_r(3), and endpwent(3) + /// enumerate it (on Linux; elsewhere one lookup error). A null element is + /// a lookup error: an entry larger than , an + /// error other than the end, or a pointer outside the buffer; the + /// enumeration ends after it. The caller holds + /// while it enumerates; the position is + /// process-global, so an enumeration elsewhere in the process that does + /// not take the lock can move it. + /// + internal static IEnumerable Passwd() + { + if (!OperatingSystem.IsLinux() || IntPtr.Size != 8) + { + yield return null; + yield break; + } + + NativeSetpwent(); + try + { + byte[] buffer = new byte[1024]; + while (true) + { + int status = NextPasswd(ref buffer, out PasswdEntry? entry); + if (status == 0) + { + yield break; + } + + yield return status > 0 ? entry : null; + if (status < 0) + { + yield break; + } + } + } + finally + { + NativeEndpwent(); + } + } + + /// + /// Reads the next password database entry into : + /// 1 when there is one, 0 at the end, -1 on a lookup error. The buffer + /// doubles on ERANGE (which leaves the position on the same entry) up to + /// . + /// + private static unsafe int NextPasswd(ref byte[] buffer, out PasswdEntry? entry) + { + entry = null; + byte* record = stackalloc byte[256]; + while (true) + { + byte* result = null; + int error; + int length = buffer.Length; + fixed (byte* data = buffer) + { + error = NativeGetpwent(record, data, (nuint)length, &result); + if (error == 0) + { + if (result == null) + { + return 0; + } + + entry = ReadAccount(record, data, length); + return entry is null ? -1 : 1; + } + } + + if (error == Enoent) + { + return 0; + } + + if (error != Erange || length >= AccountBytes) + { + return -1; + } + + buffer = new byte[length * 2]; + } } private static long Nanoseconds(long seconds, long nanoseconds) => unchecked((seconds * 1_000_000_000L) + nanoseconds); diff --git a/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj b/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj index 3cb197a..c645ca4 100644 --- a/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj +++ b/dotnet/test/JacquardHost.Tests/JacquardHost.Tests.csproj @@ -8,6 +8,8 @@ JacquardLang.Host.Tests false true + + true 8.0