unii: Phase 0/1 memory engine + oracle cross-check (#74) + libcurl adapter wired behind host/network (#75)

unii/.gitignore

@@ -0,0 +1 @@
+build/

unii/bin/unii

@@ -0,0 +1,11 @@
+#!/usr/bin/env luajit
+-- unii launcher: resolves the repository root from this script's path so it
+-- runs from any working directory.
+local script = arg[0]
+local bin = script:match("^(.*)/[^/]+$") or "."
+local root = bin .. "/../.."
+local f = io.popen(("cd %q && pwd"):format(root))
+root = f:read("*l")
+f:close()
+package.path = root .. "/?.lua;" .. root .. "/?/init.lua;" .. package.path
+require("unii.host.cli").main(arg)

unii/build.lua

@@ -0,0 +1,70 @@
+-- unii/build.lua -- typecheck the rule bundle and record the build stamp.
+--
+--   luajit unii/build.lua            (or: unii/bin/unii build)
+--
+-- 1. Checks the toolchain (LuaJIT 2.1) and, when git is available, that the
+--    shen-lua runtime outside unii/ is byte-identical to the pinned commit.
+-- 2. Loads every core file under (tc +) with the fasl cache disabled, so the
+--    kernel typechecker really runs, and confirms that a deliberately
+--    ill-typed rules module is rejected (the check is live, not vacuous).
+-- 3. Writes unii/build/bundle.stamp with the bundle hash. Boot refuses any
+--    bundle whose hash differs from the stamp.
+local ffi = require("ffi")
+ffi.cdef [[ int setenv(const char *name, const char *value, int overwrite); ]]
+ffi.C.setenv("SHEN_FASL", "off", 1)
+
+local here = debug.getinfo(1, "S").source:sub(2):match("^(.*)/build%.lua$") or "unii"
+local root = here:match("^(.*)/unii$") or "."
+package.path = root .. "/?.lua;" .. root .. "/?/init.lua;" .. package.path
+
+local manifest = require("unii.manifest")
+local core = require("unii.host.core")
+
+local function step(msg) io.stdout:write("[build] ", msg, "\n") end
+local function die(msg) io.stderr:write("[build] FAILED: ", msg, "\n"); os.exit(1) end
+
+-- 1. toolchain and pinned runtime
+if not jit or not jit.version:match("^LuaJIT 2%.1") then
+  die("LuaJIT 2.1 required, running " .. (jit and jit.version or _VERSION))
+end
+step("toolchain " .. jit.version .. " " .. jit.os .. "/" .. jit.arch)
+
+local pin = manifest.shen_lua.commit
+local git_ok = os.execute(("git -C %q rev-parse --verify -q %s^{commit} >/dev/null 2>&1"):format(root, pin))
+if git_ok == 0 or git_ok == true then
+  local same = os.execute(("git -C %q diff --quiet %s -- . ':(exclude)unii' 2>/dev/null"):format(root, pin))
+  if not (same == 0 or same == true) then
+    die("shen-lua runtime outside unii/ differs from pinned commit " .. pin)
+  end
+  local untracked = io.popen(("git -C %q ls-files --others --exclude-standard -- . ':(exclude)unii'"):format(root)):read("*a")
+  if untracked ~= "" then die("untracked files outside unii/:\n" .. untracked) end
+  step("shen-lua runtime matches pinned commit " .. pin:sub(1, 12))
+else
+  step("WARNING: git or pinned commit unavailable; runtime pin not verified")
+end
+
+-- 2. typecheck
+local t0 = os.clock()
+local ok, err = pcall(core.load_bundle, here)
+if not ok then die(err) end
+step(("typechecked %d core files under (tc +) in %.2fs CPU"):format(#manifest.core_files, os.clock() - t0))
+
+local shen = core.shen()
+shen.eval("(tc +)")
+local bad = here .. "/test/fixtures/ill_typed_rules.shen"
+local rejected, why = pcall(shen.prims.F["load"], bad)
+shen.eval("(tc -)")
+if rejected then die("typechecker accepted " .. bad .. "; type checking is not live") end
+step("ill-typed fixture rejected: " .. core.error_message(why):gsub("\n.*", ""))
+
+-- 3. stamp
+local hash = core.bundle_hash(here)
+os.execute(("mkdir -p %q"):format(here .. "/build"))
+local f = assert(io.open(core.stamp_path(here), "wb"))
+f:write("bundle ", hash, "\n",
+        "bundle_format ", manifest.bundle_format, "\n",
+        "files ", table.concat(manifest.core_files, " "), "\n",
+        "shen_lua ", pin, "\n",
+        "luajit ", jit.version, "\n")
+f:close()
+step("bundle " .. hash .. " -> " .. core.stamp_path(here))

unii/core/arith.shen

@@ -0,0 +1,92 @@
+\* unii/core/arith.shen -- exact bounded integer helpers.
+
+   The core never applies `/`. Every quantity handled here is an integer in
+   [0, 2^40) and only +, -, * and comparisons are used, so each result is
+   exact both in LuaJIT doubles and in any Shen port with exact integers.
+   Version 1 domain ceiling: message ids, counts and offsets are at most
+   2^31 - 1 (2147483647). *\
+
+(define unii.max-id
+  {--> number}
+  -> 2147483647)
+
+(define unii.nat?
+  {number --> boolean}
+  X -> (and (integer? X) (and (>= X 0) (<= X 2147483647))))
+
+(define unii.pos-nat?
+  {number --> boolean}
+  X -> (and (unii.nat? X) (> X 0)))
+
+\\ 2^39 .. 2^0, the bit weights used by unii.divmod-pow2.
+(define unii.pow2-desc
+  {--> (list number)}
+  -> [549755813888 274877906944 137438953472 68719476736 34359738368
+      17179869184 8589934592 4294967296 2147483648 1073741824 536870912
+      268435456 134217728 67108864 33554432 16777216 8388608 4194304 2097152
+      1048576 524288 262144 131072 65536 32768 16384 8192 4096 2048 1024 512
+      256 128 64 32 16 8 4 2 1])
+
+(define unii.pow2
+  {number --> number}
+  0 -> 1
+  L -> (* 2 (unii.pow2 (- L 1))) where (and (integer? L) (and (> L 0) (< L 40)))
+  L -> (error "unii.pow2: exponent ~A outside [0, 39]" L))
+
+\\ Floor quotient and remainder of N by 2^L, by binary long division from bit 39
+\\ down. Requires 0 <= N < 2^40 and 0 <= L <= 39.
+(define unii.divmod-pow2
+  {number --> number --> (number * number)}
+  N L -> (unii.dm-loop N L 39 (unii.pow2-desc) 0)
+           where (and (integer? N) (and (>= N 0) (and (< N 1099511627776)
+                 (and (integer? L) (and (>= L 0) (< L 40))))))
+  N L -> (error "unii.divmod-pow2: ~A / 2^~A outside the exact domain" N L))
+
+(define unii.dm-loop
+  {number --> number --> number --> (list number) --> number --> (number * number)}
+  M L B _ Q -> (@p Q M) where (< B L)
+  M L B [P | Ps] Q -> (unii.dm-loop (- M P) L (- B 1) Ps (+ (+ Q Q) 1)) where (>= M P)
+  M L B [_ | Ps] Q -> (unii.dm-loop M L (- B 1) Ps (+ Q Q))
+  M _ _ [] Q -> (@p Q M))
+
+(define unii.half
+  {number --> number}
+  N -> (fst (unii.divmod-pow2 N 1)))
+
+(define unii.even?
+  {number --> boolean}
+  N -> (= 0 (snd (unii.divmod-pow2 N 1))))
+
+\\ Smallest L with 2^L = N, or -1 when N is not a power of two in [1, 2^31].
+(define unii.log2-exact
+  {number --> number}
+  N -> (unii.log2-loop N 0 1))
+
+(define unii.log2-loop
+  {number --> number --> number --> number}
+  N L P -> L where (= N P)
+  N L P -> -1 where (or (> P N) (> L 31))
+  N L P -> (unii.log2-loop N (+ L 1) (+ P P)))
+
+\\ Number of decimal digits of a natural number below 10^13.
+(define unii.decimal-width
+  {number --> number}
+  N -> (unii.width-loop N 1 10))
+
+(define unii.width-loop
+  {number --> number --> number --> number}
+  N W P -> W where (or (< N P) (>= W 13))
+  N W P -> (unii.width-loop N (+ W 1) (* P 10)))
+
+\\ A 32-bit djb2-style digest (h * 33 + byte mod 2^32) over the bytes of S as
+\\ this port exposes them through pos/string->n. Used only to tag job ids with
+\\ their source; the host records full SHA-256 provenance separately.
+(define unii.digest32
+  {string --> number}
+  S -> (unii.digest-loop S 5381))
+
+(define unii.digest-loop
+  {string --> number --> number}
+  "" H -> H
+  (@s C Rest) H -> (unii.digest-loop Rest
+                     (snd (unii.divmod-pow2 (+ (* H 33) (string->n C)) 32))))

unii/core/invariants.shen

@@ -0,0 +1,130 @@
+\* unii/core/invariants.shen -- executable structural checks.
+
+   (unii.invariant-errors State) returns [] for a well-formed state, else a
+   list of violation descriptions. Checks are direct recursive functions over
+   the bounded in-memory projections (view, live nodes, jobs); Prolog would
+   not make these single-pass walks clearer. *\
+
+(define unii.invariant-errors
+  {unii.state --> (list string)}
+  S -> (let Cf (unii.st-config S)
+         (append (unii.partition-errors (unii.st-view S) 0 (unii.st-covered S))
+         (append (unii.node-errors Cf (unii.st-view S))
+         (append (unii.node-errors Cf (unii.st-live S))
+         (append (unii.live-errors S (unii.st-live S))
+         (append (unii.frontier-errors S (unii.st-covered S))
+         (append (unii.job-errors S (unii.st-jobs S) [])
+         (append (unii.provisional-errors S (unii.built-nodes S))
+           (unii.collect-errors
+             [(@p (<= (unii.st-covered S) (unii.st-count S)) "view covers past the message count")
+              (@p (unii.nat? (unii.st-count S)) "message count outside [0, 2^31 - 1]")
+              (@p (= (unii.st-view-bytes S) (unii.view-bytes-of (unii.st-view S)))
+                  "cached view bytes differ from the rendered line sum")
+              (@p (or (unii.st-batch S) (<= (unii.st-view-bytes S) (unii.cf-high Cf)))
+                  "view above the high threshold outside batch mode")
+              (@p (<= (unii.count-jobs (/. J (unii.dispatched? J)) (unii.st-jobs S))
+                      (unii.cf-max-inflight Cf))
+                  "more dispatched jobs than the inflight cap")
+              (@p (<= (unii.count-jobs (/. J (unii.leaf-job? J)) (unii.st-jobs S))
+                      (unii.cf-max-frontier Cf))
+                  "unresolved leaves exceed the frontier bound")]))))))))))
+
+\\ The view is an exact, aligned, gap-free partition of [0, Covered).
+(define unii.partition-errors
+  {(list unii.node) --> number --> number --> (list string)}
+  [] At Covered -> [] where (= At Covered)
+  [] _ _ -> ["view does not end at the covered boundary"]
+  [N | _] _ _ -> ["view contains an invalid node key"]
+    where (not (unii.valid-key? (unii.node-key N)))
+  [N | _] At _ -> ["view has a gap, overlap or misaligned node"]
+    where (not (= At (unii.key-first (unii.node-key N))))
+  [N | Ns] _ Covered -> (unii.partition-errors Ns (unii.key-end (unii.node-key N)) Covered))
+
+(define unii.node-errors
+  {unii.config --> (list unii.node) --> (list string)}
+  _ [] -> []
+  Cf [N | Ns] -> ["node text exceeds the leaf cap"]
+    where (and (> (unii.node-bytes N) (unii.cf-cap Cf)) (not (unii.provisional? N)))
+  Cf [N | Ns] -> ["node line is not the canonical rendering of its text"]
+    where (not (= [line (unii.node-line N) (unii.node-line-bytes N)]
+                  (unii.make-line (unii.node-key N) (unii.node-text N) (unii.node-bytes N))))
+  Cf [N | Ns] -> ["node origin inconsistent with its level"]
+    where (not (unii.origin-fits? (unii.node-origin N) (unii.node-key N)))
+  Cf [_ | Ns] -> (unii.node-errors Cf Ns))
+
+(define unii.origin-fits?
+  {unii.origin --> unii.key --> boolean}
+  [exact-leaf] [key L _] -> (= L 0)
+  [joined] [key L _] -> (> L 0)
+  [summarized _ _] _ -> true
+  [provisional _] [key L _] -> (= L 0))
+
+\\ Live nodes are built, unique, distinct from the view, and inside the chat.
+(define unii.live-errors
+  {unii.state --> (list unii.node) --> (list string)}
+  _ [] -> []
+  S [N | Ns] -> ["a live node duplicates a view node"]
+    where (unii.has-node? (unii.node-key N) (unii.st-view S))
+  S [N | Ns] -> ["a live node is duplicated"]
+    where (unii.has-node? (unii.node-key N) Ns)
+  S [N | Ns] -> ["a live node lies past the message count"]
+    where (> (unii.key-end (unii.node-key N)) (unii.st-count S))
+  S [_ | Ns] -> (unii.live-errors S Ns))
+
+\\ Every message in [Covered, Count) is unresolved (one leaf job) or a
+\\ committed leaf waiting in Live; the message at Covered must be unresolved.
+(define unii.frontier-errors
+  {unii.state --> number --> (list string)}
+  S M -> [] where (>= M (unii.st-count S))
+  S M -> ["the first uncovered message is not awaiting a summary"]
+    where (and (= M (unii.st-covered S))
+               (not (unii.has-job-for-key? (unii.leaf-key M) (unii.st-jobs S))))
+  S M -> ["an uncovered message is neither pending nor built"]
+    where (= (unii.leaf-states S M) 0)
+  S M -> ["an uncovered message is both pending and built"]
+    where (> (unii.leaf-states S M) 1)
+  S M -> (unii.frontier-errors S (+ M 1)))
+
+\\ A leaf job that has a provisional stand-in counts as built, not pending.
+(define unii.leaf-states
+  {unii.state --> number --> number}
+  S M -> (let K (unii.leaf-key M)
+           (+ (if (and (unii.has-job-for-key? K (unii.st-jobs S))
+                       (not (unii.has-provisional? K (unii.st-live S)))) 1 0)
+              (if (unii.has-node? K (unii.st-live S)) 1 0))))
+
+(define unii.job-errors
+  {unii.state --> (list unii.job) --> (list string) --> (list string)}
+  _ [] _ -> []
+  _ [J | _] Seen -> ["duplicate job id"] where (element? (unii.job-id J) Seen)
+  S [J | _] _ -> ["a job targets an already built node"]
+    where (unii.has-real-node? (unii.job-key J) (unii.built-nodes S))
+  S [J | _] _ -> ["a job's attempt lies outside its round"]
+    where (not (and (unii.pos-nat? (unii.job-attempt J)) (<= (unii.job-attempt J) (unii.job-round-end J))))
+  S [J | _] _ -> ["a job keeps a candidate over the leaf cap"]
+    where (and (cons? (unii.job-best J))
+               (> (unii.cand-bytes (head (unii.job-best J))) (unii.cf-cap (unii.st-config S))))
+  S [J | _] _ -> ["a leaf job targets a covered or future message"]
+    where (and (and (unii.leaf-job? J) (not (unii.has-provisional? (unii.job-key J) (unii.built-nodes S))))
+               (or (< (unii.key-first (unii.job-key J)) (unii.st-covered S))
+                   (>= (unii.key-first (unii.job-key J)) (unii.st-count S))))
+  S [J | _] _ -> ["a merge job is missing a built child"]
+    where (and (not (unii.leaf-job? J)) (not (unii.children-built? S (unii.job-key J))))
+  S [J | Js] Seen -> (unii.job-errors S Js [(unii.job-id J) | Seen]))
+
+(define unii.children-built?
+  {unii.state --> unii.key --> boolean}
+  S K -> (unii.all-built? (unii.children K) (unii.built-nodes S)))
+
+(define unii.all-built?
+  {(list unii.key) --> (list unii.node) --> boolean}
+  [] _ -> true
+  [K | Ks] Ns -> (and (unii.has-node? K Ns) (unii.all-built? Ks Ns)))
+
+\\ Every provisional node stands in for a leaf that still has a job.
+(define unii.provisional-errors
+  {unii.state --> (list unii.node) --> (list string)}
+  _ [] -> []
+  S [N | _] -> ["a provisional node has no job for its leaf"]
+    where (and (unii.provisional? N) (not (unii.has-job-for-key? (unii.node-key N) (unii.st-jobs S))))
+  S [_ | Ns] -> (unii.provisional-errors S Ns))