From 82afe277aaa04160fd8f54224425bcc64c4703e0 Mon Sep 17 00:00:00 2001 From: pyrex41 Date: Sun, 18 Oct 2026 17:00:39 -0500 Subject: [PATCH 1/3] fix(io): resolve relative file names against *home-directory* (#86) open, read-file, read-file-as-string, read-file-as-bytelist and the fasl load wrapper passed relative names straight to io.open, so after (cd Dir) they still read from the process cwd. They now prepend *home-directory* (set by cd) to non-absolute names, as the kernel's open contract and shen-go/shen-scheme do. The native fast paths share one resolver (P.home_path) with open, so they can no longer pick up a same-named file in the cwd that open would not. Fixes #86 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++++ boot.lua | 6 ++++-- prims.lua | 22 ++++++++++++++++++---- test/io_spec.lua | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea6a4a3..49cfd3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,12 @@ Notable changes to shen-lua. Release notes for earlier versions are under ### Fixed +- `open`, `read-file`, `read-file-as-string`, `read-file-as-bytelist` and + `load` resolve a relative file name against `*home-directory*`, which `cd` + sets, as shen-go and shen-scheme do; they used the process cwd. Absolute + names are used as given. The native fast paths no longer try the cwd first, + so a same-named file there cannot shadow the one `open` means + (pyrex41/shen-lua#86). - `<-address` past the end of an absvector raises instead of returning `nil`, so the kernel's vector printer stops at the end: `(vector 3)` prints `<... ... ...>` (it printed stray `funex…` entries). diff --git a/boot.lua b/boot.lua index 3afab75..e7a7a36 100644 --- a/boot.lua +++ b/boot.lua @@ -1319,14 +1319,16 @@ local function install_fasl() local orig_load = F["load"] F["load"] = function(fname) if type(fname) ~= "string" then return orig_load(fname) end - local fh = io.open(fname, "rb") + -- the file the kernel's load would read: open resolves *home-directory* + local fpath = P.home_path(fname) + local fh = io.open(fpath, "rb") if not fh then return orig_load(fname) end -- let the kernel error local content = fh:read("*a"); fh:close() -- Every file a recorded span loads is an input to that span's cache key; -- the boot image stores them so it can verify them all on the next boot. local outer = FASL_STACK[#FASL_STACK] if outer and outer.deps then - outer.deps[#outer.deps + 1] = { path = fname, content = content } + outer.deps[#outer.deps + 1] = { path = fpath, content = content } end local key = fasl_key(content) local path = dir .. "/" .. key .. ".fasl" diff --git a/prims.lua b/prims.lua index 6996169..26c7ecd 100644 --- a/prims.lua +++ b/prims.lua @@ -675,10 +675,24 @@ defprim("shen.char-stinput?", 1, function(st) return is_stream(st) end) defprim("shen.write-string", 2, function(s, st) return F["write-string"](s, st) end) defprim("shen.read-unit-string", 1, function(st) return F["read-unit-string"](st) end) +-- A file name as `open` sees it: relative to *home-directory*, which `cd` +-- sets (sys.kl). Absolute names are used as given. Every native path that +-- reads a user file (read-file, read-file-as-*, the fasl `load`) resolves +-- through this too, so it opens the same file `open` would. +local function home_path(name) + local home = GLOBALS["*home-directory*"] + if type(home) ~= "string" or home == "" or name:sub(1, 1) == "/" then + return name + end + return home .. name +end +P.home_path = home_path + -- open (NAME DIRECTION) -> stream ; DIRECTION is symbol `in` or `out` defprim("open", 2, function(name, dir) if type(name)~="string" then ERR("open: filename not a string") end local d = is_symbol(dir) and dir.name or tostring(dir) + name = home_path(name) if d == "in" then local fh, e = io.open(name, "rb") if not fh then ERR("open: cannot open "..name.." ("..tostring(e)..")") end @@ -1496,7 +1510,7 @@ function P.install_native_stdlib() local orig_read_file = F["read-file"] local function read_file(name) if type(name) ~= "string" then return orig_read_file(name) end - local fh = io.open(name, "rb") + local fh = io.open(home_path(name), "rb") if not fh then return orig_read_file(name) end -- same open error local data = fh:read("*a") or "" fh:close() @@ -1669,12 +1683,12 @@ function P.install_native_stdlib() -- read-file-as-bytelist / read-file-as-string: kernel is per-byte read-byte -- + reverse / cn (quadratic). Same path semantics as the `open` primitive - -- (filename as given; no *home-directory* prepend). + -- (relative to *home-directory*, see home_path). local orig_rfab = F["read-file-as-bytelist"] local orig_rfas = F["read-file-as-string"] local function read_file_as_bytelist(name) if type(name) ~= "string" then return orig_rfab(name) end - local fh = io.open(name, "rb") + local fh = io.open(home_path(name), "rb") if not fh then return orig_rfab(name) end local data = fh:read("*a") or "" fh:close() @@ -1684,7 +1698,7 @@ function P.install_native_stdlib() end local function read_file_as_string(name) if type(name) ~= "string" then return orig_rfas(name) end - local fh = io.open(name, "rb") + local fh = io.open(home_path(name), "rb") if not fh then return orig_rfas(name) end local data = fh:read("*a") or "" fh:close() diff --git a/test/io_spec.lua b/test/io_spec.lua index 62710dc..b556e45 100644 --- a/test/io_spec.lua +++ b/test/io_spec.lua @@ -92,6 +92,40 @@ end -- --------------------------------------------------------------------------- -- get-time: unix wall clock and run-time both return numbers. -- --------------------------------------------------------------------------- +-- --------------------------------------------------------------------------- +-- Relative file names resolve against *home-directory*, which `cd` sets +-- (issue #86): open, read-file, read-file-as-string/-bytelist and load. A +-- file of the same name in the process cwd must not shadow the one in the +-- home directory (the native fast paths used to try the cwd first). +-- --------------------------------------------------------------------------- +do + local dir = os.tmpname(); os.remove(dir) + os.execute("mkdir -p '" .. dir .. "'") + local base = dir:match("[^/]*$") .. "-io86.shen" + local function put(path, s) local f = io.open(path, "w"); f:write(s); f:close() end + put(dir .. "/" .. base, "(+ 1 2)") + put(base, "(cwd copy)") -- the shadow in the cwd + put(dir .. "/loadme.shen", "(define io86-loaded -> home)") + shen.eval(string.format('(cd "%s")', dir)) + check(evs(string.format('(read-file "%s")', base)) == "((+ 1 2))", + "#86: read-file resolves against *home-directory*") + check(shen.eval(string.format('(read-file-as-string "%s")', base)) == "(+ 1 2)", + "#86: read-file-as-string resolves against *home-directory*") + check(evs(string.format('(length (read-file-as-bytelist "%s"))', base)) == "7", + "#86: read-file-as-bytelist resolves against *home-directory*") + check(evs(string.format('(let S (open "%s" in) (let B (read-byte S) (do (close S) B)))', base)) + == "40", "#86: open resolves against *home-directory*") + shen.eval('(trap-error (load "loadme.shen") (/. E skip))') + check(evs("(io86-loaded)") == "home", "#86: load resolves against *home-directory*") + check(shen.eval(string.format('(read-file-as-string "%s/%s")', dir, base)) == "(+ 1 2)", + "#86: an absolute name is used as given after cd") + shen.eval('(cd "")') + check(shen.eval(string.format('(read-file-as-string "%s")', base)) == "(cwd copy)", + "#86: (cd \"\") resolves against the process cwd again") + os.remove(base) + os.remove(dir .. "/" .. base); os.remove(dir .. "/loadme.shen"); os.remove(dir) +end + check(type(shen.eval("(get-time run)")) == "number", "get-time run returns a number") check(type(shen.eval("(get-time unix)")) == "number", "get-time unix returns a number") From 7ad7ae47a4e040d518ae093f3c6f3c93ed3d8c4f Mon Sep 17 00:00:00 2001 From: pyrex41 Date: Sun, 18 Oct 2026 17:00:39 -0500 Subject: [PATCH 2/3] fix(boot): keep the kernel cache in the per-user cache dir (#87) With SHEN_KERNEL_CACHE unset, cache_path() returned the relative ".shen-kernel-cache..bin", so every directory shen ran from got a ~1.8 MB cache plus its .stlib sidecar and recompiled the kernel. The default is now ${XDG_CACHE_HOME:-$HOME/.cache}/shen-lua/kernel-.bin; the directory is created on the first write, and an unwritable location skips caching as before. SHEN_KERNEL_CACHE=|off is unchanged. The cleanup of the pre-per-build ".shen-kernel-cache.bin" in the cwd only ran for the old relative default, so it is removed with it. Fixes #87 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +++++ README.md | 5 +++- boot.lua | 37 +++++++++++++++++++-------- doc/PERF-HANDOFF.md | 4 ++- test/boot_cache_spec.lua | 55 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 95 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49cfd3e..43ab962 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,13 @@ Notable changes to shen-lua. Release notes for earlier versions are under ### Fixed +- The kernel bytecode cache and its stdlib-signature sidecar default to + `${XDG_CACHE_HOME:-$HOME/.cache}/shen-lua/kernel-.bin` (directory + created on first write; an unwritable location just skips caching). They + were written into the current directory, one ~1.8 MB copy per directory + shen ran from. `SHEN_KERNEL_CACHE` still overrides; old + `.shen-kernel-cache.*` files in project directories can be deleted + (pyrex41/shen-lua#87). - `open`, `read-file`, `read-file-as-string`, `read-file-as-bytelist` and `load` resolve a relative file name against `*home-directory*`, which `cd` sets, as shen-go and shen-scheme do; they used the process cwd. Absolute diff --git a/README.md b/README.md index e03dcbb..5a0c231 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,10 @@ LuaJIT trace logs, run `luajit -jv bin/shen ...`. Prolog and the typechecker run on a native engine (`prolog_engine.lua`); the portable kernel predicates that show up on compile and execution paths are overridden in `prims.lua`. Caches (kernel bytecode, stdlib signatures, user fasl) -are content-keyed and safe to delete. Internals: +are content-keyed and safe to delete. The kernel bytecode and stdlib signatures +live in `${XDG_CACHE_HOME:-$HOME/.cache}/shen-lua/` (`SHEN_KERNEL_CACHE=` +overrides, `=off` disables); user fasls in `~/.cache/shen-lua-fasl/` +(`SHEN_FASL_DIR`, `SHEN_FASL=off`). Internals: [`doc/PERF-HANDOFF.md`](doc/PERF-HANDOFF.md), [`doc/BENCHMARKS.md`](doc/BENCHMARKS.md). diff --git a/boot.lua b/boot.lua index e7a7a36..65b0c8f 100644 --- a/boot.lua +++ b/boot.lua @@ -234,16 +234,40 @@ end -- invalidate + rewrite each other's cache, recompiling the kernel every time. -- The filename suffix is a hash of the same version/arch fingerprint that -- cache_key() folds into the content key. +-- +-- The default lives in the per-user cache directory, +-- ${XDG_CACHE_HOME:-$HOME/.cache}/shen-lua/ (issue #87: it used to be a +-- relative path, so every directory shen ran from got its own ~1.8 MB copy +-- and recompiled the kernel). With neither variable set there is no default +-- and the kernel is compiled on every boot, as with SHEN_KERNEL_CACHE=off. local function cache_path() if not bit then return nil end -- PUC Lua: no `bit` -> no cache keys local p = os.getenv("SHEN_KERNEL_CACHE") if p == "off" or p == "0" then return nil end if p and p ~= "" then return p end - return ".shen-kernel-cache." + local dir = os.getenv("XDG_CACHE_HOME") + if not dir or dir == "" then + local home = os.getenv("HOME") + if not home or home == "" then return nil end + dir = home .. "/.cache" + end + return dir .. "/shen-lua/kernel-" .. bit.tohex(fnv1a(jit and (jit.version .. jit.arch) or _VERSION)) .. ".bin" end +-- io.open(path, "wb"), creating path's directory on the first write. Returns +-- nil when it still cannot be opened (read-only location): callers then skip +-- caching. +local function open_for_write(path) + local fh = io.open(path, "wb") + if fh then return fh end + local dir = path:match("^(.*)/[^/]*$") + if not dir or dir == "" then return nil end + os.execute("mkdir -p '" .. dir:gsub("'", "'\\''") .. "' 2>/dev/null") + return io.open(path, "wb") +end + local function read_file(path) local fh = io.open(path, "rb") if not fh then return nil end @@ -349,18 +373,11 @@ local function write_cache(path, key, chunks, arity, counters) end parts[#parts+1] = tostring(counters.gensym) .. "\n" .. tostring(counters.infs) .. "\n" local tmp = path .. ".tmp" - local fh = io.open(tmp, "wb") + local fh = open_for_write(tmp) if not fh then return end -- read-only dir: silently skip caching fh:write(table.concat(parts)); fh:close() os.remove(path) os.rename(tmp, path) - -- Writing a per-build default cache obsoletes the old shared-path file from - -- pre-per-build versions (it would sit stale forever otherwise). Only the - -- default path triggers this — an explicit SHEN_KERNEL_CACHE never does. - if path:match("^%.shen%-kernel%-cache%.%x+%.bin$") then - os.remove(".shen-kernel-cache.bin") - os.remove(".shen-kernel-cache.bin.tmp") - end end -- Parse a write_cache blob. key == nil skips the key check (used for the @@ -1415,7 +1432,7 @@ local function write_sigs(path, key, decl, counters) end parts[#parts+1] = tostring(counters.gensym) .. "\n" .. tostring(counters.infs) .. "\n" local tmp = path .. ".tmp" - local fh = io.open(tmp, "wb") + local fh = open_for_write(tmp) if not fh then return end fh:write(table.concat(parts)); fh:close() os.remove(path) diff --git a/doc/PERF-HANDOFF.md b/doc/PERF-HANDOFF.md index 4ff61e4..94be06e 100644 --- a/doc/PERF-HANDOFF.md +++ b/doc/PERF-HANDOFF.md @@ -74,7 +74,9 @@ is `string.dump`'d, and lands in `.shen-kernel-cache.bin` together with matches. **load_kernel 0.78 s → 0.03 s warm (26×), 0.21 s with the cache off** (one loadstring per file instead of per form was most of the cost). Corrupt/ stale caches fall back to full compile and rewrite. `SHEN_KERNEL_CACHE=off` -disables, any other value overrides the path. Two traps encoded in the design: +disables, any other value overrides the path. (The default path is now +`${XDG_CACHE_HOME:-$HOME/.cache}/shen-lua/kernel-.bin`, issue #87; it +was a file in the cwd.) Two traps encoded in the design: (1) generated bytecode reads `KDATA[i]` — a side table populated at COMPILE time by `try_const`/`try_lit_const` — so the cache serializes KDATA (pure literal cons trees; tagged format in boot.lua) and restores it BEFORE running diff --git a/test/boot_cache_spec.lua b/test/boot_cache_spec.lua index 556c955..aa652f9 100644 --- a/test/boot_cache_spec.lua +++ b/test/boot_cache_spec.lua @@ -4,7 +4,7 @@ -- Two layers of the boot are cached, and each replaces work that used to be -- redone on every single start: -- --- * the kernel bytecode cache (.shen-kernel-cache..bin) holds the +-- * the kernel bytecode cache (~/.cache/shen-lua/kernel-.bin) holds the -- compiled ShenOSKernel 42.2 modules, including the precompiled stdlib -- (klambda/stlib.kl); -- * its sidecar (.stlib) holds the 290 standard-library type @@ -140,6 +140,59 @@ do for _, d in ipairs({ d1, d2 }) do os.execute("rm -rf " .. sh_quote(d)) end end +-- --------------------------------------------------------------------------- +-- Part 1b — the default kernel cache is per-user, not per-cwd (issue #87). +-- +-- With SHEN_KERNEL_CACHE unset the cache (and its .stlib sidecar) goes to +-- $XDG_CACHE_HOME/shen-lua/, else $HOME/.cache/shen-lua/, created on first +-- write; nothing is written into the directory shen runs from. +-- --------------------------------------------------------------------------- +do + local function fresh() local d = os.tmpname(); os.remove(d); return d end + local function ls(d) + local out = {} + local h = io.popen("ls -A " .. sh_quote(d) .. " 2>/dev/null") + for l in h:lines() do out[#out + 1] = l end + h:close() + table.sort(out) + return table.concat(out, " ") + end + -- boot_in changes directory, so the launcher path must be absolute + local abs = here:sub(1, 1) == "/" and here or ((os.getenv("PWD") or ".") .. "/" .. here) + local shen_bin = sh_quote(abs .. "bin/shen") + local function boot_in(cwd, env) + os.execute("mkdir -p " .. sh_quote(cwd)) + return run("cd " .. sh_quote(cwd) .. " && env -u SHEN_KERNEL_CACHE -u XDG_CACHE_HOME " + .. env .. " SHEN_FASL=off luajit " .. shen_bin .. " -e '(+ 1 2)'") + end + + local root = fresh() + local cwd, xdg, home = root .. "/cwd", root .. "/xdg", root .. "/home" + local out = boot_in(cwd, "XDG_CACHE_HOME=" .. sh_quote(xdg) .. " HOME=" .. sh_quote(home)) + check(out:match("3%s*$") ~= nil, "#87: boot with the default cache path works (got " .. out .. ")") + check(ls(cwd) == "", "#87: nothing is written into the cwd (got " .. ls(cwd) .. ")") + local files = ls(xdg .. "/shen-lua") + check(files:match("^kernel%-%x+%.bin kernel%-%x+%.bin%.stlib$") ~= nil, + "#87: kernel cache and sidecar land in $XDG_CACHE_HOME/shen-lua (got " .. files .. ")") + check(ls(home) == "", "#87: $HOME is untouched when XDG_CACHE_HOME is set") + out = boot_in(cwd, "XDG_CACHE_HOME=" .. sh_quote(xdg) .. " HOME=" .. sh_quote(home)) + check(out:match("3%s*$") ~= nil, "#87: warm boot from the per-user cache works") + + out = boot_in(cwd, "HOME=" .. sh_quote(home)) + files = ls(home .. "/.cache/shen-lua") + check(out:match("3%s*$") ~= nil and files:match("^kernel%-%x+%.bin kernel%-%x+%.bin%.stlib$") ~= nil, + "#87: without XDG_CACHE_HOME the cache lands in $HOME/.cache/shen-lua (got " .. files .. ")") + check(ls(cwd) == "", "#87: still nothing in the cwd") + + -- an unwritable cache location is not an error: the kernel just is not cached + local ro = root .. "/ro" + os.execute("mkdir -p " .. sh_quote(ro) .. " && chmod 555 " .. sh_quote(ro)) + out = boot_in(cwd, "XDG_CACHE_HOME=" .. sh_quote(ro .. "/sub") .. " HOME=" .. sh_quote(home)) + check(out:match("3%s*$") ~= nil, "#87: an unwritable cache dir still boots (got " .. out .. ")") + os.execute("chmod 755 " .. sh_quote(ro)) + os.execute("rm -rf " .. sh_quote(root)) +end + -- --------------------------------------------------------------------------- -- Part 2 — the two natives against the compiled-KL definitions they replace. -- From c349fffb785ad1f5e5ec96eb78589dc1fe572e83 Mon Sep 17 00:00:00 2001 From: pyrex41 Date: Mon, 19 Oct 2026 06:41:09 -0500 Subject: [PATCH 3/3] fix(io): prepend *home-directory* to absolute names too, as shen-scheme does (#86) shen-scheme and shen-go join *home-directory* and the file name as plain strings, so after a non-empty cd an absolute name is joined as well. Match that instead of passing absolute names through. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++-- prims.lua | 7 ++++--- test/io_spec.lua | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43ab962..09074d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,8 +48,8 @@ Notable changes to shen-lua. Release notes for earlier versions are under (pyrex41/shen-lua#87). - `open`, `read-file`, `read-file-as-string`, `read-file-as-bytelist` and `load` resolve a relative file name against `*home-directory*`, which `cd` - sets, as shen-go and shen-scheme do; they used the process cwd. Absolute - names are used as given. The native fast paths no longer try the cwd first, + sets, as shen-go and shen-scheme do; they used the process cwd. As there, + the prefix is a plain string join, applied to absolute names too. The native fast paths no longer try the cwd first, so a same-named file there cannot shadow the one `open` means (pyrex41/shen-lua#86). - `<-address` past the end of an absvector raises instead of returning `nil`, diff --git a/prims.lua b/prims.lua index 26c7ecd..738359a 100644 --- a/prims.lua +++ b/prims.lua @@ -675,13 +675,14 @@ defprim("shen.char-stinput?", 1, function(st) return is_stream(st) end) defprim("shen.write-string", 2, function(s, st) return F["write-string"](s, st) end) defprim("shen.read-unit-string", 1, function(st) return F["read-unit-string"](st) end) --- A file name as `open` sees it: relative to *home-directory*, which `cd` --- sets (sys.kl). Absolute names are used as given. Every native path that +-- A file name as `open` sees it: *home-directory*, which `cd` sets (sys.kl), +-- prepended as a plain string, as shen-scheme and shen-go do, so an absolute +-- name after a non-empty cd is joined too. Every native path that -- reads a user file (read-file, read-file-as-*, the fasl `load`) resolves -- through this too, so it opens the same file `open` would. local function home_path(name) local home = GLOBALS["*home-directory*"] - if type(home) ~= "string" or home == "" or name:sub(1, 1) == "/" then + if type(home) ~= "string" or home == "" then return name end return home .. name diff --git a/test/io_spec.lua b/test/io_spec.lua index b556e45..ad01de8 100644 --- a/test/io_spec.lua +++ b/test/io_spec.lua @@ -117,8 +117,8 @@ do == "40", "#86: open resolves against *home-directory*") shen.eval('(trap-error (load "loadme.shen") (/. E skip))') check(evs("(io86-loaded)") == "home", "#86: load resolves against *home-directory*") - check(shen.eval(string.format('(read-file-as-string "%s/%s")', dir, base)) == "(+ 1 2)", - "#86: an absolute name is used as given after cd") + check(shen.eval(string.format('(trap-error (read-file-as-string "%s/%s") (/. E "joined"))', dir, base)) == "joined", + "#86: *home-directory* is prepended to an absolute name too, as in shen-scheme") shen.eval('(cd "")') check(shen.eval(string.format('(read-file-as-string "%s")', base)) == "(cwd copy)", "#86: (cd \"\") resolves against the process cwd again")