fix: resolve file names against *home-directory*; per-user kernel cache (#86, #87)

CHANGELOG.md

@@ -39,6 +39,19 @@ Notable changes to shen-lua. Release notes for earlier versions are under
 
 ### Fixed
 
+- The kernel bytecode cache and its stdlib-signature sidecar default to
+  `${XDG_CACHE_HOME:-$HOME/.cache}/shen-lua/kernel-<build>.bin` (directory
+  created on first write; an unwritable location just skips caching). They
+  were written into the current directory, one ~1.8 MB copy per directory
+  shen ran from. `SHEN_KERNEL_CACHE` still overrides; old
+  `.shen-kernel-cache.*` files in project directories can be deleted
+  (pyrex41/shen-lua#87).
+- `open`, `read-file`, `read-file-as-string`, `read-file-as-bytelist` and
+  `load` resolve a relative file name against `*home-directory*`, which `cd`
+  sets, as shen-go and shen-scheme do; they used the process cwd. As there,
+  the prefix is a plain string join, applied to absolute names too. The native fast paths no longer try the cwd first,
+  so a same-named file there cannot shadow the one `open` means
+  (pyrex41/shen-lua#86).
 - `<-address` past the end of an absvector raises instead of returning `nil`,
   so the kernel's vector printer stops at the end: `(vector 3)` prints
   `<... ... ...>` (it printed stray `funex…` entries).

README.md

@@ -192,7 +192,10 @@ LuaJIT trace logs, run `luajit -jv bin/shen ...`.
 Prolog and the typechecker run on a native engine (`prolog_engine.lua`); the
 portable kernel predicates that show up on compile and execution paths are
 overridden in `prims.lua`. Caches (kernel bytecode, stdlib signatures, user fasl)
-are content-keyed and safe to delete. Internals:
+are content-keyed and safe to delete. The kernel bytecode and stdlib signatures
+live in `${XDG_CACHE_HOME:-$HOME/.cache}/shen-lua/` (`SHEN_KERNEL_CACHE=<path>`
+overrides, `=off` disables); user fasls in `~/.cache/shen-lua-fasl/`
+(`SHEN_FASL_DIR`, `SHEN_FASL=off`). Internals:
 [`doc/PERF-HANDOFF.md`](doc/PERF-HANDOFF.md),
 [`doc/BENCHMARKS.md`](doc/BENCHMARKS.md).
 

boot.lua

@@ -234,16 +234,40 @@ end
 -- invalidate + rewrite each other's cache, recompiling the kernel every time.
 -- The filename suffix is a hash of the same version/arch fingerprint that
 -- cache_key() folds into the content key.
+--
+-- The default lives in the per-user cache directory,
+-- ${XDG_CACHE_HOME:-$HOME/.cache}/shen-lua/ (issue #87: it used to be a
+-- relative path, so every directory shen ran from got its own ~1.8 MB copy
+-- and recompiled the kernel). With neither variable set there is no default
+-- and the kernel is compiled on every boot, as with SHEN_KERNEL_CACHE=off.
 local function cache_path()
   if not bit then return nil end   -- PUC Lua: no `bit` -> no cache keys
   local p = os.getenv("SHEN_KERNEL_CACHE")
   if p == "off" or p == "0" then return nil end
   if p and p ~= "" then return p end
-  return ".shen-kernel-cache."
+  local dir = os.getenv("XDG_CACHE_HOME")
+  if not dir or dir == "" then
+    local home = os.getenv("HOME")
+    if not home or home == "" then return nil end
+    dir = home .. "/.cache"
+  end
+  return dir .. "/shen-lua/kernel-"
       .. bit.tohex(fnv1a(jit and (jit.version .. jit.arch) or _VERSION))
       .. ".bin"
 end
 
+-- io.open(path, "wb"), creating path's directory on the first write. Returns
+-- nil when it still cannot be opened (read-only location): callers then skip
+-- caching.
+local function open_for_write(path)
+  local fh = io.open(path, "wb")
+  if fh then return fh end
+  local dir = path:match("^(.*)/[^/]*$")
+  if not dir or dir == "" then return nil end
+  os.execute("mkdir -p '" .. dir:gsub("'", "'\\''") .. "' 2>/dev/null")
+  return io.open(path, "wb")
+end
+
 local function read_file(path)
   local fh = io.open(path, "rb")
   if not fh then return nil end
@@ -349,18 +373,11 @@ local function write_cache(path, key, chunks, arity, counters)
   end
   parts[#parts+1] = tostring(counters.gensym) .. "\n" .. tostring(counters.infs) .. "\n"
   local tmp = path .. ".tmp"
-  local fh = io.open(tmp, "wb")
+  local fh = open_for_write(tmp)
   if not fh then return end  -- read-only dir: silently skip caching
   fh:write(table.concat(parts)); fh:close()
   os.remove(path)
   os.rename(tmp, path)
-  -- Writing a per-build default cache obsoletes the old shared-path file from
-  -- pre-per-build versions (it would sit stale forever otherwise). Only the
-  -- default path triggers this — an explicit SHEN_KERNEL_CACHE never does.
-  if path:match("^%.shen%-kernel%-cache%.%x+%.bin$") then
-    os.remove(".shen-kernel-cache.bin")
-    os.remove(".shen-kernel-cache.bin.tmp")
-  end
 end
 
 -- Parse a write_cache blob. key == nil skips the key check (used for the
@@ -1319,14 +1336,16 @@ local function install_fasl()
   local orig_load = F["load"]
   F["load"] = function(fname)
     if type(fname) ~= "string" then return orig_load(fname) end
-    local fh = io.open(fname, "rb")
+    -- the file the kernel's load would read: open resolves *home-directory*
+    local fpath = P.home_path(fname)
+    local fh = io.open(fpath, "rb")
     if not fh then return orig_load(fname) end   -- let the kernel error
     local content = fh:read("*a"); fh:close()
     -- Every file a recorded span loads is an input to that span's cache key;
     -- the boot image stores them so it can verify them all on the next boot.
     local outer = FASL_STACK[#FASL_STACK]
     if outer and outer.deps then
-      outer.deps[#outer.deps + 1] = { path = fname, content = content }
+      outer.deps[#outer.deps + 1] = { path = fpath, content = content }
     end
     local key = fasl_key(content)
     local path = dir .. "/" .. key .. ".fasl"
@@ -1413,7 +1432,7 @@ local function write_sigs(path, key, decl, counters)
   end
   parts[#parts+1] = tostring(counters.gensym) .. "\n" .. tostring(counters.infs) .. "\n"
   local tmp = path .. ".tmp"
-  local fh = io.open(tmp, "wb")
+  local fh = open_for_write(tmp)
   if not fh then return end
   fh:write(table.concat(parts)); fh:close()
   os.remove(path)

doc/PERF-HANDOFF.md

@@ -74,7 +74,9 @@ is `string.dump`'d, and lands in `.shen-kernel-cache.bin` together with
 matches. **load_kernel 0.78 s → 0.03 s warm (26×), 0.21 s with the cache off**
 (one loadstring per file instead of per form was most of the cost). Corrupt/
 stale caches fall back to full compile and rewrite. `SHEN_KERNEL_CACHE=off`
-disables, any other value overrides the path. Two traps encoded in the design:
+disables, any other value overrides the path. (The default path is now
+`${XDG_CACHE_HOME:-$HOME/.cache}/shen-lua/kernel-<build>.bin`, issue #87; it
+was a file in the cwd.) Two traps encoded in the design:
 (1) generated bytecode reads `KDATA[i]` — a side table populated at COMPILE
 time by `try_const`/`try_lit_const` — so the cache serializes KDATA (pure
 literal cons trees; tagged format in boot.lua) and restores it BEFORE running

prims.lua

@@ -675,10 +675,25 @@ defprim("shen.char-stinput?", 1, function(st) return is_stream(st) end)
 defprim("shen.write-string", 2, function(s, st) return F["write-string"](s, st) end)
 defprim("shen.read-unit-string", 1, function(st) return F["read-unit-string"](st) end)
 
+-- A file name as `open` sees it: *home-directory*, which `cd` sets (sys.kl),
+-- prepended as a plain string, as shen-scheme and shen-go do, so an absolute
+-- name after a non-empty cd is joined too. Every native path that
+-- reads a user file (read-file, read-file-as-*, the fasl `load`) resolves
+-- through this too, so it opens the same file `open` would.
+local function home_path(name)
+  local home = GLOBALS["*home-directory*"]
+  if type(home) ~= "string" or home == "" then
+    return name
+  end
+  return home .. name
+end
+P.home_path = home_path
+
 -- open (NAME DIRECTION) -> stream ; DIRECTION is symbol `in` or `out`
 defprim("open", 2, function(name, dir)
   if type(name)~="string" then ERR("open: filename not a string") end
   local d = is_symbol(dir) and dir.name or tostring(dir)
+  name = home_path(name)
   if d == "in" then
     local fh, e = io.open(name, "rb")
     if not fh then ERR("open: cannot open "..name.." ("..tostring(e)..")") end
@@ -1496,7 +1511,7 @@ function P.install_native_stdlib()
   local orig_read_file = F["read-file"]
   local function read_file(name)
     if type(name) ~= "string" then return orig_read_file(name) end
-    local fh = io.open(name, "rb")
+    local fh = io.open(home_path(name), "rb")
     if not fh then return orig_read_file(name) end   -- same open error
     local data = fh:read("*a") or ""
     fh:close()
@@ -1669,12 +1684,12 @@ function P.install_native_stdlib()
 
   -- read-file-as-bytelist / read-file-as-string: kernel is per-byte read-byte
   -- + reverse / cn (quadratic). Same path semantics as the `open` primitive
-  -- (filename as given; no *home-directory* prepend).
+  -- (relative to *home-directory*, see home_path).
   local orig_rfab = F["read-file-as-bytelist"]
   local orig_rfas = F["read-file-as-string"]
   local function read_file_as_bytelist(name)
     if type(name) ~= "string" then return orig_rfab(name) end
-    local fh = io.open(name, "rb")
+    local fh = io.open(home_path(name), "rb")
     if not fh then return orig_rfab(name) end
     local data = fh:read("*a") or ""
     fh:close()
@@ -1684,7 +1699,7 @@ function P.install_native_stdlib()
   end
   local function read_file_as_string(name)
     if type(name) ~= "string" then return orig_rfas(name) end
-    local fh = io.open(name, "rb")
+    local fh = io.open(home_path(name), "rb")
     if not fh then return orig_rfas(name) end
     local data = fh:read("*a") or ""
     fh:close()

test/boot_cache_spec.lua

@@ -4,7 +4,7 @@
 -- Two layers of the boot are cached, and each replaces work that used to be
 -- redone on every single start:
 --
---   * the kernel bytecode cache (.shen-kernel-cache.<build>.bin) holds the
+--   * the kernel bytecode cache (~/.cache/shen-lua/kernel-<build>.bin) holds the
 --     compiled ShenOSKernel 42.2 modules, including the precompiled stdlib
 --     (klambda/stlib.kl);
 --   * its sidecar (<kernel cache>.stlib) holds the 290 standard-library type
@@ -140,6 +140,59 @@ do
   for _, d in ipairs({ d1, d2 }) do os.execute("rm -rf " .. sh_quote(d)) end
 end
 
+-- ---------------------------------------------------------------------------
+-- Part 1b — the default kernel cache is per-user, not per-cwd (issue #87).
+--
+-- With SHEN_KERNEL_CACHE unset the cache (and its .stlib sidecar) goes to
+-- $XDG_CACHE_HOME/shen-lua/, else $HOME/.cache/shen-lua/, created on first
+-- write; nothing is written into the directory shen runs from.
+-- ---------------------------------------------------------------------------
+do
+  local function fresh() local d = os.tmpname(); os.remove(d); return d end
+  local function ls(d)
+    local out = {}
+    local h = io.popen("ls -A " .. sh_quote(d) .. " 2>/dev/null")
+    for l in h:lines() do out[#out + 1] = l end
+    h:close()
+    table.sort(out)
+    return table.concat(out, " ")
+  end
+  -- boot_in changes directory, so the launcher path must be absolute
+  local abs = here:sub(1, 1) == "/" and here or ((os.getenv("PWD") or ".") .. "/" .. here)
+  local shen_bin = sh_quote(abs .. "bin/shen")
+  local function boot_in(cwd, env)
+    os.execute("mkdir -p " .. sh_quote(cwd))
+    return run("cd " .. sh_quote(cwd) .. " && env -u SHEN_KERNEL_CACHE -u XDG_CACHE_HOME "
+               .. env .. " SHEN_FASL=off luajit " .. shen_bin .. " -e '(+ 1 2)'")
+  end
+
+  local root = fresh()
+  local cwd, xdg, home = root .. "/cwd", root .. "/xdg", root .. "/home"
+  local out = boot_in(cwd, "XDG_CACHE_HOME=" .. sh_quote(xdg) .. " HOME=" .. sh_quote(home))
+  check(out:match("3%s*$") ~= nil, "#87: boot with the default cache path works (got " .. out .. ")")
+  check(ls(cwd) == "", "#87: nothing is written into the cwd (got " .. ls(cwd) .. ")")
+  local files = ls(xdg .. "/shen-lua")
+  check(files:match("^kernel%-%x+%.bin kernel%-%x+%.bin%.stlib$") ~= nil,
+        "#87: kernel cache and sidecar land in $XDG_CACHE_HOME/shen-lua (got " .. files .. ")")
+  check(ls(home) == "", "#87: $HOME is untouched when XDG_CACHE_HOME is set")
+  out = boot_in(cwd, "XDG_CACHE_HOME=" .. sh_quote(xdg) .. " HOME=" .. sh_quote(home))
+  check(out:match("3%s*$") ~= nil, "#87: warm boot from the per-user cache works")
+
+  out = boot_in(cwd, "HOME=" .. sh_quote(home))
+  files = ls(home .. "/.cache/shen-lua")
+  check(out:match("3%s*$") ~= nil and files:match("^kernel%-%x+%.bin kernel%-%x+%.bin%.stlib$") ~= nil,
+        "#87: without XDG_CACHE_HOME the cache lands in $HOME/.cache/shen-lua (got " .. files .. ")")
+  check(ls(cwd) == "", "#87: still nothing in the cwd")
+
+  -- an unwritable cache location is not an error: the kernel just is not cached
+  local ro = root .. "/ro"
+  os.execute("mkdir -p " .. sh_quote(ro) .. " && chmod 555 " .. sh_quote(ro))
+  out = boot_in(cwd, "XDG_CACHE_HOME=" .. sh_quote(ro .. "/sub") .. " HOME=" .. sh_quote(home))
+  check(out:match("3%s*$") ~= nil, "#87: an unwritable cache dir still boots (got " .. out .. ")")
+  os.execute("chmod 755 " .. sh_quote(ro))
+  os.execute("rm -rf " .. sh_quote(root))
+end
+
 -- ---------------------------------------------------------------------------
 -- Part 2 — the two natives against the compiled-KL definitions they replace.
 --

test/io_spec.lua

@@ -92,6 +92,40 @@ end
 -- ---------------------------------------------------------------------------
 -- get-time: unix wall clock and run-time both return numbers.
 -- ---------------------------------------------------------------------------
+-- ---------------------------------------------------------------------------
+-- Relative file names resolve against *home-directory*, which `cd` sets
+-- (issue #86): open, read-file, read-file-as-string/-bytelist and load. A
+-- file of the same name in the process cwd must not shadow the one in the
+-- home directory (the native fast paths used to try the cwd first).
+-- ---------------------------------------------------------------------------
+do
+  local dir = os.tmpname(); os.remove(dir)
+  os.execute("mkdir -p '" .. dir .. "'")
+  local base = dir:match("[^/]*$") .. "-io86.shen"
+  local function put(path, s) local f = io.open(path, "w"); f:write(s); f:close() end
+  put(dir .. "/" .. base, "(+ 1 2)")
+  put(base, "(cwd copy)")                         -- the shadow in the cwd
+  put(dir .. "/loadme.shen", "(define io86-loaded -> home)")
+  shen.eval(string.format('(cd "%s")', dir))
+  check(evs(string.format('(read-file "%s")', base)) == "((+ 1 2))",
+        "#86: read-file resolves against *home-directory*")
+  check(shen.eval(string.format('(read-file-as-string "%s")', base)) == "(+ 1 2)",
+        "#86: read-file-as-string resolves against *home-directory*")
+  check(evs(string.format('(length (read-file-as-bytelist "%s"))', base)) == "7",
+        "#86: read-file-as-bytelist resolves against *home-directory*")
+  check(evs(string.format('(let S (open "%s" in) (let B (read-byte S) (do (close S) B)))', base))
+        == "40", "#86: open resolves against *home-directory*")
+  shen.eval('(trap-error (load "loadme.shen") (/. E skip))')
+  check(evs("(io86-loaded)") == "home", "#86: load resolves against *home-directory*")
+  check(shen.eval(string.format('(trap-error (read-file-as-string "%s/%s") (/. E "joined"))', dir, base)) == "joined",
+        "#86: *home-directory* is prepended to an absolute name too, as in shen-scheme")
+  shen.eval('(cd "")')
+  check(shen.eval(string.format('(read-file-as-string "%s")', base)) == "(cwd copy)",
+        "#86: (cd \"\") resolves against the process cwd again")
+  os.remove(base)
+  os.remove(dir .. "/" .. base); os.remove(dir .. "/loadme.shen"); os.remove(dir)
+end
+
 check(type(shen.eval("(get-time run)")) == "number", "get-time run returns a number")
 check(type(shen.eval("(get-time unix)")) == "number", "get-time unix returns a number")