Make string primitives count Unicode code points (UTF-8)

README.md

@@ -42,6 +42,28 @@ the Kλ boundary — type *checking* is the kernel’s own Shen, unchanged.
 Numbers, strings, and booleans are Lua’s. Symbols are interned (identity `==`).
 `()` is a unique `NIL`. Cons is `{h,t}`; vectors are array tables.
 
+### Strings and Unicode
+
+A Shen string is a sequence of Unicode code points, held as a UTF-8 Lua
+string. `pos`, `tlstr`, `hdstr`, `string->n`, `explode`, `hash` (and anything
+built on them, such as `string.length`) count characters, not bytes.
+`(n->string N)` returns the UTF-8 encoding of any code point 0..U+10FFFF
+except the surrogates U+D800..U+DFFF, and raises an error for anything else.
+The reader decodes source files, `read-from-string`, and `(read Stream)` as
+UTF-8, so `"Zürich"` reads as six characters and prints back unchanged. `pr`
+writes UTF-8. `read-byte`, `write-byte`, and `read-file-as-bytelist` stay raw
+byte operations. ASCII text stays on a byte-indexed fast path.
+
+Invalid UTF-8 never raises an error and never loses data. Each byte that does
+not start a well-formed sequence counts as one character on its own: a stray
+continuation byte, a 0xC0/0xC1/0xF5–0xFF lead byte, a truncated, overlong, or
+surrogate sequence, or a value above U+10FFFF. `pos`, `tlstr`, and `explode`
+return that raw byte, so `(cn (hdstr S) (tlstr S))` is always `S`.
+`string->n` returns the byte's value (0x80–0xFF), so such a byte in source text
+reads as the character with that code (Latin-1). One thing does not round-trip:
+`(n->string (string->n B))` gives the valid two-byte encoding of that code, not
+the original byte.
+
 ## CLI
 
 ```sh
@@ -128,7 +150,7 @@ luajit run-kernel-tests.lua    # official 42 suite → 134/134
 ```
 
 The kernel suite is vendored in `tests/`. Port specs cover primitives, REPL,
-interop, tail-call lowering, boot caches.
+interop, tail-call lowering, boot caches, UTF-8 strings.
 
 ## Install
 

bifrost/bifrost.suite.json

@@ -0,0 +1,65 @@
+{
+  "name": "shen-lua-unicode",
+  "root": ".",
+  "programs_dir": "programs",
+  "default_cwd": ".",
+  "cases": [
+    {
+      "name": "unicode-n->string-252",
+      "mode": "eval",
+      "expr": "(n->string 252)",
+      "expect": "output",
+      "golden": "ü",
+      "doc": "issue #80: U+00FC as UTF-8 (C3 BC). shen-lua <= 0.11.1 printed a lone 0xFC byte"
+    },
+    {
+      "name": "unicode-n->string-8364",
+      "mode": "eval",
+      "expr": "(n->string 8364)",
+      "expect": "output",
+      "golden": "€",
+      "doc": "issue #80: code points above 255. shen-lua <= 0.11.1 raised bad argument to NSTR"
+    },
+    {
+      "name": "unicode-string->n-roundtrip",
+      "mode": "eval",
+      "expr": "(string->n (n->string 8364))",
+      "expect": "output",
+      "golden": "8364"
+    },
+    {
+      "name": "unicode-explode-length",
+      "mode": "eval",
+      "expr": "(length (explode (cn \"Z\" (cn (n->string 252) \"rich\"))))",
+      "expect": "output",
+      "golden": "6",
+      "doc": "explode counts characters, not UTF-8 bytes"
+    },
+    {
+      "name": "unicode-hash-codepoint",
+      "mode": "eval",
+      "expr": "(hash (n->string 252) 1000)",
+      "expect": "output",
+      "golden": "252",
+      "doc": "hash folds string->n (code points) per the kernel's sys.kl definition: 252 for U+00FC on shen-lua and Tarver's S42 SBCL build. shen-go hashes strings natively with a different function (874), so this golden is kernel-definition, not cross-port",
+      "known_divergence": "shen-go-native-string-hash"
+    },
+    {
+      "name": "unicode-codepoints",
+      "mode": "script",
+      "program": "unicode-codepoints.shen",
+      "expect": "output",
+      "golden": "n->string-252-is-one-char ok\nn->string-8364-is-one-char ok\nn->string-128512-is-one-char ok\nstring->n-252 ok\nstring->n-8364 ok\nstring->n-1114111 ok\npos-1 ok\npos-5 ok\ntlstr ok\nhdstr ok\nexplode-length ok\nexplode-codes ok\nhash-built-twice ok\nhash-agrees-with-= ok\nequal-built ok\nnot-equal-bytes ok\nput-get ok\nat-s-pattern ok",
+      "doc": "code-point string primitives over strings built with n->string, so no port's reader is involved"
+    },
+    {
+      "name": "unicode-literals",
+      "mode": "script",
+      "program": "unicode-literals.shen",
+      "expect": "output",
+      "golden": "literal-length ok\nliteral-string->n ok\nliteral-euro ok\nliteral-equals-built ok\nliteral-pos ok\nliteral-tlstr ok\nliteral-hash ok\ncontrol-escape ok\nGrüße, Zoë: 5 €",
+      "known_divergence": "latin1-source-reader",
+      "doc": "UTF-8 source literals read as code points. Tarver's S42 SBCL build, shen-cl (file loads) and shen-go read source BYTES and rebuild text with n->string per byte, i.e. they decode source as Latin-1: \"ü\" becomes the two characters U+00C3 U+00BC and prints as mojibake. shen-lua decodes UTF-8 (issue #80)"
+    }
+  ]
+}

bifrost/programs/unicode-codepoints.shen

@@ -0,0 +1,26 @@
+\\ unicode-codepoints.shen -- issue #80: strings are sequences of Unicode
+\\ code points. Every non-ASCII string here is BUILT with n->string, so the
+\\ result does not depend on how a port's reader decodes source bytes.
+\\ Output is ASCII only (codes and lengths), one token per line.
+(define u.zurich -> (cn "Z" (cn (n->string 252) "rich")))
+(define u.check Name Got Want ->
+  (do (output "~A ~A~%" Name (if (= Got Want) ok [Got /= Want])) ok))
+
+(u.check n->string-252-is-one-char (length (explode (n->string 252))) 1)
+(u.check n->string-8364-is-one-char (length (explode (n->string 8364))) 1)
+(u.check n->string-128512-is-one-char (length (explode (n->string 128512))) 1)
+(u.check string->n-252 (string->n (n->string 252)) 252)
+(u.check string->n-8364 (string->n (n->string 8364)) 8364)
+(u.check string->n-1114111 (string->n (n->string 1114111)) 1114111)
+(u.check pos-1 (string->n (pos (u.zurich) 1)) 252)
+(u.check pos-5 (pos (u.zurich) 5) "h")
+(u.check tlstr (tlstr (cn (n->string 252) "x")) "x")
+(u.check hdstr (string->n (hdstr (cn (n->string 8364) "uro"))) 8364)
+(u.check explode-length (length (explode (u.zurich))) 6)
+(u.check explode-codes (map (/. C (string->n C)) (explode (u.zurich))) [90 252 114 105 99 104])
+(u.check hash-built-twice (= (hash (n->string 252) 1000) (hash (cn "" (n->string 252)) 1000)) true)
+(u.check hash-agrees-with-= (= (hash (u.zurich) 99991) (hash (cn (cn "Z" (n->string 252)) "rich") 99991)) (= (u.zurich) (cn (cn "Z" (n->string 252)) "rich")))
+(u.check equal-built (= (n->string 252) (cn "" (n->string 252))) true)
+(u.check not-equal-bytes (= (u.zurich) "Zurich") false)
+(u.check put-get (do (put (u.zurich) city yes) (get (cn "Z" (cn (n->string 252) "rich")) city)) yes)
+(u.check at-s-pattern ((/. S (length (explode S))) (@s (n->string 252) (n->string 223))) 2)

bifrost/programs/unicode-literals.shen

@@ -0,0 +1,15 @@
+\\ unicode-literals.shen -- issue #80: a non-ASCII string LITERAL in a UTF-8
+\\ source file reads as its characters, prints back byte-identical, and agrees
+\\ with the same string built from code points.
+(define u.check Name Got Want ->
+  (do (output "~A ~A~%" Name (if (= Got Want) ok [Got /= Want])) ok))
+
+(u.check literal-length (length (explode "Zürich")) 6)
+(u.check literal-string->n (string->n "ü") 252)
+(u.check literal-euro (string->n "€") 8364)
+(u.check literal-equals-built (= "ü" (n->string 252)) true)
+(u.check literal-pos (pos "Zürich" 1) (n->string 252))
+(u.check literal-tlstr (tlstr "üx") "x")
+(u.check literal-hash (hash "ü" 1000) 252)
+(u.check control-escape (= "c#252;" "ü") true)
+(output "~A~%" "Grüße, Zoë: 5 €")

bin/yggdrasil-build.lua

@@ -121,9 +121,11 @@ local P = require("prims")    -- registers primitive arities into C.ARITY
 -- Contract check: every primitive the manifest expects must be provided by
 -- this port — a prims.lua F-table entry, a compiler special form, a global
 -- the generated lib boot sets, or a name that is guarded-dead here (the
--- kernel only calls it behind a port predicate this port answers false to:
--- shen.char-stoutput?/shen.char-stinput? are hardwired false in prims.lua,
--- so the shen.write-string / shen.read-unit-string branches never run).
+-- kernel only calls it behind a port predicate this port answers false to).
+-- shen.write-string and shen.read-unit-string are prims.lua entries now
+-- (shen.char-stoutput? is true so pr writes UTF-8 strings whole, issue #80;
+-- shen.char-stinput? stays false); they remain listed below so a manifest
+-- from an older stage 1 that names them never warns.
 local SPECIAL = {
   ["if"]=true, ["cond"]=true, ["let"]=true, ["do"]=true, ["trap-error"]=true,
   ["and"]=true, ["or"]=true, ["lambda"]=true, ["freeze"]=true, ["defun"]=true,
@@ -162,7 +164,7 @@ end
 -- 42 kernel (klambda/*.kl) is BACKFILLED into the kernel chunk — with a
 -- loud warning, because each backfill is a stage-1 shaker bug that should be
 -- fixed in yggdrasil.shen. Names found nowhere are warn-only (they may be
--- guarded-dead, like shen.write-string behind shen.char-stoutput?).
+-- guarded-dead, like shen.read-unit-string behind shen.char-stinput?).
 -- Limitation (same one stage 1 has): only head-position references are
 -- traced; a function passed by bare name in argument position is invisible.
 local function walk_calls(form, bound, called)

boot.lua

@@ -211,7 +211,12 @@ P.GLOBALS["*release*"]        = "0.1"  -- port release; kernel *version* comes f
 -- select direct-call vs APP codegen), the file list, and the LuaJIT version/
 -- arch (bytecode is not portable across either). SHEN_KERNEL_CACHE=off
 -- disables; any other value overrides the cache path.
-local CACHE_FORMAT = "SHENKC4"  -- 4: hoisted declare typeforms evaluated before
+local CACHE_FORMAT = "SHENKC5"  -- 5: code-point string primitives (issue #80):
+                                --    kernel chunks, fasls and stdlib images
+                                --    built over byte-based pos/tlstr/
+                                --    string->n/n->string/hash are stale
+                                --    (fasl_key and image_path fold this key)
+                                -- 4: hoisted declare typeforms evaluated before
                                 --    `declare` (issue #62); 3: per-chunk hoisted
                                 --    (declare ...) block + gensym/inference
                                 --    counters

compiler.lua

@@ -974,7 +974,19 @@ local function ccall(form, env)
         if name == "@s" and ar == 2 then return "CN(" .. argstr .. ")" end
         if name == "pos" and ar == 2 then return "POS(" .. argstr .. ")" end
         if name == "tlstr" and ar == 1 then return "TLSTR(" .. argstr .. ")" end
-        if name == "string->n" and ar == 1 then return "STRN(" .. argstr .. ")" end
+        if name == "string->n" and ar == 1 then
+          -- (string->n (pos S I)) / (string->n (hdstr S)): read the code in
+          -- place (STRNPOS) instead of building a one-character string just
+          -- to take its code. The argument's own lowering is exactly
+          -- "POS(<args>)" (see pos/hdstr below) unless it was hoisted.
+          local a = args[1]
+          if is_cons(a) and is_symbol(car(a)) and not env[car(a).name]
+             and (car(a).name == "pos" or car(a).name == "hdstr") then
+            local inner = cargs[1]:match("^POS%((.*)%)$")
+            if inner then return "STRNPOS(" .. inner .. ")" end
+          end
+          return "STRN(" .. argstr .. ")"
+        end
         if name == "n->string" and ar == 1 then return "NSTR(" .. argstr .. ")" end
         if name == "hdstr" and ar == 1 then return "POS(" .. argstr .. ", 0)" end
         if name == "shen.hds=?" and ar == 2 then return "HDS_EQ(" .. argstr .. ")" end