lexer/parser: report non-ASCII unexpected chars as valid UTF-8

lib/lua/lexer.ex

@@ -268,9 +268,16 @@ defmodule Lua.Lexer do
     scan_identifier(<<c, rest::binary>>, "", acc, pos, pos)
   end
 
-  # Unexpected character
-  defp do_tokenize(<<c, _rest::binary>>, _acc, pos) do
-    {:error, {:unexpected_character, c, pos}}
+  # Unexpected character — carry the full codepoint so error messages stay
+  # valid UTF-8 even for multibyte characters (a byte-level match would keep
+  # only the UTF-8 lead byte).
+  defp do_tokenize(<<cp::utf8, _rest::binary>>, _acc, pos) do
+    {:error, {:unexpected_character, cp, pos}}
+  end
+
+  # Genuinely invalid UTF-8 lead byte (no valid codepoint here).
+  defp do_tokenize(<<byte, _rest::binary>>, _acc, pos) do
+    {:error, {:invalid_byte, byte, pos}}
   end
 
   # Scan single-line comment (collect text until newline)
@@ -302,6 +309,10 @@ defmodule Lua.Lexer do
     {:ok, Enum.reverse([{:eof, pos}, token | acc])}
   end
 
+  defp scan_single_line_comment_content(<<cp::utf8, rest::binary>>, text, acc, pos, start_pos) when cp > 127 do
+    scan_single_line_comment_content(rest, text <> <<cp::utf8>>, acc, advance_utf8(pos, cp), start_pos)
+  end
+
   defp scan_single_line_comment_content(<<c, rest::binary>>, text, acc, pos, start_pos) do
     scan_single_line_comment_content(rest, text <> <<c>>, acc, advance_column(pos, 1), start_pos)
   end
@@ -337,6 +348,17 @@ defmodule Lua.Lexer do
     {:error, {:unclosed_comment, pos}}
   end
 
+  defp scan_multiline_comment_text(<<cp::utf8, rest::binary>>, text, acc, pos, start_pos, level) when cp > 127 do
+    scan_multiline_comment_text(
+      rest,
+      text <> <<cp::utf8>>,
+      acc,
+      advance_utf8(pos, cp),
+      start_pos,
+      level
+    )
+  end
+
   defp scan_multiline_comment_text(<<c, rest::binary>>, text, acc, pos, start_pos, level) do
     scan_multiline_comment_text(
       rest,
@@ -435,6 +457,10 @@ defmodule Lua.Lexer do
     {:error, {:unclosed_string, pos}}
   end
 
+  defp scan_string(<<cp::utf8, rest::binary>>, str_acc, acc, pos, start_pos, quote) when cp > 127 do
+    scan_string(rest, str_acc <> <<cp::utf8>>, acc, advance_utf8(pos, cp), start_pos, quote)
+  end
+
   defp scan_string(<<c, rest::binary>>, str_acc, acc, pos, start_pos, quote) do
     scan_string(rest, str_acc <> <<c>>, acc, advance_column(pos, 1), start_pos, quote)
   end
@@ -621,6 +647,10 @@ defmodule Lua.Lexer do
     {:error, {:unclosed_long_string, pos}}
   end
 
+  defp scan_long_string(<<cp::utf8, rest::binary>>, str_acc, acc, pos, start_pos, level) when cp > 127 do
+    scan_long_string(rest, str_acc <> <<cp::utf8>>, acc, advance_utf8(pos, cp), start_pos, level)
+  end
+
   defp scan_long_string(<<c, rest::binary>>, str_acc, acc, pos, start_pos, level) do
     scan_long_string(rest, str_acc <> <<c>>, acc, advance_column(pos, 1), start_pos, level)
   end
@@ -857,6 +887,13 @@ defmodule Lua.Lexer do
     %{pos | column: pos.column + n, byte_offset: pos.byte_offset + n}
   end
 
+  # Advance one display column for a codepoint that occupies its full UTF-8
+  # byte width in the source, so `column` stays per-codepoint while
+  # `byte_offset` stays per-byte.
+  defp advance_utf8(pos, cp) do
+    %{pos | column: pos.column + 1, byte_offset: pos.byte_offset + byte_size(<<cp::utf8>>)}
+  end
+
   # Advance one source line, consuming `n` raw bytes (the backslash plus the
   # one- or two-byte line ending of a \<newline> continuation).
   defp advance_string_line(pos, n) do

lib/lua/parser.ex

@@ -1501,15 +1501,24 @@ defmodule Lua.Parser do
        "To use this expression, return it or assign it to a name."}
   end
 
-  defp convert_lexer_error({:unexpected_character, char, pos}, _code) do
-    Error.new(:invalid_syntax, "Unexpected character: #{<<char>>}", pos,
+  defp convert_lexer_error({:unexpected_character, cp, pos}, _code) do
+    Error.new(:invalid_syntax, "Unexpected character: #{<<cp::utf8>>} (U+#{codepoint_hex(cp)})", pos,
       suggestion: """
       This character is not valid in Lua syntax.
       Check for typos or invisible characters.
       """
     )
   end
 
+  defp convert_lexer_error({:invalid_byte, byte, pos}, _code) do
+    Error.new(:invalid_syntax, "Invalid byte 0x#{byte_hex(byte)}", pos,
+      suggestion: """
+      The source contains a byte that is not valid UTF-8.
+      Check the file encoding.
+      """
+    )
+  end
+
   defp convert_lexer_error({:unclosed_string, pos}, _code) do
     Error.new(:unclosed_delimiter, "Unclosed string literal", pos,
       suggestion: """
@@ -1535,6 +1544,14 @@ defmodule Lua.Parser do
     Error.new(:lexer_error, "Lexer error: #{inspect(other)}", nil)
   end
 
+  defp codepoint_hex(cp) do
+    cp |> Integer.to_string(16) |> String.upcase() |> String.pad_leading(4, "0")
+  end
+
+  defp byte_hex(byte) do
+    byte |> Integer.to_string(16) |> String.upcase() |> String.pad_leading(2, "0")
+  end
+
   defp suggest_for_token_error(type, message) do
     cond do
       type == :eof ->

lib/lua/parser/error.ex

@@ -155,7 +155,9 @@ defmodule Lua.Parser.Error do
   The shape is identical to the map produced for runtime errors, so runtime
   and parse errors can flow through a single renderer (HTML, JSON, structured
   logs). No ANSI escapes appear in any string field, and leading/trailing
-  whitespace from the internal message/suggestion templates is trimmed.
+  whitespace from the internal message/suggestion templates is trimmed. Every
+  string field is valid UTF-8 and safe to `Jason.encode!/1` — even when the
+  offending source token is a non-ASCII or malformed byte.
 
       %{
         type: atom(),
@@ -200,7 +202,7 @@ defmodule Lua.Parser.Error do
   end
 
   defp clean(nil), do: nil
-  defp clean(text) when is_binary(text), do: String.trim(text)
+  defp clean(text) when is_binary(text), do: text |> String.replace_invalid() |> String.trim()
 
   # Deliberately mirrors `Lua.VM.ErrorFormatter`'s private source-context
   # windowing (same 2-before/2-after math) to keep the wire shapes in lockstep.
@@ -222,7 +224,9 @@ defmodule Lua.Parser.Error do
         |> Enum.slice((start_line - 1)..(end_line - 1))
         |> Enum.with_index(start_line)
         |> Enum.map(fn {text, num} ->
-          %{number: num, text: text, highlight?: num == line}
+          # Source may contain malformed bytes; scrub so the wire map stays
+          # valid UTF-8 and JSON-encodable.
+          %{number: num, text: String.replace_invalid(text), highlight?: num == line}
         end)
 
       %{lines: rendered_lines, pointer_column: position.column}

test/lua/lexer_test.exs

@@ -714,6 +714,40 @@ defmodule Lua.LexerTest do
       assert {:error, {:unexpected_character, ?`, _}} = Lexer.tokenize("`")
     end
 
+    test "carries the full codepoint for a multibyte unexpected character" do
+      # A box-drawing char (U+2502, bytes E2 94 82) must be reported by its
+      # whole codepoint, not just the UTF-8 lead byte.
+      assert {:error, {:unexpected_character, 0x2502, _}} = Lexer.tokenize("│")
+    end
+
+    test "reports a genuinely invalid UTF-8 byte separately" do
+      assert {:error, {:invalid_byte, 0xFF, _}} = Lexer.tokenize(<<0xFF>>)
+    end
+
+    test "counts a multibyte character as one column inside a string" do
+      # `column` advances per codepoint while `byte_offset` stays per byte, so
+      # a 2-byte `é` in the string body must not overcount the column of the
+      # following unexpected `@`.
+      {:error, {:unexpected_character, ?@, ascii_pos}} = Lexer.tokenize(~s("cafe" @))
+      {:error, {:unexpected_character, ?@, utf8_pos}} = Lexer.tokenize(~s("café" @))
+      assert ascii_pos.column == utf8_pos.column
+      assert utf8_pos.byte_offset == ascii_pos.byte_offset + 1
+    end
+
+    test "counts a multibyte character as one column inside a comment" do
+      {:error, {:unexpected_character, ?@, ascii_pos}} = Lexer.tokenize("--[[a]]@")
+      {:error, {:unexpected_character, ?@, utf8_pos}} = Lexer.tokenize("--[[é]]@")
+      assert ascii_pos.column == utf8_pos.column
+      assert utf8_pos.byte_offset == ascii_pos.byte_offset + 1
+    end
+
+    test "counts a multibyte character as one column inside a long string" do
+      {:error, {:unexpected_character, ?@, ascii_pos}} = Lexer.tokenize("[[a]]@")
+      {:error, {:unexpected_character, ?@, utf8_pos}} = Lexer.tokenize("[[é]]@")
+      assert ascii_pos.column == utf8_pos.column
+      assert utf8_pos.byte_offset == ascii_pos.byte_offset + 1
+    end
+
     test "handles consecutive operators" do
       assert {:ok, tokens} = Lexer.tokenize("+-*/")
 

test/lua/parser/error_to_map_test.exs

@@ -130,8 +130,38 @@ defmodule Lua.Parser.ErrorToMapTest do
     end
   end
 
+  describe "UTF-8 wire safety" do
+    # Valid UTF-8 in every string field is what makes the map JSON-encodable;
+    # the library carries no JSON dependency, so we assert the invariant directly.
+    test "a multibyte unexpected character produces a valid, wire-safe message" do
+      code = "local x = │"
+      map = code |> error!() |> Error.to_map(code)
+
+      assert String.valid?(map.message)
+      assert map.message =~ "U+2502"
+      assert_all_strings_valid(map)
+    end
+
+    test "every string field stays valid UTF-8 when the source has malformed bytes" do
+      code = <<"local x = ", 0xFF>>
+      map = code |> error!() |> Error.to_map(code)
+
+      assert map.message == "Invalid byte 0xFF"
+      assert_all_strings_valid(map)
+    end
+  end
+
+  defp assert_all_strings_valid(map) do
+    assert String.valid?(map.message)
+    assert String.valid?(map.suggestion || "")
+
+    for line <- map.source_context.lines do
+      assert String.valid?(line.text)
+    end
+  end
+
   defp error!(code) do
-    {:error, [error]} = Parser.parse_structured(code)
+    {:error, [error | _]} = Parser.parse_structured(code)
     error
   end
 end