From bea9d6bc5e89812df01d1258bf72f4da7ab9af29 Mon Sep 17 00:00:00 2001 From: Dave Lucia Date: Thu, 16 Jul 2026 11:10:14 -0700 Subject: [PATCH] lexer/parser: report non-ASCII unexpected chars as valid UTF-8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lexer matched a single byte for an unexpected character, capturing only the UTF-8 lead byte of a multibyte character. The parser spliced that raw byte into the error message, producing invalid UTF-8 that crashed any JSON encoder — despite Lua.Parser.Error.to_map/2's "wire-safe" contract. - Lexer: match the full codepoint (<>); add an :invalid_byte fallback for genuinely malformed bytes. - Parser: render the codepoint safely and name it U+XXXX; add an :invalid_byte message. - Advance `column` per codepoint (not per byte) in the string, comment, and long-string scanners so positions stay UTF-8-correct. - Make to_map/2's wire-safe guarantee real: scrub source-context text and messages with String.replace_invalid/1, and document that every string field is valid UTF-8. Fixes #394 --- lib/lua/lexer.ex | 43 +++++++++++++++++++++++++-- lib/lua/parser.ex | 21 +++++++++++-- lib/lua/parser/error.ex | 10 +++++-- test/lua/lexer_test.exs | 34 +++++++++++++++++++++ test/lua/parser/error_to_map_test.exs | 32 +++++++++++++++++++- 5 files changed, 131 insertions(+), 9 deletions(-) diff --git a/lib/lua/lexer.ex b/lib/lua/lexer.ex index 39299a2e..b2338a6d 100644 --- a/lib/lua/lexer.ex +++ b/lib/lua/lexer.ex @@ -268,9 +268,16 @@ defmodule Lua.Lexer do scan_identifier(<>, "", acc, pos, pos) end - # Unexpected character - defp do_tokenize(<>, _acc, pos) do - {:error, {:unexpected_character, c, pos}} + # Unexpected character — carry the full codepoint so error messages stay + # valid UTF-8 even for multibyte characters (a byte-level match would keep + # only the UTF-8 lead byte). + defp do_tokenize(<>, _acc, pos) do + {:error, {:unexpected_character, cp, pos}} + end + + # Genuinely invalid UTF-8 lead byte (no valid codepoint here). + defp do_tokenize(<>, _acc, pos) do + {:error, {:invalid_byte, byte, pos}} end # Scan single-line comment (collect text until newline) @@ -302,6 +309,10 @@ defmodule Lua.Lexer do {:ok, Enum.reverse([{:eof, pos}, token | acc])} end + defp scan_single_line_comment_content(<>, text, acc, pos, start_pos) when cp > 127 do + scan_single_line_comment_content(rest, text <> <>, acc, advance_utf8(pos, cp), start_pos) + end + defp scan_single_line_comment_content(<>, text, acc, pos, start_pos) do scan_single_line_comment_content(rest, text <> <>, acc, advance_column(pos, 1), start_pos) end @@ -337,6 +348,17 @@ defmodule Lua.Lexer do {:error, {:unclosed_comment, pos}} end + defp scan_multiline_comment_text(<>, text, acc, pos, start_pos, level) when cp > 127 do + scan_multiline_comment_text( + rest, + text <> <>, + acc, + advance_utf8(pos, cp), + start_pos, + level + ) + end + defp scan_multiline_comment_text(<>, text, acc, pos, start_pos, level) do scan_multiline_comment_text( rest, @@ -435,6 +457,10 @@ defmodule Lua.Lexer do {:error, {:unclosed_string, pos}} end + defp scan_string(<>, str_acc, acc, pos, start_pos, quote) when cp > 127 do + scan_string(rest, str_acc <> <>, acc, advance_utf8(pos, cp), start_pos, quote) + end + defp scan_string(<>, str_acc, acc, pos, start_pos, quote) do scan_string(rest, str_acc <> <>, acc, advance_column(pos, 1), start_pos, quote) end @@ -621,6 +647,10 @@ defmodule Lua.Lexer do {:error, {:unclosed_long_string, pos}} end + defp scan_long_string(<>, str_acc, acc, pos, start_pos, level) when cp > 127 do + scan_long_string(rest, str_acc <> <>, acc, advance_utf8(pos, cp), start_pos, level) + end + defp scan_long_string(<>, str_acc, acc, pos, start_pos, level) do scan_long_string(rest, str_acc <> <>, acc, advance_column(pos, 1), start_pos, level) end @@ -857,6 +887,13 @@ defmodule Lua.Lexer do %{pos | column: pos.column + n, byte_offset: pos.byte_offset + n} end + # Advance one display column for a codepoint that occupies its full UTF-8 + # byte width in the source, so `column` stays per-codepoint while + # `byte_offset` stays per-byte. + defp advance_utf8(pos, cp) do + %{pos | column: pos.column + 1, byte_offset: pos.byte_offset + byte_size(<>)} + end + # Advance one source line, consuming `n` raw bytes (the backslash plus the # one- or two-byte line ending of a \ continuation). defp advance_string_line(pos, n) do diff --git a/lib/lua/parser.ex b/lib/lua/parser.ex index 51671cca..4c43efef 100644 --- a/lib/lua/parser.ex +++ b/lib/lua/parser.ex @@ -1501,8 +1501,8 @@ defmodule Lua.Parser do "To use this expression, return it or assign it to a name."} end - defp convert_lexer_error({:unexpected_character, char, pos}, _code) do - Error.new(:invalid_syntax, "Unexpected character: #{<>}", pos, + defp convert_lexer_error({:unexpected_character, cp, pos}, _code) do + Error.new(:invalid_syntax, "Unexpected character: #{<>} (U+#{codepoint_hex(cp)})", pos, suggestion: """ This character is not valid in Lua syntax. Check for typos or invisible characters. @@ -1510,6 +1510,15 @@ defmodule Lua.Parser do ) end + defp convert_lexer_error({:invalid_byte, byte, pos}, _code) do + Error.new(:invalid_syntax, "Invalid byte 0x#{byte_hex(byte)}", pos, + suggestion: """ + The source contains a byte that is not valid UTF-8. + Check the file encoding. + """ + ) + end + defp convert_lexer_error({:unclosed_string, pos}, _code) do Error.new(:unclosed_delimiter, "Unclosed string literal", pos, suggestion: """ @@ -1535,6 +1544,14 @@ defmodule Lua.Parser do Error.new(:lexer_error, "Lexer error: #{inspect(other)}", nil) end + defp codepoint_hex(cp) do + cp |> Integer.to_string(16) |> String.upcase() |> String.pad_leading(4, "0") + end + + defp byte_hex(byte) do + byte |> Integer.to_string(16) |> String.upcase() |> String.pad_leading(2, "0") + end + defp suggest_for_token_error(type, message) do cond do type == :eof -> diff --git a/lib/lua/parser/error.ex b/lib/lua/parser/error.ex index 1d25c610..2ec17203 100644 --- a/lib/lua/parser/error.ex +++ b/lib/lua/parser/error.ex @@ -155,7 +155,9 @@ defmodule Lua.Parser.Error do The shape is identical to the map produced for runtime errors, so runtime and parse errors can flow through a single renderer (HTML, JSON, structured logs). No ANSI escapes appear in any string field, and leading/trailing - whitespace from the internal message/suggestion templates is trimmed. + whitespace from the internal message/suggestion templates is trimmed. Every + string field is valid UTF-8 and safe to `Jason.encode!/1` — even when the + offending source token is a non-ASCII or malformed byte. %{ type: atom(), @@ -200,7 +202,7 @@ defmodule Lua.Parser.Error do end defp clean(nil), do: nil - defp clean(text) when is_binary(text), do: String.trim(text) + defp clean(text) when is_binary(text), do: text |> String.replace_invalid() |> String.trim() # Deliberately mirrors `Lua.VM.ErrorFormatter`'s private source-context # windowing (same 2-before/2-after math) to keep the wire shapes in lockstep. @@ -222,7 +224,9 @@ defmodule Lua.Parser.Error do |> Enum.slice((start_line - 1)..(end_line - 1)) |> Enum.with_index(start_line) |> Enum.map(fn {text, num} -> - %{number: num, text: text, highlight?: num == line} + # Source may contain malformed bytes; scrub so the wire map stays + # valid UTF-8 and JSON-encodable. + %{number: num, text: String.replace_invalid(text), highlight?: num == line} end) %{lines: rendered_lines, pointer_column: position.column} diff --git a/test/lua/lexer_test.exs b/test/lua/lexer_test.exs index 5ffd9bcb..fa1ab581 100644 --- a/test/lua/lexer_test.exs +++ b/test/lua/lexer_test.exs @@ -714,6 +714,40 @@ defmodule Lua.LexerTest do assert {:error, {:unexpected_character, ?`, _}} = Lexer.tokenize("`") end + test "carries the full codepoint for a multibyte unexpected character" do + # A box-drawing char (U+2502, bytes E2 94 82) must be reported by its + # whole codepoint, not just the UTF-8 lead byte. + assert {:error, {:unexpected_character, 0x2502, _}} = Lexer.tokenize("│") + end + + test "reports a genuinely invalid UTF-8 byte separately" do + assert {:error, {:invalid_byte, 0xFF, _}} = Lexer.tokenize(<<0xFF>>) + end + + test "counts a multibyte character as one column inside a string" do + # `column` advances per codepoint while `byte_offset` stays per byte, so + # a 2-byte `é` in the string body must not overcount the column of the + # following unexpected `@`. + {:error, {:unexpected_character, ?@, ascii_pos}} = Lexer.tokenize(~s("cafe" @)) + {:error, {:unexpected_character, ?@, utf8_pos}} = Lexer.tokenize(~s("café" @)) + assert ascii_pos.column == utf8_pos.column + assert utf8_pos.byte_offset == ascii_pos.byte_offset + 1 + end + + test "counts a multibyte character as one column inside a comment" do + {:error, {:unexpected_character, ?@, ascii_pos}} = Lexer.tokenize("--[[a]]@") + {:error, {:unexpected_character, ?@, utf8_pos}} = Lexer.tokenize("--[[é]]@") + assert ascii_pos.column == utf8_pos.column + assert utf8_pos.byte_offset == ascii_pos.byte_offset + 1 + end + + test "counts a multibyte character as one column inside a long string" do + {:error, {:unexpected_character, ?@, ascii_pos}} = Lexer.tokenize("[[a]]@") + {:error, {:unexpected_character, ?@, utf8_pos}} = Lexer.tokenize("[[é]]@") + assert ascii_pos.column == utf8_pos.column + assert utf8_pos.byte_offset == ascii_pos.byte_offset + 1 + end + test "handles consecutive operators" do assert {:ok, tokens} = Lexer.tokenize("+-*/") diff --git a/test/lua/parser/error_to_map_test.exs b/test/lua/parser/error_to_map_test.exs index ba5ad570..b9721269 100644 --- a/test/lua/parser/error_to_map_test.exs +++ b/test/lua/parser/error_to_map_test.exs @@ -130,8 +130,38 @@ defmodule Lua.Parser.ErrorToMapTest do end end + describe "UTF-8 wire safety" do + # Valid UTF-8 in every string field is what makes the map JSON-encodable; + # the library carries no JSON dependency, so we assert the invariant directly. + test "a multibyte unexpected character produces a valid, wire-safe message" do + code = "local x = │" + map = code |> error!() |> Error.to_map(code) + + assert String.valid?(map.message) + assert map.message =~ "U+2502" + assert_all_strings_valid(map) + end + + test "every string field stays valid UTF-8 when the source has malformed bytes" do + code = <<"local x = ", 0xFF>> + map = code |> error!() |> Error.to_map(code) + + assert map.message == "Invalid byte 0xFF" + assert_all_strings_valid(map) + end + end + + defp assert_all_strings_valid(map) do + assert String.valid?(map.message) + assert String.valid?(map.suggestion || "") + + for line <- map.source_context.lines do + assert String.valid?(line.text) + end + end + defp error!(code) do - {:error, [error]} = Parser.parse_structured(code) + {:error, [error | _]} = Parser.parse_structured(code) error end end