From a95d90517df7af19972be041ae65f8dcbb1f3d51 Mon Sep 17 00:00:00 2001 From: x_ror <16686943+x-ror@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:20:58 +0200 Subject: [PATCH 1/2] feat(html): incremental HTML parser over lexbor, with a pause at every script pkg/html binds lexbor (Apache-2.0, pinned commit, built from source into .deps/ by scripts/build-lexbor.sh) and wraps it in an incremental Parser: - feed/finish take network bytes; pump parses until the next parser-blocking script, the end of the available input, or the end of the document; - when pump returns a script, the tree holds exactly what precedes it, as a browser's parser would have it; - write inserts document.write text at the insertion point, before the rest of the input. lexbor has no pause and no document.write and does not need them: we feed it slices that never run past a `` end tag, and a small C shim (pkg/html/shim) wraps lexbor's token callback to learn which script element that end tag closed. `lava html [--scripts] ` parses a file in 16 KiB pieces and prints the serialized document; --scripts reports each script where the parser pauses. Not done: charset sniffing (input is taken as UTF-8), fetching a URL, script execution, a DOM API. No tests: paused for this phase. Co-Authored-By: Claude Opus 5.5 --- NOTICE | 2 + cmd/lava/main.odin | 78 ++++++++++++ pkg/html/lexbor.odin | 52 ++++++++ pkg/html/parser.odin | 227 +++++++++++++++++++++++++++++++++++ pkg/html/shim/larva_html.c | 95 +++++++++++++++ scripts/build-lexbor.sh | 60 +++++++++ scripts/build-native-deps.sh | 3 + 7 files changed, 517 insertions(+) create mode 100644 pkg/html/lexbor.odin create mode 100644 pkg/html/parser.odin create mode 100644 pkg/html/shim/larva_html.c create mode 100755 scripts/build-lexbor.sh diff --git a/NOTICE b/NOTICE index a87ad0d3..0ea3b74b 100644 --- a/NOTICE +++ b/NOTICE @@ -25,4 +25,6 @@ Libraries linked at build time, not distributed here JavaScriptCore (WebKitGTK) LGPL-2.1-or-later and BSD-2-Clause; linked dynamically OpenSSL 3 Apache-2.0 SQLite public domain +lexbor (pinned commit) Apache-2.0; linked statically, built from source by + scripts/build-lexbor.sh (pkg/html) Odin core library zlib License diff --git a/cmd/lava/main.odin b/cmd/lava/main.odin index 8f4ca34b..b451961d 100644 --- a/cmd/lava/main.odin +++ b/cmd/lava/main.odin @@ -2,6 +2,8 @@ package main import "core:fmt" import "core:os" +import "core:strings" +import html "lava:pkg/html" import lava_runtime "lava:pkg/runtime" import eventloop "lava:pkg/runtime/eventloop" @@ -24,6 +26,8 @@ main :: proc() { eval_command(args[2:]) case "run": run_command(args[2:]) + case "html": + html_command(args[2:]) case: fmt.eprintfln("unknown command: %s", args[1]) fmt.eprintln("try `lava --help`") @@ -38,6 +42,8 @@ print_help :: proc() { Usage: lava eval Evaluate JavaScript source lava run Run a JavaScript file + lava html [--scripts] + Parse HTML and print the serialized document lava --version Print version lava --help Print help @@ -115,3 +121,75 @@ print_result :: proc(result: lava_runtime.Result, user_data: rawptr) { lava_runtime.process_write(os.stderr, result.message, "\n") } + +// html_command parses an HTML file (or stdin, `-`) with pkg/html and prints the +// serialized document. The input is fed in 16 KiB pieces, as a network read would +// deliver it, so the incremental path is the one exercised. Scripts are not run: with +// --scripts each parser-blocking script is reported on stderr at the moment the +// parser pauses for it, which is where a page loader would execute it. +html_command :: proc(args: []string) { + list_scripts := false + path := "" + for a in args { + if a == "--scripts" do list_scripts = true + else do path = a + } + if path == "" { + fmt.eprintln("usage: lava html [--scripts] ") + os.exit(2) + } + data: []byte + err: os.Error + if path == "-" do data, err = os.read_entire_file_from_file(os.stdin, context.allocator) + else do data, err = os.read_entire_file(path, context.allocator) + if err != nil { + fmt.eprintfln("lava html: cannot read %s: %v", path, err) + os.exit(1) + } + defer delete(data) + + p: html.Parser + defer html.destroy(&p) + if !html.init(&p) { + fmt.eprintln("lava html: lexbor could not create a document") + os.exit(1) + } + CHUNK :: 16 * 1024 + off, scripts := 0, 0 + loop: for { + script, st := html.pump(&p) + switch st { + case .Script: + scripts += 1 + if list_scripts { + if src, ok := html.attr(script, "src"); ok { + fmt.eprintfln("[script %d] src=%s", scripts, src) + } else { + body := html.text(script) + fmt.eprintfln("[script %d] inline, %d bytes", scripts, len(body)) + delete(body) + } + } + case .Need_Input: + if off >= len(data) { + html.finish(&p) + continue + } + end := min(off + CHUNK, len(data)) + html.feed(&p, data[off:end]) + off = end + case .Done: + break loop + case .Failed: + fmt.eprintln("lava html: lexbor reported a parse failure") + os.exit(1) + } + } + b := strings.builder_make() + defer strings.builder_destroy(&b) + if !html.serialize(&p, &b) { + fmt.eprintln("lava html: serialization failed") + os.exit(1) + } + fmt.println(strings.to_string(b)) +} diff --git a/pkg/html/lexbor.odin b/pkg/html/lexbor.odin new file mode 100644 index 00000000..18b07c44 --- /dev/null +++ b/pkg/html/lexbor.odin @@ -0,0 +1,52 @@ +package larva_html + +import "core:c" + +// Bindings to lexbor (https://github.com/lexbor/lexbor, Apache-2.0) and to the small C +// shim beside it (shim/larva_html.c). Only what pkg/html uses is declared; lexbor's +// structs stay opaque to Odin — anything that needs a field read, a macro or an inline +// function goes through the shim instead, so this file never mirrors a struct layout. +// +// Both archives are built by scripts/build-lexbor.sh into the gitignored .deps/ +// (pinned lexbor commit, sha256-checked). The shim comes first: it references lexbor, +// and a static link resolves left to right. + +when ODIN_OS == .Linux || ODIN_OS == .Darwin { + foreign import lexbor { + "../../.deps/lexbor/lib/liblarva_html_shim.a", + "../../.deps/lexbor/lib/liblexbor_static.a", + } +} else { + // No build of lexbor exists for this target yet (Linux-first). Declared against the + // same names so the package type-checks everywhere; linking it here fails loudly. + foreign import lexbor "system:lexbor" +} + +lxb_status_t :: c.uint +LXB_STATUS_OK :: lxb_status_t(0) + +Document :: distinct rawptr // lxb_html_document_t * +Hook :: distinct rawptr // larva_html_hook * (shim) + +// Node is a DOM node owned by its document; it is valid until the document is +// destroyed (or the node is removed — nothing in pkg/html removes one yet). +Node :: distinct rawptr // lxb_dom_node_t * + +serialize_cb :: #type proc "c" (data: [^]u8, len: c.size_t, ctx: rawptr) -> lxb_status_t + +@(default_calling_convention = "c") +foreign lexbor { + lxb_html_document_create :: proc() -> Document --- + lxb_html_document_destroy :: proc(doc: Document) -> Document --- + lxb_html_document_parse_chunk :: proc(doc: Document, data: [^]u8, len: c.size_t) -> lxb_status_t --- + lxb_html_document_parse_chunk_end :: proc(doc: Document) -> lxb_status_t --- + lxb_html_serialize_deep_cb :: proc(node: Node, cb: serialize_cb, ctx: rawptr) -> lxb_status_t --- + + larva_html_begin :: proc(doc: Document) -> Hook --- + larva_html_take_script :: proc(h: Hook) -> Node --- + larva_html_hook_free :: proc(h: Hook) --- + larva_html_document_node :: proc(doc: Document) -> Node --- + larva_html_text :: proc(node: Node, len: ^c.size_t) -> [^]u8 --- + larva_html_text_free :: proc(node: Node, text: [^]u8) --- + larva_html_attr :: proc(node: Node, name: [^]u8, name_len: c.size_t, len: ^c.size_t) -> [^]u8 --- +} diff --git a/pkg/html/parser.odin b/pkg/html/parser.odin new file mode 100644 index 00000000..b7468623 --- /dev/null +++ b/pkg/html/parser.odin @@ -0,0 +1,227 @@ +package larva_html + +import "base:runtime" +import "core:c" +import "core:strings" + +// Parser is an incremental HTML parser over lexbor that hands control back to the +// caller at every parser-blocking script, the way a browser's parser does. +// +// The caller owns the input and decides when parsing moves on. lexbor itself has no +// pause and no document.write (its tree builder leaves both as TODO at ``, +// html/tree/insertion_mode/text.c), and needs neither: we feed it slices, and a slice +// never runs past the end tag of a script. So when `pump` returns a script, the tree +// holds exactly what precedes that script and nothing after it — what the script +// would see in a browser. Text the script writes (`write`) goes in front of the rest +// of the input, i.e. at the insertion point, and is parsed before it. +// +// p: Parser +// init(&p) +// feed(&p, chunk) // as bytes arrive; finish(&p) at the end +// for { +// script, st := pump(&p) +// switch st { +// case .Script: run it; write(&p, ...) for each document.write +// case .Need_Input: feed(&p, next chunk) or finish(&p) +// case .Done, .Failed: break +// } +// } +// serialize(&p, ...) +// destroy(&p) +// +// Limits, known: +// - Input is taken as UTF-8. Charset sniffing (BOM, , Content-Type) +// is not done yet. +// - A slice is cut after the first `>` that follows a `` in an attribute (``) is legal HTML +// and is cut too early; the script then closes inside the next, larger slice and +// the pause lands late. No real page is known to do this. +// - Script execution and document.write semantics beyond "insert at the insertion +// point" (script nesting level, document.open after load) are the caller's. +Parser :: struct { + doc: Document, + hook: Hook, + input: [dynamic]byte, // unparsed input, from `off` + off: int, + eof: bool, + ended: bool, + failed: bool, + allocator: runtime.Allocator, +} + +Pump_Status :: enum { + Need_Input, // every byte fed so far is parsed; feed more or finish + Script, // parsing stopped right after this script's end tag + Done, // finish was called and the document is complete + Failed, // lexbor reported an error; the tree is whatever was built +} + +// init creates the document and starts chunked parsing. false when lexbor could not +// allocate; the parser is then unusable and destroy is still safe. +init :: proc(p: ^Parser, allocator := context.allocator) -> bool { + p^ = {} + p.allocator = allocator + p.input = make([dynamic]byte, 0, 0, allocator) + p.doc = lxb_html_document_create() + if p.doc == nil do return false + p.hook = larva_html_begin(p.doc) + if p.hook == nil { + p.failed = true + return false + } + return true +} + +destroy :: proc(p: ^Parser) { + if p.doc != nil do lxb_html_document_destroy(p.doc) + if p.hook != nil do larva_html_hook_free(p.hook) + delete(p.input) + p^ = {} +} + +// feed appends network bytes. Nothing is parsed until pump. +feed :: proc(p: ^Parser, data: []byte) { + append(&p.input, ..data) +} + +// finish marks the end of the input; the next pump that runs out of input completes +// the document. +finish :: proc(p: ^Parser) { + p.eof = true +} + +// write inserts `text` at the insertion point — right after the script pump returned, +// before the rest of the input — as document.write does. Meant to be called while +// paused on a script; scripts it contains pause in turn. +write :: proc(p: ^Parser, text: string) { + inject_at(&p.input, p.off, ..transmute([]byte)text) +} + +// pump parses until the next parser-blocking script, the end of the available input, +// or the end of the document. +pump :: proc(p: ^Parser) -> (script: Node, status: Pump_Status) { + if p.failed do return nil, .Failed + for { + rest := p.input[p.off:] + if len(rest) == 0 { + if p.ended do return nil, .Done + if p.eof { + p.ended = true + if lxb_html_document_parse_chunk_end(p.doc) != LXB_STATUS_OK { + p.failed = true + return nil, .Failed + } + return nil, .Done + } + compact(p) + return nil, .Need_Input + } + n := next_cut(rest, p.eof) + if n == 0 { + compact(p) + return nil, .Need_Input + } + if lxb_html_document_parse_chunk(p.doc, raw_data(rest), c.size_t(n)) != LXB_STATUS_OK { + p.failed = true + return nil, .Failed + } + p.off += n + if s := larva_html_take_script(p.hook); s != nil do return s, .Script + } +} + +// document_node is the Document node, for serialization or a walk. +document_node :: proc(p: ^Parser) -> Node { + return larva_html_document_node(p.doc) +} + +// serialize appends the document's markup — the doctype, then the html element — to +// `out`. Comparing it with Chrome's output is the oracle's job (#2); not done yet. +serialize :: proc(p: ^Parser, out: ^strings.Builder) -> bool { + cb :: proc "c" (data: [^]u8, len: c.size_t, ctx: rawptr) -> lxb_status_t { + context = runtime.default_context() + b := cast(^strings.Builder)ctx + strings.write_bytes(b, data[:len]) + return LXB_STATUS_OK + } + return lxb_html_serialize_deep_cb(document_node(p), cb, out) == LXB_STATUS_OK +} + +// text returns `node`'s textContent, copied into `allocator`. +text :: proc(node: Node, allocator := context.allocator) -> string { + n: c.size_t + t := larva_html_text(node, &n) + if t == nil do return "" + defer larva_html_text_free(node, t) + return strings.clone(string(t[:n]), allocator) +} + +// attr returns attribute `name` of element `node`. The string points into the +// document; copy it to keep it past the next change to that element. +attr :: proc(node: Node, name: string) -> (value: string, ok: bool) { + n: c.size_t + v := larva_html_attr(node, raw_data(name), c.size_t(len(name)), &n) + if v == nil do return "", false + return string(v[:n]), true +} + +@(private = "file") +compact :: proc(p: ^Parser) { + if p.off == 0 do return + remove_range(&p.input, 0, p.off) + p.off = 0 +} + +@(private = "file") +SCRIPT_END :: "` after a ` int { + if i := index_ci(rest, SCRIPT_END); i >= 0 { + for j in i + len(SCRIPT_END) ..< len(rest) { + if rest[j] == '>' do return j + 1 + } + return eof ? len(rest) : i + } + if eof do return len(rest) + return len(rest) - partial_tail(rest) +} + +// index_ci finds `needle` (lower-case ASCII) in `hay`, ASCII case-insensitively. +@(private = "file") +index_ci :: proc(hay: []byte, needle: string) -> int { + if len(hay) < len(needle) do return -1 + outer: for i in 0 ..= len(hay) - len(needle) { + if hay[i] != '<' do continue + for k in 1 ..< len(needle) { + if hay[i + k] | 0x20 != needle[k] && hay[i + k] != needle[k] do continue outer + } + return i + } + return -1 +} + +// partial_tail is the length of the longest suffix of `b` that is a proper prefix of +// SCRIPT_END, case-insensitively. +@(private = "file") +partial_tail :: proc(b: []byte) -> int { + needle := SCRIPT_END + for n := min(len(needle) - 1, len(b)); n > 0; n -= 1 { + tail := b[len(b) - n:] + if tail[0] != '<' do continue + match := true + for k in 1 ..< n { + if tail[k] | 0x20 != needle[k] && tail[k] != needle[k] { + match = false + break + } + } + if match do return n + } + return 0 +} diff --git a/pkg/html/shim/larva_html.c b/pkg/html/shim/larva_html.c new file mode 100644 index 00000000..59f3de05 --- /dev/null +++ b/pkg/html/shim/larva_html.c @@ -0,0 +1,95 @@ +// The C side of pkg/html: the few lexbor calls that are inline functions, macros or +// struct reads, which Odin cannot bind directly, plus the one hook the incremental +// parser needs (see larva_html_begin). Everything else is bound from Odin +// (pkg/html/lexbor.odin). Built by scripts/build-lexbor.sh. + +#include +#include +#include + +// One per document being parsed. The tree builder installs its own token callback on +// the tokenizer when chunk parsing begins; we wrap it so we see every token first. +typedef struct { + lxb_html_tokenizer_token_f orig; + void *orig_ctx; + lxb_html_tree_t *tree; + lxb_dom_node_t *script; // the ` that is not +// closing a script element (inside a comment, in plain text) never reaches here as an +// end tag with that current node, so it is ignored. +static lxb_html_token_t * +larva_hook_cb(lxb_html_tokenizer_t *tkz, lxb_html_token_t *token, void *ctx) +{ + larva_html_hook *h = ctx; + if (token->tag_id == LXB_TAG_SCRIPT && (token->type & LXB_HTML_TOKEN_TYPE_CLOSE)) { + lxb_dom_node_t *cur = lxb_html_tree_current_node(h->tree); + if (cur != NULL && cur->local_name == LXB_TAG_SCRIPT) { + h->script = cur; + } + } + return h->orig(tkz, token, h->orig_ctx); +} + +// Starts chunked parsing of `doc` and installs the hook. NULL on failure; free with +// larva_html_hook_free after the document's parse has ended. +larva_html_hook * +larva_html_begin(lxb_html_document_t *doc) +{ + if (lxb_html_document_parse_chunk_begin(doc) != LXB_STATUS_OK) return NULL; + lxb_html_parser_t *parser = doc->dom_document.parser; + if (parser == NULL || parser->tkz == NULL || parser->tree == NULL) return NULL; + larva_html_hook *h = calloc(1, sizeof(*h)); + if (h == NULL) return NULL; + h->orig = parser->tkz->callback_token_done; + h->orig_ctx = parser->tkz->callback_token_ctx; + h->tree = parser->tree; + lxb_html_tokenizer_callback_token_done_set(parser->tkz, larva_hook_cb, h); + return h; +} + +// Returns the script element closed since the last call, and clears it. +lxb_dom_node_t * +larva_html_take_script(larva_html_hook *h) +{ + lxb_dom_node_t *s = h->script; + h->script = NULL; + return s; +} + +void +larva_html_hook_free(larva_html_hook *h) +{ + free(h); +} + +lxb_dom_node_t * +larva_html_document_node(lxb_html_document_t *doc) +{ + return lxb_dom_interface_node(doc); +} + +// textContent of `node`. The bytes belong to the document; release them with +// larva_html_text_free. +lxb_char_t * +larva_html_text(lxb_dom_node_t *node, size_t *len) +{ + return lxb_dom_node_text_content(node, len); +} + +void +larva_html_text_free(lxb_dom_node_t *node, lxb_char_t *text) +{ + if (text != NULL) lxb_dom_document_destroy_text(node->owner_document, text); +} + +// The value of attribute `name` on element `node`, or NULL when absent. Owned by the +// element; valid until the element changes. +const lxb_char_t * +larva_html_attr(lxb_dom_node_t *node, const lxb_char_t *name, size_t name_len, size_t *len) +{ + return lxb_dom_element_get_attribute(lxb_dom_interface_element(node), name, name_len, len); +} diff --git a/scripts/build-lexbor.sh b/scripts/build-lexbor.sh new file mode 100755 index 00000000..03f2799f --- /dev/null +++ b/scripts/build-lexbor.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env sh +set -eu + +# Builds lexbor (https://github.com/lexbor/lexbor, Apache-2.0) as a static library for +# pkg/html, and the small C shim next to it (pkg/html/shim). Pinned to one commit and +# checked against its tarball's sha256, so a build never picks up an unreviewed +# lexbor. Idempotent: does nothing when the archive for this pin already exists. +# +# Why a source build and not the distro package: liblexbor-dev is not installed on the +# dev machines or the CI image, and a pinned static archive keeps the parser identical +# everywhere. Why not vendored into the tree: lexbor's source is ~22 MB. +# +# Output (both under the gitignored .deps/): +# .deps/lexbor/lib/liblexbor_static.a +# .deps/lexbor/include/lexbor/... (headers, for the shim) +# .deps/lexbor/lib/liblarva_html_shim.a + +ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +LEXBOR_COMMIT=e609d9446f7adf06c67b89dcdc4a7c39430f0bd8 +LEXBOR_SHA256=c69fa3f8ea24cc1a9ea8ea295ff2d3c7a2e8347f211d05f8e0bea7e29fba3b33 +CC=${CC:-cc} + +OUT="$ROOT_DIR/.deps/lexbor" +STAMP="$OUT/.commit" +LIB="$OUT/lib/liblexbor_static.a" +SHIM_SRC="$ROOT_DIR/pkg/html/shim/larva_html.c" +SHIM_LIB="$OUT/lib/liblarva_html_shim.a" + +if [ ! -f "$LIB" ] || [ "$(cat "$STAMP" 2>/dev/null)" != "$LEXBOR_COMMIT" ]; then + command -v cmake >/dev/null || { + echo "build-lexbor: cmake is required to build lexbor" >&2 + exit 1 + } + work="$ROOT_DIR/.deps/lexbor-src" + rm -rf "$work" "$OUT" + mkdir -p "$work" "$OUT" + tarball="$work/lexbor.tar.gz" + curl -fsSL -o "$tarball" "https://github.com/lexbor/lexbor/archive/$LEXBOR_COMMIT.tar.gz" + echo "$LEXBOR_SHA256 $tarball" | sha256sum -c - >/dev/null || { + echo "build-lexbor: sha256 mismatch for lexbor $LEXBOR_COMMIT" >&2 + exit 1 + } + tar xzf "$tarball" -C "$work" --strip-components=1 + cmake -S "$work" -B "$work/build" -DCMAKE_BUILD_TYPE=Release \ + -DLEXBOR_BUILD_SHARED=OFF -DLEXBOR_BUILD_STATIC=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX="$OUT" >/dev/null + cmake --build "$work/build" -j "$(nproc 2>/dev/null || echo 4)" >/dev/null + cmake --install "$work/build" >/dev/null + # Some lexbor versions install into lib64. + [ -f "$LIB" ] || { mkdir -p "$OUT/lib" && cp "$work/build/liblexbor_static.a" "$LIB"; } + rm -rf "$work" + echo "$LEXBOR_COMMIT" >"$STAMP" + printf 'built %s\n' ".deps/lexbor (lexbor ${LEXBOR_COMMIT%"${LEXBOR_COMMIT#???????}"})" +fi + +if [ ! -f "$SHIM_LIB" ] || [ "$SHIM_SRC" -nt "$SHIM_LIB" ] || [ "$LIB" -nt "$SHIM_LIB" ]; then + "$CC" -O2 -fPIC -I "$OUT/include" -c "$SHIM_SRC" -o "$OUT/lib/larva_html_shim.o" + ar rcs "$SHIM_LIB" "$OUT/lib/larva_html_shim.o" + printf 'built %s\n' ".deps/lexbor/lib/liblarva_html_shim.a" +fi diff --git a/scripts/build-native-deps.sh b/scripts/build-native-deps.sh index b77c7547..3730f52c 100755 --- a/scripts/build-native-deps.sh +++ b/scripts/build-native-deps.sh @@ -32,3 +32,6 @@ build_archive() { } build_archive "$ROOT_DIR/pkg/runtime/picohttpparser" picohttpparser + +# lexbor + the pkg/html shim (pinned source build into .deps/; see the script). +"$ROOT_DIR/scripts/build-lexbor.sh" From 5b41d0a8a60d4ff61182b8c932f1c6c9314e64db Mon Sep 17 00:00:00 2001 From: x_ror <16686943+x-ror@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:35:07 +0200 Subject: [PATCH 2/2] =?UTF-8?q?fix(html):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20lost=20scripts,=20write=20order,=20quadratic=20input;=20hide?= =?UTF-8?q?=20lexbor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the quality and native-safety reviews of the first version: - A script was silently lost when an end tag had a quoted `>` in an attribute (``): the slice ended early, the next one closed two scripts, and the hook kept only the last. The cut now skips quoted attribute values as the tokenizer does, and the shim counts closes: two in one slice is a failure, never a dropped script. - Consecutive writes came out reversed (each was inserted at the same point). Written text now has its own buffer and an insertion cursor that advances. - One unterminated ` --- CLAUDE.md | 1 + Makefile | 2 +- README.md | 3 + cmd/lava/main.odin | 112 ++++++------- docs/ARCHITECTURE.md | 28 ++++ pkg/html/cut.odin | 183 ++++++++++++++++++++++ pkg/html/lexbor.odin | 67 ++++---- pkg/html/lexbor_other.odin | 31 ++++ pkg/html/node.odin | 29 ++++ pkg/html/parser.odin | 311 ++++++++++++++++++------------------- pkg/html/shim/larva_html.c | 10 +- 11 files changed, 527 insertions(+), 250 deletions(-) create mode 100644 pkg/html/cut.odin create mode 100644 pkg/html/lexbor_other.odin create mode 100644 pkg/html/node.odin diff --git a/CLAUDE.md b/CLAUDE.md index cd689984..b69ea74e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,6 +14,7 @@ Layering (never blur it): | Embedded spec surface | `pkg/runtime/js/**.js` | `Buffer`, `fetch`, streams, `URL`, `http`… | | Engine FFI / private ABI | `pkg/jsc` | JSC C API, string/view/host ABI, probes | | Event loop | `pkg/runtime/eventloop` | phases, timers, io_uring/epoll, threadpool | +| HTML parser | `pkg/html` | lexbor FFI + C shim, incremental `Parser` | --- diff --git a/Makefile b/Makefile index 5a412992..9a212f1b 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ help: @printf '%s\n' '' @printf '%s\n' 'Build / run' @printf '%s\n' ' make build Build the lava CLI' - @printf '%s\n' ' make native-deps Build vendored C deps (picohttpparser)' + @printf '%s\n' ' make native-deps Build vendored C deps (picohttpparser, lexbor)' @printf '%s\n' ' make bootstrap-windows-deps Fetch ignored Windows native deps into .deps/' @printf '%s\n' ' make build-sqlite-windows Build build/sqlite3.lib via MSVC' @printf '%s\n' ' make run FILE=app.js Run a JavaScript file through lava' diff --git a/README.md b/README.md index 6ef6b16c..29d1d1ad 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ for how the runtime pieces fit together. - SQLite development headers and libraries (for `node:sqlite`) - OpenSSL development headers and libraries (for `fetch` HTTPS/TLS). The roadmap replaces it with BoringSSL (stage 2); until that lands, OpenSSL is what links. +- `cmake`, `curl` and network access on the first build (Linux): `pkg/html` links + [lexbor](https://github.com/lexbor/lexbor), which `scripts/build-lexbor.sh` downloads + at a pinned commit, checks against its sha256 and builds into `.deps/`. On Linux, the JSC package is exposed through `pkg-config` as `javascriptcoregtk-6.0`, SQLite as `sqlite3`, and OpenSSL as `openssl` diff --git a/cmd/lava/main.odin b/cmd/lava/main.odin index b451961d..2b19d7ed 100644 --- a/cmd/lava/main.odin +++ b/cmd/lava/main.odin @@ -2,7 +2,7 @@ package main import "core:fmt" import "core:os" -import "core:strings" +import "core:strconv" import html "lava:pkg/html" import lava_runtime "lava:pkg/runtime" import eventloop "lava:pkg/runtime/eventloop" @@ -42,7 +42,7 @@ print_help :: proc() { Usage: lava eval Evaluate JavaScript source lava run Run a JavaScript file - lava html [--scripts] + lava html [--scripts] [--chunk N] Parse HTML and print the serialized document lava --version Print version lava --help Print help @@ -122,22 +122,41 @@ print_result :: proc(result: lava_runtime.Result, user_data: rawptr) { lava_runtime.process_write(os.stderr, result.message, "\n") } -// html_command parses an HTML file (or stdin, `-`) with pkg/html and prints the -// serialized document. The input is fed in 16 KiB pieces, as a network read would -// deliver it, so the incremental path is the one exercised. Scripts are not run: with -// --scripts each parser-blocking script is reported on stderr at the moment the -// parser pauses for it, which is where a page loader would execute it. +// html_command parses an HTML file (or stdin, `-`) and prints the serialized +// document. No script runs: --scripts reports each parser-blocking script on stderr +// at the moment the parser pauses for it, where a page loader would execute it. +// --chunk N feeds the input N bytes at a time, as network reads would, to exercise +// the incremental path; by default it is fed at once. html_command :: proc(args: []string) { - list_scripts := false - path := "" - for a in args { - if a == "--scripts" do list_scripts = true - else do path = a - } - if path == "" { - fmt.eprintln("usage: lava html [--scripts] ") + usage :: proc() -> ! { + fmt.eprintln("usage: lava html [--scripts] [--chunk N] ") os.exit(2) } + Html_Options :: struct { + list_scripts: bool, + scripts: int, + } + opts: Html_Options + chunk := 0 + path := "" + for i := 0; i < len(args); i += 1 { + switch a := args[i]; { + case a == "--scripts": + opts.list_scripts = true + case a == "--chunk": + i += 1 + if i == len(args) do usage() + n, ok := strconv.parse_int(args[i]) + if !ok || n <= 0 do usage() + chunk = n + case len(a) > 1 && a[0] == '-', path != "": + usage() + case: + path = a + } + } + if path == "" do usage() + data: []byte err: os.Error if path == "-" do data, err = os.read_entire_file_from_file(os.stdin, context.allocator) @@ -146,50 +165,31 @@ html_command :: proc(args: []string) { fmt.eprintfln("lava html: cannot read %s: %v", path, err) os.exit(1) } - defer delete(data) - p: html.Parser - defer html.destroy(&p) - if !html.init(&p) { - fmt.eprintln("lava html: lexbor could not create a document") - os.exit(1) - } - CHUNK :: 16 * 1024 - off, scripts := 0, 0 - loop: for { - script, st := html.pump(&p) - switch st { - case .Script: - scripts += 1 - if list_scripts { - if src, ok := html.attr(script, "src"); ok { - fmt.eprintfln("[script %d] src=%s", scripts, src) - } else { - body := html.text(script) - fmt.eprintfln("[script %d] inline, %d bytes", scripts, len(body)) - delete(body) - } - } - case .Need_Input: - if off >= len(data) { - html.finish(&p) - continue - } - end := min(off + CHUNK, len(data)) - html.feed(&p, data[off:end]) - off = end - case .Done: - break loop - case .Failed: - fmt.eprintln("lava html: lexbor reported a parse failure") - os.exit(1) + report :: proc(p: ^html.Parser, script: html.Node, user_data: rawptr) { + o := cast(^Html_Options)user_data + o.scripts += 1 + if !o.list_scripts do return + // An SVG ` inside a comment or a JS string + does not pause. More than one script closing in a slice is a failure, never a + dropped script; +- `write` puts `document.write` text at the insertion point, after earlier writes and + before the rest of the input; scripts in it pause in turn. + +The cut scan resumes where it stopped and the consumed input is compacted only once +it is half the buffer, so feeding a page in many small reads stays linear. One +unterminated `` drives it from the CLI. + --- ## 4. Cross-cutting concerns diff --git a/pkg/html/cut.odin b/pkg/html/cut.odin new file mode 100644 index 00000000..6d54ea93 --- /dev/null +++ b/pkg/html/cut.odin @@ -0,0 +1,183 @@ +package larva_html + +import "core:bytes" + +// Where to cut the input before handing it to lexbor. +// +// The parser pauses at a script by never letting a slice run past a script's end tag: +// each slice ends right after the `>` of the next ``. A real end tag is a few bytes long (attributes on an end +// tag are a parse error and carry nothing); past this, the candidate is fed through as +// not being one. Without the bound, one stray `` after it would +// hold back the rest of the page. If it was a real end tag after all, the script then +// closes inside a larger slice and the hook counts it (see pump), so nothing is lost +// silently. +@(private = "file") +END_TAG_MAX :: 4096 + +// Source is bytes waiting to be parsed: the network input, or text from +// document.write. Everything before `off` has been handed to lexbor. +@(private) +Source :: struct { + buf: [dynamic]byte, + off: int, + // Resumable scan state, all absolute indices into buf. + scan: int, // the search for the next candidate resumes here + tag: int, // start of a candidate whose `>` is not found yet; -1 when none + tag_pos: int, // inside that candidate, the scan resumes here + quote: byte, // inside that candidate, the open quote of an attribute value + after_eq: bool, // inside that candidate, an attribute value may start next +} + +@(private) +source_init :: proc(s: ^Source, allocator := context.allocator) { + s^ = { + buf = make([dynamic]byte, 0, 0, allocator), + tag = -1, + } +} + +@(private) +source_pending :: proc(s: ^Source) -> int { + return len(s.buf) - s.off +} + +// source_reset empties `s`, keeping its storage. +@(private) +source_reset :: proc(s: ^Source) { + clear(&s.buf) + s.off, s.scan, s.tag_pos, s.quote, s.after_eq = 0, 0, 0, 0, false + s.tag = -1 +} + +// source_compact drops what lexbor has consumed once it is at least half the buffer, +// so the copy is amortized instead of paid on every piece of input. +@(private) +source_compact :: proc(s: ^Source) { + if s.off == 0 || s.off * 2 < len(s.buf) do return + n := s.off + remove_range(&s.buf, 0, n) + s.off = 0 + s.scan = max(s.scan - n, 0) + if s.tag >= 0 { + s.tag -= n + s.tag_pos -= n + } +} + +// next_cut returns how many bytes after `s.off` can be parsed now; 0 means wait for +// more input. `eof` says no more bytes will follow these in this source. +@(private) +next_cut :: proc(s: ^Source, eof: bool) -> int { + end := len(s.buf) + if s.tag < 0 { + s.scan = max(s.scan, s.off) + search: for s.scan < end { + k := bytes.index_byte(s.buf[s.scan:], '<') + if k < 0 { + s.scan = end + break + } + at := s.scan + k + switch match_script_end(s.buf[at:]) { + case .No: + s.scan = at + 1 + continue + case .Need_More: + if eof { + s.scan = end + return end - s.off + } + s.scan = at + return at - s.off + case .Yes: + s.tag = at + s.tag_pos = at + len(SCRIPT_END_TAG) + s.quote, s.after_eq = 0, false + break search + } + } + if s.tag < 0 do return end - s.off + } + + // Inside a candidate: find its `>`, skipping quoted attribute values, as the + // tokenizer does in the attribute states of an end tag. + for ; s.tag_pos < end; s.tag_pos += 1 { + b := s.buf[s.tag_pos] + if s.quote != 0 { + if b == s.quote do s.quote = 0 + continue + } + switch { + case b == '>': + cut := s.tag_pos + 1 + s.tag = -1 + s.scan = cut + return cut - s.off + case (b == '"' || b == '\'') && s.after_eq: + s.quote = b + s.after_eq = false + case b == '=': + s.after_eq = true + case is_html_space(b): + // keeps after_eq: `a = "v"` is one attribute + case: + s.after_eq = false + } + } + if eof || s.tag_pos - s.tag > END_TAG_MAX { + s.tag = -1 + s.scan = end + return end - s.off + } + return s.tag - s.off +} + +@(private = "file") +Match :: enum { + No, + Yes, + Need_More, // `b` ends before it can be told +} + +// match_script_end says whether `b` starts with an end tag named "script": the +// characters of SCRIPT_END_TAG in any ASCII case, then whitespace, `/` or `>`. Anything +// else after the name (``) is a different tag. +@(private = "file") +match_script_end :: proc(b: []byte) -> Match { + needle := SCRIPT_END_TAG + for i in 0 ..< len(needle) { + if i >= len(b) do return .Need_More + if ascii_lower(b[i]) != needle[i] do return .No + } + if len(b) == len(needle) do return .Need_More + next := b[len(needle)] + return .Yes if next == '/' || next == '>' || is_html_space(next) else .No +} + +// ascii_lower folds A–Z only. HTML tag names compare ASCII case-insensitively; a +// Unicode fold (core:strings.equal_fold) would also match U+017F to 's', which the +// tokenizer does not. +@(private = "file") +ascii_lower :: proc(b: byte) -> byte { + return b + ('a' - 'A') if 'A' <= b && b <= 'Z' else b +} + +// is_html_space is the HTML spec's ASCII whitespace: tab, LF, FF, CR, space. +@(private = "file") +is_html_space :: proc(b: byte) -> bool { + return b == '\t' || b == '\n' || b == '\f' || b == '\r' || b == ' ' +} diff --git a/pkg/html/lexbor.odin b/pkg/html/lexbor.odin index 18b07c44..c0e53e1d 100644 --- a/pkg/html/lexbor.odin +++ b/pkg/html/lexbor.odin @@ -1,51 +1,50 @@ +#+build linux +#+private package larva_html import "core:c" -// Bindings to lexbor (https://github.com/lexbor/lexbor, Apache-2.0) and to the small C -// shim beside it (shim/larva_html.c). Only what pkg/html uses is declared; lexbor's -// structs stay opaque to Odin — anything that needs a field read, a macro or an inline -// function goes through the shim instead, so this file never mirrors a struct layout. +// Bindings to lexbor (https://github.com/lexbor/lexbor, Apache-2.0) and to the C shim +// beside it (shim/larva_html.c). Private to the package: callers see Parser and Node +// only, so lexbor stays replaceable behind them (docs/LARVA.md, decision 4). // -// Both archives are built by scripts/build-lexbor.sh into the gitignored .deps/ -// (pinned lexbor commit, sha256-checked). The shim comes first: it references lexbor, -// and a static link resolves left to right. - -when ODIN_OS == .Linux || ODIN_OS == .Darwin { - foreign import lexbor { - "../../.deps/lexbor/lib/liblarva_html_shim.a", - "../../.deps/lexbor/lib/liblexbor_static.a", - } -} else { - // No build of lexbor exists for this target yet (Linux-first). Declared against the - // same names so the package type-checks everywhere; linking it here fails loudly. - foreign import lexbor "system:lexbor" +// Hand-written, not generated with odin-c-bindgen: the surface is a dozen procs, and +// everything lexbor exposes as an inline function, a macro or a struct field goes +// through the shim, which a generator could not bind anyway. lexbor's structs stay +// opaque here. Revisit if the declared surface grows to dozens of procs. +// +// Linux only for now: scripts/build-lexbor.sh builds for the host, and only Linux is +// built and run (CLAUDE.md, "Linux-first"). lexbor_other.odin answers elsewhere. + +// The paths are relative to this file and must match OUT/LIB/SHIM_LIB in +// scripts/build-lexbor.sh; `foreign import` takes only literals. The shim comes first: +// it references lexbor, and a static link resolves left to right. +foreign import lexbor { + "../../.deps/lexbor/lib/liblarva_html_shim.a", + "../../.deps/lexbor/lib/liblexbor_static.a", } +// lexbor/core/base.h: `typedef unsigned int lxb_status_t`, LXB_STATUS_OK = 0x0000. lxb_status_t :: c.uint LXB_STATUS_OK :: lxb_status_t(0) -Document :: distinct rawptr // lxb_html_document_t * -Hook :: distinct rawptr // larva_html_hook * (shim) - -// Node is a DOM node owned by its document; it is valid until the document is -// destroyed (or the node is removed — nothing in pkg/html removes one yet). -Node :: distinct rawptr // lxb_dom_node_t * +lxb_document :: distinct rawptr // lxb_html_document_t * +larva_hook :: distinct rawptr // larva_html_hook * (shim) -serialize_cb :: #type proc "c" (data: [^]u8, len: c.size_t, ctx: rawptr) -> lxb_status_t +lxb_serialize_cb :: #type proc "c" (data: [^]u8, len: c.size_t, ctx: rawptr) -> lxb_status_t @(default_calling_convention = "c") foreign lexbor { - lxb_html_document_create :: proc() -> Document --- - lxb_html_document_destroy :: proc(doc: Document) -> Document --- - lxb_html_document_parse_chunk :: proc(doc: Document, data: [^]u8, len: c.size_t) -> lxb_status_t --- - lxb_html_document_parse_chunk_end :: proc(doc: Document) -> lxb_status_t --- - lxb_html_serialize_deep_cb :: proc(node: Node, cb: serialize_cb, ctx: rawptr) -> lxb_status_t --- - - larva_html_begin :: proc(doc: Document) -> Hook --- - larva_html_take_script :: proc(h: Hook) -> Node --- - larva_html_hook_free :: proc(h: Hook) --- - larva_html_document_node :: proc(doc: Document) -> Node --- + lxb_html_document_create :: proc() -> lxb_document --- + lxb_html_document_destroy :: proc(doc: lxb_document) -> lxb_document --- + lxb_html_document_parse_chunk :: proc(doc: lxb_document, data: [^]u8, len: c.size_t) -> lxb_status_t --- + lxb_html_document_parse_chunk_end :: proc(doc: lxb_document) -> lxb_status_t --- + lxb_html_serialize_deep_cb :: proc(node: Node, cb: lxb_serialize_cb, ctx: rawptr) -> lxb_status_t --- + + larva_html_begin :: proc(doc: lxb_document) -> larva_hook --- + larva_html_take_script :: proc(h: larva_hook, closed: ^c.size_t) -> Node --- + larva_html_hook_free :: proc(h: larva_hook) --- + larva_html_document_node :: proc(doc: lxb_document) -> Node --- larva_html_text :: proc(node: Node, len: ^c.size_t) -> [^]u8 --- larva_html_text_free :: proc(node: Node, text: [^]u8) --- larva_html_attr :: proc(node: Node, name: [^]u8, name_len: c.size_t, len: ^c.size_t) -> [^]u8 --- diff --git a/pkg/html/lexbor_other.odin b/pkg/html/lexbor_other.odin new file mode 100644 index 00000000..7d2d6b35 --- /dev/null +++ b/pkg/html/lexbor_other.odin @@ -0,0 +1,31 @@ +#+build !linux +#+private +package larva_html + +import "core:c" + +// No lexbor build exists off Linux yet (lexbor.odin). These stand in for the bindings +// with the same names, so the package compiles on every target and `init` reports the +// truth there: lxb_html_document_create returns nil, and init fails. + +lxb_status_t :: c.uint +LXB_STATUS_OK :: lxb_status_t(0) +LXB_STATUS_ERROR :: lxb_status_t(1) + +lxb_document :: distinct rawptr +larva_hook :: distinct rawptr + +lxb_serialize_cb :: #type proc "c" (data: [^]u8, len: c.size_t, ctx: rawptr) -> lxb_status_t + +lxb_html_document_create :: proc() -> lxb_document {return nil} +lxb_html_document_destroy :: proc(doc: lxb_document) -> lxb_document {return nil} +lxb_html_document_parse_chunk :: proc(doc: lxb_document, data: [^]u8, len: c.size_t) -> lxb_status_t {return LXB_STATUS_ERROR} +lxb_html_document_parse_chunk_end :: proc(doc: lxb_document) -> lxb_status_t {return LXB_STATUS_ERROR} +lxb_html_serialize_deep_cb :: proc(node: Node, cb: lxb_serialize_cb, ctx: rawptr) -> lxb_status_t {return LXB_STATUS_ERROR} +larva_html_begin :: proc(doc: lxb_document) -> larva_hook {return nil} +larva_html_take_script :: proc(h: larva_hook, closed: ^c.size_t) -> Node {closed^ = 0; return nil} +larva_html_hook_free :: proc(h: larva_hook) {} +larva_html_document_node :: proc(doc: lxb_document) -> Node {return nil} +larva_html_text :: proc(node: Node, len: ^c.size_t) -> [^]u8 {return nil} +larva_html_text_free :: proc(node: Node, text: [^]u8) {} +larva_html_attr :: proc(node: Node, name: [^]u8, name_len: c.size_t, len: ^c.size_t) -> [^]u8 {return nil} diff --git a/pkg/html/node.odin b/pkg/html/node.odin new file mode 100644 index 00000000..86139e02 --- /dev/null +++ b/pkg/html/node.odin @@ -0,0 +1,29 @@ +package larva_html + +import "core:c" +import "core:strings" + +// Node is a DOM node owned by its Parser's document. It stays valid until destroy: +// lexbor detaches nodes from the tree but frees them only with the document. +Node :: distinct rawptr + +// text returns `node`'s textContent, copied into `allocator`; "" for none. +text :: proc(node: Node, allocator := context.allocator) -> string { + n: c.size_t + t := larva_html_text(node, &n) + if t == nil do return "" + defer larva_html_text_free(node, t) + return strings.clone(string(t[:n]), allocator) +} + +// attr returns the value of attribute `name` on element `node`, copied into +// `allocator`. +// +// Returns: +// ok false when the attribute is absent (an empty value is ok true). +attr :: proc(node: Node, name: string, allocator := context.allocator) -> (value: string, ok: bool) { + n: c.size_t + v := larva_html_attr(node, raw_data(name), c.size_t(len(name)), &n) + if v == nil do return "", false + return strings.clone(string(v[:n]), allocator), true +} diff --git a/pkg/html/parser.odin b/pkg/html/parser.odin index b7468623..22d74471 100644 --- a/pkg/html/parser.odin +++ b/pkg/html/parser.odin @@ -4,224 +4,221 @@ import "base:runtime" import "core:c" import "core:strings" -// Parser is an incremental HTML parser over lexbor that hands control back to the -// caller at every parser-blocking script, the way a browser's parser does. +// Parser is an incremental HTML parser that hands control back to the caller at every +// parser-blocking script, the way a browser's parser does (#10). // -// The caller owns the input and decides when parsing moves on. lexbor itself has no -// pause and no document.write (its tree builder leaves both as TODO at ``, -// html/tree/insertion_mode/text.c), and needs neither: we feed it slices, and a slice -// never runs past the end tag of a script. So when `pump` returns a script, the tree -// holds exactly what precedes that script and nothing after it — what the script -// would see in a browser. Text the script writes (`write`) goes in front of the rest -// of the input, i.e. at the insertion point, and is parsed before it. +// The caller owns the input and decides when parsing moves on. lexbor has no pause and +// no document.write (its tree builder leaves both as TODO at ``, +// html/tree/insertion_mode/text.c) and needs neither: we feed it slices that never run +// past a script's end tag (cut.odin). So when `pump` returns a script, the tree holds +// exactly what precedes it — what the script would see in a browser. Text the script +// writes (`write`) is parsed next, before the rest of the input. // // p: Parser -// init(&p) -// feed(&p, chunk) // as bytes arrive; finish(&p) at the end +// init(&p) or_return +// defer destroy(&p) +// feed(&p, bytes) // as they arrive; finish(&p) after the last // for { -// script, st := pump(&p) -// switch st { +// script, status := pump(&p) +// switch status { // case .Script: run it; write(&p, ...) for each document.write -// case .Need_Input: feed(&p, next chunk) or finish(&p) -// case .Done, .Failed: break +// case .Need_Input: feed(&p, ...) or finish(&p) +// case .Done: // the document is complete +// case .Failed: // error_message(&p) says why // } // } -// serialize(&p, ...) -// destroy(&p) // -// Limits, known: -// - Input is taken as UTF-8. Charset sniffing (BOM, , Content-Type) -// is not done yet. -// - A slice is cut after the first `>` that follows a `` in an attribute (``) is legal HTML -// and is cut too early; the script then closes inside the next, larger slice and -// the pause lands late. No real page is known to do this. -// - Script execution and document.write semantics beyond "insert at the insertion -// point" (script nesting level, document.open after load) are the caller's. +// parse_all wraps that loop for input already in memory. +// +// Known limits: +// - Input is taken as UTF-8: no charset sniffing (BOM, , Content-Type). +// - Script nesting level and document.open (a write after the parse ended) are not +// modelled; write is refused once the document is done. +// - A ` bool { p^ = {} - p.allocator = allocator - p.input = make([dynamic]byte, 0, 0, allocator) + source_init(&p.input, allocator) + source_init(&p.written, allocator) + when ODIN_OS != .Linux { + p.err = "pkg/html: no lexbor build for this target (Linux only)" + return false + } p.doc = lxb_html_document_create() - if p.doc == nil do return false + if p.doc == nil { + p.err = "pkg/html: lexbor could not create a document" + return false + } p.hook = larva_html_begin(p.doc) if p.hook == nil { - p.failed = true + p.err = "pkg/html: lexbor could not start parsing" return false } return true } +// destroy frees the document and everything in it. Nodes from this parser are +// dangling afterwards. destroy :: proc(p: ^Parser) { + // The document first: the hook is lexbor's tokenizer callback context until then. if p.doc != nil do lxb_html_document_destroy(p.doc) if p.hook != nil do larva_html_hook_free(p.hook) - delete(p.input) + delete(p.input.buf) + delete(p.written.buf) p^ = {} } -// feed appends network bytes. Nothing is parsed until pump. +// feed appends network bytes. They are parsed by pump. feed :: proc(p: ^Parser, data: []byte) { - append(&p.input, ..data) + append(&p.input.buf, ..data) } -// finish marks the end of the input; the next pump that runs out of input completes -// the document. +// finish marks the end of the input; the pump that runs out of input then ends the +// document. finish :: proc(p: ^Parser) { p.eof = true } -// write inserts `text` at the insertion point — right after the script pump returned, -// before the rest of the input — as document.write does. Meant to be called while -// paused on a script; scripts it contains pause in turn. -write :: proc(p: ^Parser, text: string) { - inject_at(&p.input, p.off, ..transmute([]byte)text) +// write inserts `text` at the insertion point: right after the script pump last +// returned, after any text earlier writes put there, before the rest of the input — +// as document.write does. Scripts in `text` pause in turn. +// +// Returns: +// false, and nothing is inserted, once the document is done or the parser failed. +write :: proc(p: ^Parser, text: string) -> bool { + if p.ended || p.err != "" do return false + inject_at(&p.written.buf, p.ins, ..transmute([]byte)text) + p.ins += len(text) + return true } -// pump parses until the next parser-blocking script, the end of the available input, -// or the end of the document. +// pump parses until the next parser-blocking script, the end of what has been fed, or +// the end of the document. pump :: proc(p: ^Parser) -> (script: Node, status: Pump_Status) { - if p.failed do return nil, .Failed + if p.err != "" do return nil, .Failed + if p.ended do return nil, .Done for { - rest := p.input[p.off:] - if len(rest) == 0 { - if p.ended do return nil, .Done - if p.eof { - p.ended = true - if lxb_html_document_parse_chunk_end(p.doc) != LXB_STATUS_OK { - p.failed = true - return nil, .Failed - } - return nil, .Done + src, src_eof := &p.input, p.eof + if source_pending(&p.written) > 0 { + src, src_eof = &p.written, true + } else if len(p.written.buf) > 0 { + source_reset(&p.written) + p.ins = 0 + } + + if source_pending(src) == 0 { + if !p.eof { + source_compact(&p.input) + return nil, .Need_Input } - compact(p) - return nil, .Need_Input + p.ended = true + if lxb_html_document_parse_chunk_end(p.doc) != LXB_STATUS_OK { + return nil, fail(p, "pkg/html: lexbor failed to end the document") + } + return nil, .Done } - n := next_cut(rest, p.eof) + + n := next_cut(src, src_eof) if n == 0 { - compact(p) + source_compact(&p.input) return nil, .Need_Input } - if lxb_html_document_parse_chunk(p.doc, raw_data(rest), c.size_t(n)) != LXB_STATUS_OK { - p.failed = true - return nil, .Failed + if lxb_html_document_parse_chunk(p.doc, raw_data(src.buf[src.off:]), c.size_t(n)) != LXB_STATUS_OK { + return nil, fail(p, "pkg/html: lexbor failed to parse") + } + src.off += n + + closed: c.size_t + s := larva_html_take_script(p.hook, &closed) + if closed > 1 { + // A slice closed two scripts: the first was never handed out. cut.odin keeps + // this from happening; failing beats losing a script quietly. + return nil, fail(p, "pkg/html: two scripts closed in one slice") + } + if s != nil { + p.ins = p.written.off + return s, .Script } - p.off += n - if s := larva_html_take_script(p.hook); s != nil do return s, .Script } } -// document_node is the Document node, for serialization or a walk. -document_node :: proc(p: ^Parser) -> Node { - return larva_html_document_node(p.doc) -} +// Script_Proc runs at each parser-blocking script during parse_all. It may call write. +Script_Proc :: #type proc(p: ^Parser, script: Node, user_data: rawptr) -// serialize appends the document's markup — the doctype, then the html element — to -// `out`. Comparing it with Chrome's output is the oracle's job (#2); not done yet. -serialize :: proc(p: ^Parser, out: ^strings.Builder) -> bool { - cb :: proc "c" (data: [^]u8, len: c.size_t, ctx: rawptr) -> lxb_status_t { - context = runtime.default_context() - b := cast(^strings.Builder)ctx - strings.write_bytes(b, data[:len]) - return LXB_STATUS_OK +// parse_all parses `data` as the whole document, calling `on_script` at each pause. +// `chunk` > 0 feeds the input that many bytes at a time, as network reads would; 0 +// feeds it at once. +parse_all :: proc(p: ^Parser, data: []byte, on_script: Script_Proc = nil, user_data: rawptr = nil, chunk := 0) -> bool { + off := 0 + for { + script, status := pump(p) + switch status { + case .Script: + if on_script != nil do on_script(p, script, user_data) + case .Need_Input: + if off == len(data) { + finish(p) + continue + } + end := len(data) if chunk <= 0 else min(off + chunk, len(data)) + feed(p, data[off:end]) + off = end + case .Done: + return true + case .Failed: + return false + } } - return lxb_html_serialize_deep_cb(document_node(p), cb, out) == LXB_STATUS_OK -} - -// text returns `node`'s textContent, copied into `allocator`. -text :: proc(node: Node, allocator := context.allocator) -> string { - n: c.size_t - t := larva_html_text(node, &n) - if t == nil do return "" - defer larva_html_text_free(node, t) - return strings.clone(string(t[:n]), allocator) -} - -// attr returns attribute `name` of element `node`. The string points into the -// document; copy it to keep it past the next change to that element. -attr :: proc(node: Node, name: string) -> (value: string, ok: bool) { - n: c.size_t - v := larva_html_attr(node, raw_data(name), c.size_t(len(name)), &n) - if v == nil do return "", false - return string(v[:n]), true } -@(private = "file") -compact :: proc(p: ^Parser) { - if p.off == 0 do return - remove_range(&p.input, 0, p.off) - p.off = 0 +// error_message says why the parser failed; "" when it has not. +error_message :: proc(p: ^Parser) -> string { + return p.err } -@(private = "file") -SCRIPT_END :: "` after a ` int { - if i := index_ci(rest, SCRIPT_END); i >= 0 { - for j in i + len(SCRIPT_END) ..< len(rest) { - if rest[j] == '>' do return j + 1 - } - return eof ? len(rest) : i +// serialize returns the document's markup — the doctype, then the html element. +// Comparing it with Chrome's is the oracle's job (#2), not done yet. +serialize :: proc(p: ^Parser, allocator := context.allocator) -> (markup: string, ok: bool) { + collect :: proc "c" (data: [^]u8, len: c.size_t, ctx: rawptr) -> lxb_status_t { + context = runtime.default_context() + strings.write_bytes(cast(^strings.Builder)ctx, data[:len]) + return LXB_STATUS_OK } - if eof do return len(rest) - return len(rest) - partial_tail(rest) -} - -// index_ci finds `needle` (lower-case ASCII) in `hay`, ASCII case-insensitively. -@(private = "file") -index_ci :: proc(hay: []byte, needle: string) -> int { - if len(hay) < len(needle) do return -1 - outer: for i in 0 ..= len(hay) - len(needle) { - if hay[i] != '<' do continue - for k in 1 ..< len(needle) { - if hay[i + k] | 0x20 != needle[k] && hay[i + k] != needle[k] do continue outer - } - return i + b := strings.builder_make(allocator) + if p.doc == nil || lxb_html_serialize_deep_cb(larva_html_document_node(p.doc), collect, &b) != LXB_STATUS_OK { + strings.builder_destroy(&b) + return "", false } - return -1 + return strings.to_string(b), true } -// partial_tail is the length of the longest suffix of `b` that is a proper prefix of -// SCRIPT_END, case-insensitively. @(private = "file") -partial_tail :: proc(b: []byte) -> int { - needle := SCRIPT_END - for n := min(len(needle) - 1, len(b)); n > 0; n -= 1 { - tail := b[len(b) - n:] - if tail[0] != '<' do continue - match := true - for k in 1 ..< n { - if tail[k] | 0x20 != needle[k] && tail[k] != needle[k] { - match = false - break - } - } - if match do return n - } - return 0 +fail :: proc(p: ^Parser, why: string) -> Pump_Status { + p.err = why + return .Failed } diff --git a/pkg/html/shim/larva_html.c b/pkg/html/shim/larva_html.c index 59f3de05..1c7de9a7 100644 --- a/pkg/html/shim/larva_html.c +++ b/pkg/html/shim/larva_html.c @@ -14,6 +14,7 @@ typedef struct { void *orig_ctx; lxb_html_tree_t *tree; lxb_dom_node_t *script; // the