diff --git a/CLAUDE.md b/CLAUDE.md index cd68998..b69ea74 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,6 +14,7 @@ Layering (never blur it): | Embedded spec surface | `pkg/runtime/js/**.js` | `Buffer`, `fetch`, streams, `URL`, `http`… | | Engine FFI / private ABI | `pkg/jsc` | JSC C API, string/view/host ABI, probes | | Event loop | `pkg/runtime/eventloop` | phases, timers, io_uring/epoll, threadpool | +| HTML parser | `pkg/html` | lexbor FFI + C shim, incremental `Parser` | --- diff --git a/Makefile b/Makefile index 5a41299..9a212f1 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ help: @printf '%s\n' '' @printf '%s\n' 'Build / run' @printf '%s\n' ' make build Build the lava CLI' - @printf '%s\n' ' make native-deps Build vendored C deps (picohttpparser)' + @printf '%s\n' ' make native-deps Build vendored C deps (picohttpparser, lexbor)' @printf '%s\n' ' make bootstrap-windows-deps Fetch ignored Windows native deps into .deps/' @printf '%s\n' ' make build-sqlite-windows Build build/sqlite3.lib via MSVC' @printf '%s\n' ' make run FILE=app.js Run a JavaScript file through lava' diff --git a/NOTICE b/NOTICE index a87ad0d..0ea3b74 100644 --- a/NOTICE +++ b/NOTICE @@ -25,4 +25,6 @@ Libraries linked at build time, not distributed here JavaScriptCore (WebKitGTK) LGPL-2.1-or-later and BSD-2-Clause; linked dynamically OpenSSL 3 Apache-2.0 SQLite public domain +lexbor (pinned commit) Apache-2.0; linked statically, built from source by + scripts/build-lexbor.sh (pkg/html) Odin core library zlib License diff --git a/README.md b/README.md index 6ef6b16..29d1d1a 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ for how the runtime pieces fit together. - SQLite development headers and libraries (for `node:sqlite`) - OpenSSL development headers and libraries (for `fetch` HTTPS/TLS). The roadmap replaces it with BoringSSL (stage 2); until that lands, OpenSSL is what links. +- `cmake`, `curl` and network access on the first build (Linux): `pkg/html` links + [lexbor](https://github.com/lexbor/lexbor), which `scripts/build-lexbor.sh` downloads + at a pinned commit, checks against its sha256 and builds into `.deps/`. On Linux, the JSC package is exposed through `pkg-config` as `javascriptcoregtk-6.0`, SQLite as `sqlite3`, and OpenSSL as `openssl` diff --git a/cmd/lava/main.odin b/cmd/lava/main.odin index 8f4ca34..2b19d7e 100644 --- a/cmd/lava/main.odin +++ b/cmd/lava/main.odin @@ -2,6 +2,8 @@ package main import "core:fmt" import "core:os" +import "core:strconv" +import html "lava:pkg/html" import lava_runtime "lava:pkg/runtime" import eventloop "lava:pkg/runtime/eventloop" @@ -24,6 +26,8 @@ main :: proc() { eval_command(args[2:]) case "run": run_command(args[2:]) + case "html": + html_command(args[2:]) case: fmt.eprintfln("unknown command: %s", args[1]) fmt.eprintln("try `lava --help`") @@ -38,6 +42,8 @@ print_help :: proc() { Usage: lava eval Evaluate JavaScript source lava run Run a JavaScript file + lava html [--scripts] [--chunk N] + Parse HTML and print the serialized document lava --version Print version lava --help Print help @@ -115,3 +121,75 @@ print_result :: proc(result: lava_runtime.Result, user_data: rawptr) { lava_runtime.process_write(os.stderr, result.message, "\n") } + +// html_command parses an HTML file (or stdin, `-`) and prints the serialized +// document. No script runs: --scripts reports each parser-blocking script on stderr +// at the moment the parser pauses for it, where a page loader would execute it. +// --chunk N feeds the input N bytes at a time, as network reads would, to exercise +// the incremental path; by default it is fed at once. +html_command :: proc(args: []string) { + usage :: proc() -> ! { + fmt.eprintln("usage: lava html [--scripts] [--chunk N] ") + os.exit(2) + } + Html_Options :: struct { + list_scripts: bool, + scripts: int, + } + opts: Html_Options + chunk := 0 + path := "" + for i := 0; i < len(args); i += 1 { + switch a := args[i]; { + case a == "--scripts": + opts.list_scripts = true + case a == "--chunk": + i += 1 + if i == len(args) do usage() + n, ok := strconv.parse_int(args[i]) + if !ok || n <= 0 do usage() + chunk = n + case len(a) > 1 && a[0] == '-', path != "": + usage() + case: + path = a + } + } + if path == "" do usage() + + data: []byte + err: os.Error + if path == "-" do data, err = os.read_entire_file_from_file(os.stdin, context.allocator) + else do data, err = os.read_entire_file(path, context.allocator) + if err != nil { + fmt.eprintfln("lava html: cannot read %s: %v", path, err) + os.exit(1) + } + + report :: proc(p: ^html.Parser, script: html.Node, user_data: rawptr) { + o := cast(^Html_Options)user_data + o.scripts += 1 + if !o.list_scripts do return + // An SVG ` inside a comment or a JS string + does not pause. More than one script closing in a slice is a failure, never a + dropped script; +- `write` puts `document.write` text at the insertion point, after earlier writes and + before the rest of the input; scripts in it pause in turn. + +The cut scan resumes where it stopped and the consumed input is compacted only once +it is half the buffer, so feeding a page in many small reads stays linear. One +unterminated `` drives it from the CLI. + --- ## 4. Cross-cutting concerns diff --git a/pkg/html/cut.odin b/pkg/html/cut.odin new file mode 100644 index 0000000..6d54ea9 --- /dev/null +++ b/pkg/html/cut.odin @@ -0,0 +1,183 @@ +package larva_html + +import "core:bytes" + +// Where to cut the input before handing it to lexbor. +// +// The parser pauses at a script by never letting a slice run past a script's end tag: +// each slice ends right after the `>` of the next ``. A real end tag is a few bytes long (attributes on an end +// tag are a parse error and carry nothing); past this, the candidate is fed through as +// not being one. Without the bound, one stray `` after it would +// hold back the rest of the page. If it was a real end tag after all, the script then +// closes inside a larger slice and the hook counts it (see pump), so nothing is lost +// silently. +@(private = "file") +END_TAG_MAX :: 4096 + +// Source is bytes waiting to be parsed: the network input, or text from +// document.write. Everything before `off` has been handed to lexbor. +@(private) +Source :: struct { + buf: [dynamic]byte, + off: int, + // Resumable scan state, all absolute indices into buf. + scan: int, // the search for the next candidate resumes here + tag: int, // start of a candidate whose `>` is not found yet; -1 when none + tag_pos: int, // inside that candidate, the scan resumes here + quote: byte, // inside that candidate, the open quote of an attribute value + after_eq: bool, // inside that candidate, an attribute value may start next +} + +@(private) +source_init :: proc(s: ^Source, allocator := context.allocator) { + s^ = { + buf = make([dynamic]byte, 0, 0, allocator), + tag = -1, + } +} + +@(private) +source_pending :: proc(s: ^Source) -> int { + return len(s.buf) - s.off +} + +// source_reset empties `s`, keeping its storage. +@(private) +source_reset :: proc(s: ^Source) { + clear(&s.buf) + s.off, s.scan, s.tag_pos, s.quote, s.after_eq = 0, 0, 0, 0, false + s.tag = -1 +} + +// source_compact drops what lexbor has consumed once it is at least half the buffer, +// so the copy is amortized instead of paid on every piece of input. +@(private) +source_compact :: proc(s: ^Source) { + if s.off == 0 || s.off * 2 < len(s.buf) do return + n := s.off + remove_range(&s.buf, 0, n) + s.off = 0 + s.scan = max(s.scan - n, 0) + if s.tag >= 0 { + s.tag -= n + s.tag_pos -= n + } +} + +// next_cut returns how many bytes after `s.off` can be parsed now; 0 means wait for +// more input. `eof` says no more bytes will follow these in this source. +@(private) +next_cut :: proc(s: ^Source, eof: bool) -> int { + end := len(s.buf) + if s.tag < 0 { + s.scan = max(s.scan, s.off) + search: for s.scan < end { + k := bytes.index_byte(s.buf[s.scan:], '<') + if k < 0 { + s.scan = end + break + } + at := s.scan + k + switch match_script_end(s.buf[at:]) { + case .No: + s.scan = at + 1 + continue + case .Need_More: + if eof { + s.scan = end + return end - s.off + } + s.scan = at + return at - s.off + case .Yes: + s.tag = at + s.tag_pos = at + len(SCRIPT_END_TAG) + s.quote, s.after_eq = 0, false + break search + } + } + if s.tag < 0 do return end - s.off + } + + // Inside a candidate: find its `>`, skipping quoted attribute values, as the + // tokenizer does in the attribute states of an end tag. + for ; s.tag_pos < end; s.tag_pos += 1 { + b := s.buf[s.tag_pos] + if s.quote != 0 { + if b == s.quote do s.quote = 0 + continue + } + switch { + case b == '>': + cut := s.tag_pos + 1 + s.tag = -1 + s.scan = cut + return cut - s.off + case (b == '"' || b == '\'') && s.after_eq: + s.quote = b + s.after_eq = false + case b == '=': + s.after_eq = true + case is_html_space(b): + // keeps after_eq: `a = "v"` is one attribute + case: + s.after_eq = false + } + } + if eof || s.tag_pos - s.tag > END_TAG_MAX { + s.tag = -1 + s.scan = end + return end - s.off + } + return s.tag - s.off +} + +@(private = "file") +Match :: enum { + No, + Yes, + Need_More, // `b` ends before it can be told +} + +// match_script_end says whether `b` starts with an end tag named "script": the +// characters of SCRIPT_END_TAG in any ASCII case, then whitespace, `/` or `>`. Anything +// else after the name (``) is a different tag. +@(private = "file") +match_script_end :: proc(b: []byte) -> Match { + needle := SCRIPT_END_TAG + for i in 0 ..< len(needle) { + if i >= len(b) do return .Need_More + if ascii_lower(b[i]) != needle[i] do return .No + } + if len(b) == len(needle) do return .Need_More + next := b[len(needle)] + return .Yes if next == '/' || next == '>' || is_html_space(next) else .No +} + +// ascii_lower folds A–Z only. HTML tag names compare ASCII case-insensitively; a +// Unicode fold (core:strings.equal_fold) would also match U+017F to 's', which the +// tokenizer does not. +@(private = "file") +ascii_lower :: proc(b: byte) -> byte { + return b + ('a' - 'A') if 'A' <= b && b <= 'Z' else b +} + +// is_html_space is the HTML spec's ASCII whitespace: tab, LF, FF, CR, space. +@(private = "file") +is_html_space :: proc(b: byte) -> bool { + return b == '\t' || b == '\n' || b == '\f' || b == '\r' || b == ' ' +} diff --git a/pkg/html/lexbor.odin b/pkg/html/lexbor.odin new file mode 100644 index 0000000..c0e53e1 --- /dev/null +++ b/pkg/html/lexbor.odin @@ -0,0 +1,51 @@ +#+build linux +#+private +package larva_html + +import "core:c" + +// Bindings to lexbor (https://github.com/lexbor/lexbor, Apache-2.0) and to the C shim +// beside it (shim/larva_html.c). Private to the package: callers see Parser and Node +// only, so lexbor stays replaceable behind them (docs/LARVA.md, decision 4). +// +// Hand-written, not generated with odin-c-bindgen: the surface is a dozen procs, and +// everything lexbor exposes as an inline function, a macro or a struct field goes +// through the shim, which a generator could not bind anyway. lexbor's structs stay +// opaque here. Revisit if the declared surface grows to dozens of procs. +// +// Linux only for now: scripts/build-lexbor.sh builds for the host, and only Linux is +// built and run (CLAUDE.md, "Linux-first"). lexbor_other.odin answers elsewhere. + +// The paths are relative to this file and must match OUT/LIB/SHIM_LIB in +// scripts/build-lexbor.sh; `foreign import` takes only literals. The shim comes first: +// it references lexbor, and a static link resolves left to right. +foreign import lexbor { + "../../.deps/lexbor/lib/liblarva_html_shim.a", + "../../.deps/lexbor/lib/liblexbor_static.a", +} + +// lexbor/core/base.h: `typedef unsigned int lxb_status_t`, LXB_STATUS_OK = 0x0000. +lxb_status_t :: c.uint +LXB_STATUS_OK :: lxb_status_t(0) + +lxb_document :: distinct rawptr // lxb_html_document_t * +larva_hook :: distinct rawptr // larva_html_hook * (shim) + +lxb_serialize_cb :: #type proc "c" (data: [^]u8, len: c.size_t, ctx: rawptr) -> lxb_status_t + +@(default_calling_convention = "c") +foreign lexbor { + lxb_html_document_create :: proc() -> lxb_document --- + lxb_html_document_destroy :: proc(doc: lxb_document) -> lxb_document --- + lxb_html_document_parse_chunk :: proc(doc: lxb_document, data: [^]u8, len: c.size_t) -> lxb_status_t --- + lxb_html_document_parse_chunk_end :: proc(doc: lxb_document) -> lxb_status_t --- + lxb_html_serialize_deep_cb :: proc(node: Node, cb: lxb_serialize_cb, ctx: rawptr) -> lxb_status_t --- + + larva_html_begin :: proc(doc: lxb_document) -> larva_hook --- + larva_html_take_script :: proc(h: larva_hook, closed: ^c.size_t) -> Node --- + larva_html_hook_free :: proc(h: larva_hook) --- + larva_html_document_node :: proc(doc: lxb_document) -> Node --- + larva_html_text :: proc(node: Node, len: ^c.size_t) -> [^]u8 --- + larva_html_text_free :: proc(node: Node, text: [^]u8) --- + larva_html_attr :: proc(node: Node, name: [^]u8, name_len: c.size_t, len: ^c.size_t) -> [^]u8 --- +} diff --git a/pkg/html/lexbor_other.odin b/pkg/html/lexbor_other.odin new file mode 100644 index 0000000..7d2d6b3 --- /dev/null +++ b/pkg/html/lexbor_other.odin @@ -0,0 +1,31 @@ +#+build !linux +#+private +package larva_html + +import "core:c" + +// No lexbor build exists off Linux yet (lexbor.odin). These stand in for the bindings +// with the same names, so the package compiles on every target and `init` reports the +// truth there: lxb_html_document_create returns nil, and init fails. + +lxb_status_t :: c.uint +LXB_STATUS_OK :: lxb_status_t(0) +LXB_STATUS_ERROR :: lxb_status_t(1) + +lxb_document :: distinct rawptr +larva_hook :: distinct rawptr + +lxb_serialize_cb :: #type proc "c" (data: [^]u8, len: c.size_t, ctx: rawptr) -> lxb_status_t + +lxb_html_document_create :: proc() -> lxb_document {return nil} +lxb_html_document_destroy :: proc(doc: lxb_document) -> lxb_document {return nil} +lxb_html_document_parse_chunk :: proc(doc: lxb_document, data: [^]u8, len: c.size_t) -> lxb_status_t {return LXB_STATUS_ERROR} +lxb_html_document_parse_chunk_end :: proc(doc: lxb_document) -> lxb_status_t {return LXB_STATUS_ERROR} +lxb_html_serialize_deep_cb :: proc(node: Node, cb: lxb_serialize_cb, ctx: rawptr) -> lxb_status_t {return LXB_STATUS_ERROR} +larva_html_begin :: proc(doc: lxb_document) -> larva_hook {return nil} +larva_html_take_script :: proc(h: larva_hook, closed: ^c.size_t) -> Node {closed^ = 0; return nil} +larva_html_hook_free :: proc(h: larva_hook) {} +larva_html_document_node :: proc(doc: lxb_document) -> Node {return nil} +larva_html_text :: proc(node: Node, len: ^c.size_t) -> [^]u8 {return nil} +larva_html_text_free :: proc(node: Node, text: [^]u8) {} +larva_html_attr :: proc(node: Node, name: [^]u8, name_len: c.size_t, len: ^c.size_t) -> [^]u8 {return nil} diff --git a/pkg/html/node.odin b/pkg/html/node.odin new file mode 100644 index 0000000..86139e0 --- /dev/null +++ b/pkg/html/node.odin @@ -0,0 +1,29 @@ +package larva_html + +import "core:c" +import "core:strings" + +// Node is a DOM node owned by its Parser's document. It stays valid until destroy: +// lexbor detaches nodes from the tree but frees them only with the document. +Node :: distinct rawptr + +// text returns `node`'s textContent, copied into `allocator`; "" for none. +text :: proc(node: Node, allocator := context.allocator) -> string { + n: c.size_t + t := larva_html_text(node, &n) + if t == nil do return "" + defer larva_html_text_free(node, t) + return strings.clone(string(t[:n]), allocator) +} + +// attr returns the value of attribute `name` on element `node`, copied into +// `allocator`. +// +// Returns: +// ok false when the attribute is absent (an empty value is ok true). +attr :: proc(node: Node, name: string, allocator := context.allocator) -> (value: string, ok: bool) { + n: c.size_t + v := larva_html_attr(node, raw_data(name), c.size_t(len(name)), &n) + if v == nil do return "", false + return strings.clone(string(v[:n]), allocator), true +} diff --git a/pkg/html/parser.odin b/pkg/html/parser.odin new file mode 100644 index 0000000..22d7447 --- /dev/null +++ b/pkg/html/parser.odin @@ -0,0 +1,224 @@ +package larva_html + +import "base:runtime" +import "core:c" +import "core:strings" + +// Parser is an incremental HTML parser that hands control back to the caller at every +// parser-blocking script, the way a browser's parser does (#10). +// +// The caller owns the input and decides when parsing moves on. lexbor has no pause and +// no document.write (its tree builder leaves both as TODO at ``, +// html/tree/insertion_mode/text.c) and needs neither: we feed it slices that never run +// past a script's end tag (cut.odin). So when `pump` returns a script, the tree holds +// exactly what precedes it — what the script would see in a browser. Text the script +// writes (`write`) is parsed next, before the rest of the input. +// +// p: Parser +// init(&p) or_return +// defer destroy(&p) +// feed(&p, bytes) // as they arrive; finish(&p) after the last +// for { +// script, status := pump(&p) +// switch status { +// case .Script: run it; write(&p, ...) for each document.write +// case .Need_Input: feed(&p, ...) or finish(&p) +// case .Done: // the document is complete +// case .Failed: // error_message(&p) says why +// } +// } +// +// parse_all wraps that loop for input already in memory. +// +// Known limits: +// - Input is taken as UTF-8: no charset sniffing (BOM, , Content-Type). +// - Script nesting level and document.open (a write after the parse ended) are not +// modelled; write is refused once the document is done. +// - A ` bool { + p^ = {} + source_init(&p.input, allocator) + source_init(&p.written, allocator) + when ODIN_OS != .Linux { + p.err = "pkg/html: no lexbor build for this target (Linux only)" + return false + } + p.doc = lxb_html_document_create() + if p.doc == nil { + p.err = "pkg/html: lexbor could not create a document" + return false + } + p.hook = larva_html_begin(p.doc) + if p.hook == nil { + p.err = "pkg/html: lexbor could not start parsing" + return false + } + return true +} + +// destroy frees the document and everything in it. Nodes from this parser are +// dangling afterwards. +destroy :: proc(p: ^Parser) { + // The document first: the hook is lexbor's tokenizer callback context until then. + if p.doc != nil do lxb_html_document_destroy(p.doc) + if p.hook != nil do larva_html_hook_free(p.hook) + delete(p.input.buf) + delete(p.written.buf) + p^ = {} +} + +// feed appends network bytes. They are parsed by pump. +feed :: proc(p: ^Parser, data: []byte) { + append(&p.input.buf, ..data) +} + +// finish marks the end of the input; the pump that runs out of input then ends the +// document. +finish :: proc(p: ^Parser) { + p.eof = true +} + +// write inserts `text` at the insertion point: right after the script pump last +// returned, after any text earlier writes put there, before the rest of the input — +// as document.write does. Scripts in `text` pause in turn. +// +// Returns: +// false, and nothing is inserted, once the document is done or the parser failed. +write :: proc(p: ^Parser, text: string) -> bool { + if p.ended || p.err != "" do return false + inject_at(&p.written.buf, p.ins, ..transmute([]byte)text) + p.ins += len(text) + return true +} + +// pump parses until the next parser-blocking script, the end of what has been fed, or +// the end of the document. +pump :: proc(p: ^Parser) -> (script: Node, status: Pump_Status) { + if p.err != "" do return nil, .Failed + if p.ended do return nil, .Done + for { + src, src_eof := &p.input, p.eof + if source_pending(&p.written) > 0 { + src, src_eof = &p.written, true + } else if len(p.written.buf) > 0 { + source_reset(&p.written) + p.ins = 0 + } + + if source_pending(src) == 0 { + if !p.eof { + source_compact(&p.input) + return nil, .Need_Input + } + p.ended = true + if lxb_html_document_parse_chunk_end(p.doc) != LXB_STATUS_OK { + return nil, fail(p, "pkg/html: lexbor failed to end the document") + } + return nil, .Done + } + + n := next_cut(src, src_eof) + if n == 0 { + source_compact(&p.input) + return nil, .Need_Input + } + if lxb_html_document_parse_chunk(p.doc, raw_data(src.buf[src.off:]), c.size_t(n)) != LXB_STATUS_OK { + return nil, fail(p, "pkg/html: lexbor failed to parse") + } + src.off += n + + closed: c.size_t + s := larva_html_take_script(p.hook, &closed) + if closed > 1 { + // A slice closed two scripts: the first was never handed out. cut.odin keeps + // this from happening; failing beats losing a script quietly. + return nil, fail(p, "pkg/html: two scripts closed in one slice") + } + if s != nil { + p.ins = p.written.off + return s, .Script + } + } +} + +// Script_Proc runs at each parser-blocking script during parse_all. It may call write. +Script_Proc :: #type proc(p: ^Parser, script: Node, user_data: rawptr) + +// parse_all parses `data` as the whole document, calling `on_script` at each pause. +// `chunk` > 0 feeds the input that many bytes at a time, as network reads would; 0 +// feeds it at once. +parse_all :: proc(p: ^Parser, data: []byte, on_script: Script_Proc = nil, user_data: rawptr = nil, chunk := 0) -> bool { + off := 0 + for { + script, status := pump(p) + switch status { + case .Script: + if on_script != nil do on_script(p, script, user_data) + case .Need_Input: + if off == len(data) { + finish(p) + continue + } + end := len(data) if chunk <= 0 else min(off + chunk, len(data)) + feed(p, data[off:end]) + off = end + case .Done: + return true + case .Failed: + return false + } + } +} + +// error_message says why the parser failed; "" when it has not. +error_message :: proc(p: ^Parser) -> string { + return p.err +} + +// serialize returns the document's markup — the doctype, then the html element. +// Comparing it with Chrome's is the oracle's job (#2), not done yet. +serialize :: proc(p: ^Parser, allocator := context.allocator) -> (markup: string, ok: bool) { + collect :: proc "c" (data: [^]u8, len: c.size_t, ctx: rawptr) -> lxb_status_t { + context = runtime.default_context() + strings.write_bytes(cast(^strings.Builder)ctx, data[:len]) + return LXB_STATUS_OK + } + b := strings.builder_make(allocator) + if p.doc == nil || lxb_html_serialize_deep_cb(larva_html_document_node(p.doc), collect, &b) != LXB_STATUS_OK { + strings.builder_destroy(&b) + return "", false + } + return strings.to_string(b), true +} + +@(private = "file") +fail :: proc(p: ^Parser, why: string) -> Pump_Status { + p.err = why + return .Failed +} diff --git a/pkg/html/shim/larva_html.c b/pkg/html/shim/larva_html.c new file mode 100644 index 0000000..1c7de9a --- /dev/null +++ b/pkg/html/shim/larva_html.c @@ -0,0 +1,101 @@ +// The C side of pkg/html: the few lexbor calls that are inline functions, macros or +// struct reads, which Odin cannot bind directly, plus the one hook the incremental +// parser needs (see larva_html_begin). Everything else is bound from Odin +// (pkg/html/lexbor.odin). Built by scripts/build-lexbor.sh. + +#include +#include +#include + +// One per document being parsed. The tree builder installs its own token callback on +// the tokenizer when chunk parsing begins; we wrap it so we see every token first. +typedef struct { + lxb_html_tokenizer_token_f orig; + void *orig_ctx; + lxb_html_tree_t *tree; + lxb_dom_node_t *script; // the ` that is not +// closing a script element (inside a comment, in plain text) never reaches here as an +// end tag with that current node, so it is ignored. +static lxb_html_token_t * +larva_hook_cb(lxb_html_tokenizer_t *tkz, lxb_html_token_t *token, void *ctx) +{ + larva_html_hook *h = ctx; + if (token->tag_id == LXB_TAG_SCRIPT && (token->type & LXB_HTML_TOKEN_TYPE_CLOSE)) { + lxb_dom_node_t *cur = lxb_html_tree_current_node(h->tree); + if (cur != NULL && cur->local_name == LXB_TAG_SCRIPT) { + h->script = cur; + h->closed++; + } + } + return h->orig(tkz, token, h->orig_ctx); +} + +// Starts chunked parsing of `doc` and installs the hook. NULL on failure; free with +// larva_html_hook_free after the document's parse has ended. +larva_html_hook * +larva_html_begin(lxb_html_document_t *doc) +{ + if (lxb_html_document_parse_chunk_begin(doc) != LXB_STATUS_OK) return NULL; + lxb_html_parser_t *parser = doc->dom_document.parser; + if (parser == NULL || parser->tkz == NULL || parser->tree == NULL) return NULL; + larva_html_hook *h = calloc(1, sizeof(*h)); + if (h == NULL) return NULL; + h->orig = parser->tkz->callback_token_done; + h->orig_ctx = parser->tkz->callback_token_ctx; + h->tree = parser->tree; + lxb_html_tokenizer_callback_token_done_set(parser->tkz, larva_hook_cb, h); + return h; +} + +// Returns the script element closed since the last call, and clears it. *closed is +// how many script end tags were seen in that time: the parser cuts its slices so that +// it is never more than 1, and treats more as a failure rather than lose a script. +lxb_dom_node_t * +larva_html_take_script(larva_html_hook *h, size_t *closed) +{ + lxb_dom_node_t *s = h->script; + *closed = h->closed; + h->script = NULL; + h->closed = 0; + return s; +} + +void +larva_html_hook_free(larva_html_hook *h) +{ + free(h); +} + +lxb_dom_node_t * +larva_html_document_node(lxb_html_document_t *doc) +{ + return lxb_dom_interface_node(doc); +} + +// textContent of `node`. The bytes belong to the document; release them with +// larva_html_text_free. +lxb_char_t * +larva_html_text(lxb_dom_node_t *node, size_t *len) +{ + return lxb_dom_node_text_content(node, len); +} + +void +larva_html_text_free(lxb_dom_node_t *node, lxb_char_t *text) +{ + if (text != NULL) lxb_dom_document_destroy_text(node->owner_document, text); +} + +// The value of attribute `name` on element `node`, or NULL when absent. Owned by the +// element; valid until the element changes. +const lxb_char_t * +larva_html_attr(lxb_dom_node_t *node, const lxb_char_t *name, size_t name_len, size_t *len) +{ + return lxb_dom_element_get_attribute(lxb_dom_interface_element(node), name, name_len, len); +} diff --git a/scripts/build-lexbor.sh b/scripts/build-lexbor.sh new file mode 100755 index 0000000..03f2799 --- /dev/null +++ b/scripts/build-lexbor.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env sh +set -eu + +# Builds lexbor (https://github.com/lexbor/lexbor, Apache-2.0) as a static library for +# pkg/html, and the small C shim next to it (pkg/html/shim). Pinned to one commit and +# checked against its tarball's sha256, so a build never picks up an unreviewed +# lexbor. Idempotent: does nothing when the archive for this pin already exists. +# +# Why a source build and not the distro package: liblexbor-dev is not installed on the +# dev machines or the CI image, and a pinned static archive keeps the parser identical +# everywhere. Why not vendored into the tree: lexbor's source is ~22 MB. +# +# Output (both under the gitignored .deps/): +# .deps/lexbor/lib/liblexbor_static.a +# .deps/lexbor/include/lexbor/... (headers, for the shim) +# .deps/lexbor/lib/liblarva_html_shim.a + +ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +LEXBOR_COMMIT=e609d9446f7adf06c67b89dcdc4a7c39430f0bd8 +LEXBOR_SHA256=c69fa3f8ea24cc1a9ea8ea295ff2d3c7a2e8347f211d05f8e0bea7e29fba3b33 +CC=${CC:-cc} + +OUT="$ROOT_DIR/.deps/lexbor" +STAMP="$OUT/.commit" +LIB="$OUT/lib/liblexbor_static.a" +SHIM_SRC="$ROOT_DIR/pkg/html/shim/larva_html.c" +SHIM_LIB="$OUT/lib/liblarva_html_shim.a" + +if [ ! -f "$LIB" ] || [ "$(cat "$STAMP" 2>/dev/null)" != "$LEXBOR_COMMIT" ]; then + command -v cmake >/dev/null || { + echo "build-lexbor: cmake is required to build lexbor" >&2 + exit 1 + } + work="$ROOT_DIR/.deps/lexbor-src" + rm -rf "$work" "$OUT" + mkdir -p "$work" "$OUT" + tarball="$work/lexbor.tar.gz" + curl -fsSL -o "$tarball" "https://github.com/lexbor/lexbor/archive/$LEXBOR_COMMIT.tar.gz" + echo "$LEXBOR_SHA256 $tarball" | sha256sum -c - >/dev/null || { + echo "build-lexbor: sha256 mismatch for lexbor $LEXBOR_COMMIT" >&2 + exit 1 + } + tar xzf "$tarball" -C "$work" --strip-components=1 + cmake -S "$work" -B "$work/build" -DCMAKE_BUILD_TYPE=Release \ + -DLEXBOR_BUILD_SHARED=OFF -DLEXBOR_BUILD_STATIC=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX="$OUT" >/dev/null + cmake --build "$work/build" -j "$(nproc 2>/dev/null || echo 4)" >/dev/null + cmake --install "$work/build" >/dev/null + # Some lexbor versions install into lib64. + [ -f "$LIB" ] || { mkdir -p "$OUT/lib" && cp "$work/build/liblexbor_static.a" "$LIB"; } + rm -rf "$work" + echo "$LEXBOR_COMMIT" >"$STAMP" + printf 'built %s\n' ".deps/lexbor (lexbor ${LEXBOR_COMMIT%"${LEXBOR_COMMIT#???????}"})" +fi + +if [ ! -f "$SHIM_LIB" ] || [ "$SHIM_SRC" -nt "$SHIM_LIB" ] || [ "$LIB" -nt "$SHIM_LIB" ]; then + "$CC" -O2 -fPIC -I "$OUT/include" -c "$SHIM_SRC" -o "$OUT/lib/larva_html_shim.o" + ar rcs "$SHIM_LIB" "$OUT/lib/larva_html_shim.o" + printf 'built %s\n' ".deps/lexbor/lib/liblarva_html_shim.a" +fi diff --git a/scripts/build-native-deps.sh b/scripts/build-native-deps.sh index b77c754..3730f52 100755 --- a/scripts/build-native-deps.sh +++ b/scripts/build-native-deps.sh @@ -32,3 +32,6 @@ build_archive() { } build_archive "$ROOT_DIR/pkg/runtime/picohttpparser" picohttpparser + +# lexbor + the pkg/html shim (pinned source build into .deps/; see the script). +"$ROOT_DIR/scripts/build-lexbor.sh"