Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ Layering (never blur it):
| Embedded spec surface | `pkg/runtime/js/**.js` | `Buffer`, `fetch`, streams, `URL`, `http`… |
| Engine FFI / private ABI | `pkg/jsc` | JSC C API, string/view/host ABI, probes |
| Event loop | `pkg/runtime/eventloop` | phases, timers, io_uring/epoll, threadpool |
| HTML parser | `pkg/html` | lexbor FFI + C shim, incremental `Parser` |

---

Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ help:
@printf '%s\n' ''
@printf '%s\n' 'Build / run'
@printf '%s\n' ' make build Build the lava CLI'
@printf '%s\n' ' make native-deps Build vendored C deps (picohttpparser)'
@printf '%s\n' ' make native-deps Build vendored C deps (picohttpparser, lexbor)'
@printf '%s\n' ' make bootstrap-windows-deps Fetch ignored Windows native deps into .deps/'
@printf '%s\n' ' make build-sqlite-windows Build build/sqlite3.lib via MSVC'
@printf '%s\n' ' make run FILE=app.js Run a JavaScript file through lava'
Expand Down
2 changes: 2 additions & 0 deletions NOTICE
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,6 @@ Libraries linked at build time, not distributed here
JavaScriptCore (WebKitGTK) LGPL-2.1-or-later and BSD-2-Clause; linked dynamically
OpenSSL 3 Apache-2.0
SQLite public domain
lexbor (pinned commit) Apache-2.0; linked statically, built from source by
scripts/build-lexbor.sh (pkg/html)
Odin core library zlib License
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ for how the runtime pieces fit together.
- SQLite development headers and libraries (for `node:sqlite`)
- OpenSSL development headers and libraries (for `fetch` HTTPS/TLS). The roadmap
replaces it with BoringSSL (stage 2); until that lands, OpenSSL is what links.
- `cmake`, `curl` and network access on the first build (Linux): `pkg/html` links
[lexbor](https://github.com/lexbor/lexbor), which `scripts/build-lexbor.sh` downloads
at a pinned commit, checks against its sha256 and builds into `.deps/`.

On Linux, the JSC package is exposed through `pkg-config` as `javascriptcoregtk-6.0`,
SQLite as `sqlite3`, and OpenSSL as `openssl`
Expand Down
78 changes: 78 additions & 0 deletions cmd/lava/main.odin
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package main

import "core:fmt"
import "core:os"
import "core:strconv"
import html "lava:pkg/html"
import lava_runtime "lava:pkg/runtime"
import eventloop "lava:pkg/runtime/eventloop"

Expand All @@ -24,6 +26,8 @@ main :: proc() {
eval_command(args[2:])
case "run":
run_command(args[2:])
case "html":
html_command(args[2:])
case:
fmt.eprintfln("unknown command: %s", args[1])
fmt.eprintln("try `lava --help`")
Expand All @@ -38,6 +42,8 @@ print_help :: proc() {
Usage:
lava eval <source> Evaluate JavaScript source
lava run <file> Run a JavaScript file
lava html [--scripts] [--chunk N] <file|->
Parse HTML and print the serialized document
lava --version Print version
lava --help Print help

Expand Down Expand Up @@ -115,3 +121,75 @@ print_result :: proc(result: lava_runtime.Result, user_data: rawptr) {

lava_runtime.process_write(os.stderr, result.message, "\n")
}

// html_command parses an HTML file (or stdin, `-`) and prints the serialized
// document. No script runs: --scripts reports each parser-blocking script on stderr
// at the moment the parser pauses for it, where a page loader would execute it.
// --chunk N feeds the input N bytes at a time, as network reads would, to exercise
// the incremental path; by default it is fed at once.
html_command :: proc(args: []string) {
usage :: proc() -> ! {
fmt.eprintln("usage: lava html [--scripts] [--chunk N] <file|->")
os.exit(2)
}
Html_Options :: struct {
list_scripts: bool,
scripts: int,
}
opts: Html_Options
chunk := 0
path := ""
for i := 0; i < len(args); i += 1 {
switch a := args[i]; {
case a == "--scripts":
opts.list_scripts = true
case a == "--chunk":
i += 1
if i == len(args) do usage()
n, ok := strconv.parse_int(args[i])
if !ok || n <= 0 do usage()
chunk = n
case len(a) > 1 && a[0] == '-', path != "":
usage()
case:
path = a
}
}
if path == "" do usage()

data: []byte
err: os.Error
if path == "-" do data, err = os.read_entire_file_from_file(os.stdin, context.allocator)
else do data, err = os.read_entire_file(path, context.allocator)
if err != nil {
fmt.eprintfln("lava html: cannot read %s: %v", path, err)
os.exit(1)
}

report :: proc(p: ^html.Parser, script: html.Node, user_data: rawptr) {
o := cast(^Html_Options)user_data
o.scripts += 1
if !o.list_scripts do return
// An SVG <script> names its file with href, an HTML one with src.
src, ok := html.attr(script, "src", context.temp_allocator)
if !ok do src, ok = html.attr(script, "href", context.temp_allocator)
if ok {
fmt.eprintfln("[script %d] src=%s", o.scripts, src)
} else {
fmt.eprintfln("[script %d] inline, %d bytes", o.scripts, len(html.text(script, context.temp_allocator)))
}
}

p: html.Parser
ok := html.init(&p) && html.parse_all(&p, data, report, &opts, chunk)
markup: string
if ok do markup, ok = html.serialize(&p)
if !ok {
msg := html.error_message(&p)
fmt.eprintfln("lava html: %s", msg if msg != "" else "serialization failed")
html.destroy(&p)
os.exit(1)
}
fmt.println(markup)
html.destroy(&p)
}
28 changes: 28 additions & 0 deletions docs/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,34 @@ swappable backend (OpenSSL, or a rejecting stub). DNS — `fetch`'s and `node:dn
incremental `ReadableStream`s with backpressure. This is the second-strongest
subsystem after the loop, and it validates the `post_async` design end to end.

### 3.6 HTML parser (`pkg/html`)

An incremental parser over [lexbor](https://github.com/lexbor/lexbor) (Apache-2.0,
pinned commit, built from source into `.deps/` by `scripts/build-lexbor.sh`). lexbor
stays behind the package: its bindings (`lexbor.odin`) are private, and callers see
`Parser` and `Node` only (docs/LARVA.md, decision 4). Linux only for now; elsewhere
`init` fails with a message (`lexbor_other.odin`).

The contract a page loader builds on is the pause. `pump` parses until the next
parser-blocking script and returns it with the tree holding exactly what precedes it,
as a browser's parser would. lexbor has no pause and no `document.write`, and needs
neither:

- the caller holds the input, and `cut.odin` cuts it so that a slice never runs past a
`</script` end tag's `>` (quoted attribute values skipped, as the tokenizer does);
- the C shim (`shim/larva_html.c`) wraps lexbor's token callback and records which
script element an end tag closed, so a `</script>` inside a comment or a JS string
does not pause. More than one script closing in a slice is a failure, never a
dropped script;
- `write` puts `document.write` text at the insertion point, after earlier writes and
before the rest of the input; scripts in it pause in turn.

The cut scan resumes where it stopped and the consumed input is compacted only once
it is half the buffer, so feeding a page in many small reads stays linear. One
unterminated `</script` holds back at most 4 KiB (`END_TAG_MAX`).

`lava html [--scripts] [--chunk N] <file|->` drives it from the CLI.

---

## 4. Cross-cutting concerns
Expand Down
183 changes: 183 additions & 0 deletions pkg/html/cut.odin
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
package larva_html

import "core:bytes"

// Where to cut the input before handing it to lexbor.
//
// The parser pauses at a script by never letting a slice run past a script's end tag:
// each slice ends right after the `>` of the next `</script` end tag, so when lexbor
// reports that it closed a script (the shim's hook), nothing after the script has been
// parsed. Whether a candidate really closes a script is lexbor's call — this only
// places the cuts, and a cut in the wrong place costs nothing but an extra call.
//
// The scan is resumable: a Source remembers how far it has looked, so input that
// arrives in many small pieces is scanned once, not once per piece.

// SCRIPT_END_TAG is what an end tag that can close a script starts with; it is
// matched ASCII case-insensitively, as the tokenizer does.
@(private = "file")
SCRIPT_END_TAG :: "</script"

// END_TAG_MAX bounds how many bytes of one unterminated `</script…` candidate are held
// back waiting for its `>`. A real end tag is a few bytes long (attributes on an end
// tag are a parse error and carry nothing); past this, the candidate is fed through as
// not being one. Without the bound, one stray `</script ` with no `>` after it would
// hold back the rest of the page. If it was a real end tag after all, the script then
// closes inside a larger slice and the hook counts it (see pump), so nothing is lost
// silently.
@(private = "file")
END_TAG_MAX :: 4096

// Source is bytes waiting to be parsed: the network input, or text from
// document.write. Everything before `off` has been handed to lexbor.
@(private)
Source :: struct {
buf: [dynamic]byte,
off: int,
// Resumable scan state, all absolute indices into buf.
scan: int, // the search for the next candidate resumes here
tag: int, // start of a candidate whose `>` is not found yet; -1 when none
tag_pos: int, // inside that candidate, the scan resumes here
quote: byte, // inside that candidate, the open quote of an attribute value
after_eq: bool, // inside that candidate, an attribute value may start next
}

@(private)
source_init :: proc(s: ^Source, allocator := context.allocator) {
s^ = {
buf = make([dynamic]byte, 0, 0, allocator),
tag = -1,
}
}

@(private)
source_pending :: proc(s: ^Source) -> int {
return len(s.buf) - s.off
}

// source_reset empties `s`, keeping its storage.
@(private)
source_reset :: proc(s: ^Source) {
clear(&s.buf)
s.off, s.scan, s.tag_pos, s.quote, s.after_eq = 0, 0, 0, 0, false
s.tag = -1
}

// source_compact drops what lexbor has consumed once it is at least half the buffer,
// so the copy is amortized instead of paid on every piece of input.
@(private)
source_compact :: proc(s: ^Source) {
if s.off == 0 || s.off * 2 < len(s.buf) do return
n := s.off
remove_range(&s.buf, 0, n)
s.off = 0
s.scan = max(s.scan - n, 0)
if s.tag >= 0 {
s.tag -= n
s.tag_pos -= n
}
}

// next_cut returns how many bytes after `s.off` can be parsed now; 0 means wait for
// more input. `eof` says no more bytes will follow these in this source.
@(private)
next_cut :: proc(s: ^Source, eof: bool) -> int {
end := len(s.buf)
if s.tag < 0 {
s.scan = max(s.scan, s.off)
search: for s.scan < end {
k := bytes.index_byte(s.buf[s.scan:], '<')
if k < 0 {
s.scan = end
break
}
at := s.scan + k
switch match_script_end(s.buf[at:]) {
case .No:
s.scan = at + 1
continue
case .Need_More:
if eof {
s.scan = end
return end - s.off
}
s.scan = at
return at - s.off
case .Yes:
s.tag = at
s.tag_pos = at + len(SCRIPT_END_TAG)
s.quote, s.after_eq = 0, false
break search
}
}
if s.tag < 0 do return end - s.off
}

// Inside a candidate: find its `>`, skipping quoted attribute values, as the
// tokenizer does in the attribute states of an end tag.
for ; s.tag_pos < end; s.tag_pos += 1 {
b := s.buf[s.tag_pos]
if s.quote != 0 {
if b == s.quote do s.quote = 0
continue
}
switch {
case b == '>':
cut := s.tag_pos + 1
s.tag = -1
s.scan = cut
return cut - s.off
case (b == '"' || b == '\'') && s.after_eq:
s.quote = b
s.after_eq = false
case b == '=':
s.after_eq = true
case is_html_space(b):
// keeps after_eq: `a = "v"` is one attribute
case:
s.after_eq = false
}
}
if eof || s.tag_pos - s.tag > END_TAG_MAX {
s.tag = -1
s.scan = end
return end - s.off
}
return s.tag - s.off
}

@(private = "file")
Match :: enum {
No,
Yes,
Need_More, // `b` ends before it can be told
}

// match_script_end says whether `b` starts with an end tag named "script": the
// characters of SCRIPT_END_TAG in any ASCII case, then whitespace, `/` or `>`. Anything
// else after the name (`</scripts>`) is a different tag.
@(private = "file")
match_script_end :: proc(b: []byte) -> Match {
needle := SCRIPT_END_TAG
for i in 0 ..< len(needle) {
if i >= len(b) do return .Need_More
if ascii_lower(b[i]) != needle[i] do return .No
}
if len(b) == len(needle) do return .Need_More
next := b[len(needle)]
return .Yes if next == '/' || next == '>' || is_html_space(next) else .No
}

// ascii_lower folds A–Z only. HTML tag names compare ASCII case-insensitively; a
// Unicode fold (core:strings.equal_fold) would also match U+017F to 's', which the
// tokenizer does not.
@(private = "file")
ascii_lower :: proc(b: byte) -> byte {
return b + ('a' - 'A') if 'A' <= b && b <= 'Z' else b
}

// is_html_space is the HTML spec's ASCII whitespace: tab, LF, FF, CR, space.
@(private = "file")
is_html_space :: proc(b: byte) -> bool {
return b == '\t' || b == '\n' || b == '\f' || b == '\r' || b == ' '
}
Loading
Loading