From 344d94bf3991fab66b3e788bd7c73c5d67bc927e Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 03:24:17 -0500 Subject: [PATCH 01/36] docs(spec,plan): fold in Muse's review notes on the teach recorder From the review of #362, none blocking: - search for the two values encoded together (base64 of user:pass), not only each alone - the recording says which script world the page listener ran in - name in-page encryption as the common case of transformed/needs_person - piece 2's player depends on #355 The plan gains search_pair, ListenerWorld and the tests for both, and now says the branch is cut from master after #362 merged. Refs #361. Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- .../plans/2026-11-18-361-teach-recorder.md | 17 ++++++++++++++--- .../specs/2026-11-18-teach-recorder-design.md | 17 ++++++++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/superpowers/plans/2026-11-18-361-teach-recorder.md b/docs/superpowers/plans/2026-11-18-361-teach-recorder.md index 13759d5..18dc6ba 100644 --- a/docs/superpowers/plans/2026-11-18-361-teach-recorder.md +++ b/docs/superpowers/plans/2026-11-18-361-teach-recorder.md @@ -12,7 +12,7 @@ ## Global Constraints -- Work in the worktree `~/projects/kharon-361-teach`, on a branch named `duet/361-teach-recorder` cut from `duet/361-teach-recorder-design` (the branch that holds the spec and this plan, PR #362). The implementation PR is opened with that branch as its base; when #362 merges, its base moves to `master` and master is merged in. No rebase, no force-push. Set `CARGO_TARGET_DIR=$PWD/target` in every cargo command; the machine's default target dir is shared between checkouts. +- Work in the worktree `~/projects/kharon-361-teach`, on branch `duet/361-teach-recorder`, cut from `master` at `947a457` (the merge of #362, which holds the spec and this plan). No rebase, no force-push. Set `CARGO_TARGET_DIR=$PWD/target` in every cargo command; the machine's default target dir is shared between checkouts. - Every commit message ends with these two lines and no other `Co-Authored-By` line: ``` Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 @@ -136,11 +136,13 @@ pub enum Step { EnterKey { t_ms: u64, frame: FrameRef, element: ElementDescriptor }, } pub enum EndedBy { Confirmed, WindowClosed } +pub enum ListenerWorld { Isolated, Main } pub struct Viewport { pub width: u32, pub height: u32 } pub struct BrowserInfo { pub product: String, pub user_agent: String, pub viewport: Viewport } pub struct StrippedCounts { pub requests_seen: u64, pub requests_stored: u64, pub request_bodies_reduced: u64, pub query_values_removed: u64, pub path_segments_replaced: u64, pub typed_values_removed: u64 } pub struct Recording { pub format_version: u32, pub tool_version: String, pub browser: BrowserInfo, + pub listener_world: ListenerWorld, pub started_at: String, pub ended_at: String, pub ended_by: EndedBy, pub success_confirmed: bool, pub start_url: StrippedUrl, pub final_url: Option, pub steps: Vec, pub requests: Vec, pub flags: Flags, pub classification: Classification, @@ -288,6 +290,9 @@ impl Marker { } /// Every way `marker` occurs in `haystack`. Digests are searched only for Role::Password. pub fn search(haystack: &[u8], marker: &Marker) -> Vec; +/// The base64 and base64url (padded or not) of `username:password`. A match returns two hits, +/// one per role, both `Encoding::Base64` (or `Base64Url`). +pub fn search_pair(haystack: &[u8], username: &Marker, password: &Marker) -> Vec; pub const LOW_CONFIDENCE_BELOW: u32 = 6; ``` @@ -315,6 +320,10 @@ fn encodings(h: &[u8], m: &Marker) -> Vec { search(h, m).into_iter().m #[test] fn non_ascii_percent_encoded() { /* "pässwörd✓12" as UTF-8 percent-encoded -> [Percent] */ } #[test] fn non_ascii_json_escaped_both_forms() { /* raw UTF-8 inside quotes -> [AsTyped]; ä-style escapes -> [JsonEscaped] */ } #[test] fn two_hundred_characters() { /* a 200-char value is found as typed */ } +#[test] fn pair_is_found_where_neither_value_alone_is() { + // haystack: {"auth":""}; search() finds nothing for either marker; + // search_pair() returns [Username Base64, Password Base64] +} ``` Also add this doc test on `Marker`, which is the compile-time guard: @@ -423,7 +432,7 @@ pub struct RawFrame { pub main: bool, pub url: String } pub enum PageEventKind { Input, Change, Click, Submit, EnterKey } pub enum RawBody { None, Unavailable, Bytes(zeroize::Zeroizing>) } pub enum CaptureEvent { - Started { browser: BrowserInfo }, // the main frame left the notice page + Started { browser: BrowserInfo, listener_world: ListenerWorld }, // the main frame left the notice page Navigation { t_ms: u64, frame: RawFrame }, // main and child frames alike Page { t_ms: u64, kind: PageEventKind, frame: RawFrame, element: RawElement, value: Option> }, // value only for Input and Change @@ -446,6 +455,7 @@ impl Capture { Decisions `finish` implements, beyond the spec's text: +- Once the username is resolved, every buffered request's URL, `Authorization` header and body are also searched with `search_pair`; its hits are added to that request's hits (at whole-body level for a body). - Elements are keyed by `(document_id, element_id)`. One `Marker` per element holds its latest value. An element is a password element if any of its reports had `is_password_now`. - Password markers: the latest values of password elements, with equal values merged. More than one distinct value sets `multiple_passwords`. - Username: among non-password elements with a non-empty value, prefer one in the same form as a password element; else one whose value is found in a request that also carries the password; else one whose value is found in any request; else the last one typed in before the first password input. Other typed values are dropped. @@ -471,6 +481,7 @@ Decisions `finish` implements, beyond the spec's text: #[test] fn captcha_frame_sets_the_flag() #[test] fn otp_input_after_the_password_sets_the_flag() #[test] fn oversized_body_is_unavailable_and_flagged() // a MAX_BODY_BYTES + 1 body on a POST to the login site +#[test] fn a_combined_base64_credential_in_a_json_body_is_a_plain_hit() // classifies as ScriptPost, not Transformed #[test] fn only_the_three_kinds_of_request_are_stored() // a GET image and a third-party POST are counted, not stored #[test] fn events_before_started_are_ignored_for_steps() #[test] fn the_serialized_recording_holds_no_typed_value() { @@ -641,7 +652,7 @@ pub async fn record(opts: RecordOptions<'_>, ready: tokio::sync::oneshot::Sender end: tokio::sync::oneshot::Receiver) -> Result; ``` -Wiring, in order: `Profile::sweep_stale`; `Profile::create`; `Browser::launch` with `first_url = notice_page_url(..)`; `Cdp::connect`; `Target.setAutoAttach { autoAttach: true, waitForDebuggerOnStart: true, flatten: true }` on the browser session and again on each attached page session; for each attached target of type `page` or `iframe`: `Page.enable`, `Network.enable`, `Runtime.addBinding { name: "__kharonTeach", executionContextName: "kharon-teach" }`, `Page.addScriptToEvaluateOnNewDocument { source: LISTENER_JS, worldName: "kharon-teach" }`, then `Runtime.runIfWaitingForDebugger`. If the binding is not visible in the isolated world on the CI browser, install script and binding in the main world instead and record that in the commit message. Other target types are released without attaching streams. A second `page` target after the first sends `CaptureEvent::Popup`. `CaptureEvent::Started` is sent when the main frame first navigates to a URL that is not the notice page; its `BrowserInfo` comes from `Browser.getVersion` (product, user agent) and `Page.getLayoutMetrics` (the CSS viewport). A request with `hasPostData` and no inline body is completed with `Network.getRequestPostData`; an error there is `RawBody::Unavailable`. The window counts as closed when the last `page` target is destroyed or the browser process exits; that ends as `EndedBy::WindowClosed` with `success_confirmed: false`, or `Outcome::NeverStarted` if `Started` never fired. +Wiring, in order: `Profile::sweep_stale`; `Profile::create`; `Browser::launch` with `first_url = notice_page_url(..)`; `Cdp::connect`; `Target.setAutoAttach { autoAttach: true, waitForDebuggerOnStart: true, flatten: true }` on the browser session and again on each attached page session; for each attached target of type `page` or `iframe`: `Page.enable`, `Network.enable`, `Runtime.addBinding { name: "__kharonTeach", executionContextName: "kharon-teach" }`, `Page.addScriptToEvaluateOnNewDocument { source: LISTENER_JS, worldName: "kharon-teach" }`, then `Runtime.runIfWaitingForDebugger`. If the binding is not visible in the isolated world, install script and binding in the main world instead. Which world was used goes into `CaptureEvent::Started` as `listener_world`, and so into the file; say in the commit message which one the CI browser needed. Other target types are released without attaching streams. A second `page` target after the first sends `CaptureEvent::Popup`. `CaptureEvent::Started` is sent when the main frame first navigates to a URL that is not the notice page; its `BrowserInfo` comes from `Browser.getVersion` (product, user agent) and `Page.getLayoutMetrics` (the CSS viewport). A request with `hasPostData` and no inline body is completed with `Network.getRequestPostData`; an error there is `RawBody::Unavailable`. The window counts as closed when the last `page` target is destroyed or the browser process exits; that ends as `EndedBy::WindowClosed` with `success_confirmed: false`, or `Outcome::NeverStarted` if `Started` never fired. `listener.js` reports, in the capture phase on `document`: `input` and `change` on `input`, `textarea`, `select`; `click` on `button`, `a`, `input[type=submit]`, `input[type=button]`, `[role=button]`; `submit` on forms; `keydown` with `key === "Enter"` on an `input`. It numbers elements with a `WeakMap` counter, gives each document a random `document_id`, uses `event.composedPath()[0]` to see into open shadow roots, and builds `selectors` (how is the builder's call; the first entry must match the element with `document.querySelector` at the time of the event when the element is not in a shadow root). It sends `value` only for `input` and `change`. It keeps no state other than the counter. diff --git a/docs/superpowers/specs/2026-11-18-teach-recorder-design.md b/docs/superpowers/specs/2026-11-18-teach-recorder-design.md index 15b8ec4..5df9ba1 100644 --- a/docs/superpowers/specs/2026-11-18-teach-recorder-design.md +++ b/docs/superpowers/specs/2026-11-18-teach-recorder-design.md @@ -41,7 +41,8 @@ The work is three pieces, in this order (Duet's call, accepted): that names the login's shape. 2. The login-method format, a player that replays a method with a real secret at rung 2, importing a recording into a broker, and the - per-site registry. + per-site registry. Rung 2 includes the post-submit scrub, so the + player needs #355 to land first. 3. The never-in-the-page rung. The recorder goes first because pieces 2 and 3 need real recordings to @@ -288,6 +289,12 @@ password it is also searched for the hex and base64 of its MD5, SHA-1, SHA-256 and SHA-512. A marker shorter than six characters is searched but its matches are marked low-confidence. +Some logins encode the two values together, as HTTP Basic does with +`username:password`. Neither value's own base64 appears in that. So +once the username is known, each request is also searched for the +base64 and base64url of `username:password`, and a match counts as a +hit for both (Muse's review of #362). + What is recorded is the **role** and the place: "the `password` key of this urlencoded body is the password, as typed". The value is not. @@ -323,6 +330,10 @@ directory unless `--out` says otherwise. It holds: - the tool version, the browser's product string and user agent, the viewport size, start and end times, how it ended, and `success_confirmed` +- `listener_world`: `isolated` or `main`, the script world the page + listener ran in. In the main world a page's own scripts can see and + fake the listener's reports, so a reader should trust such a + recording less - the start URL and the last main-frame URL, stripped as above - the ordered steps: navigations and the person's events - the stored requests, as shapes with roles @@ -457,3 +468,7 @@ For the implementation PR, wording to be reviewed there: recorder still writes it nowhere, but the program read it and the browser held it until the profile was deleted. - Passkey logins are not detected in this piece. +- `transformed` with no named digest will mostly be sites that encrypt + the password in the page with a key the site supplies, which banks + often do. It is the expected common case of `needs_person`, not an + oddity, and piece 2 should be designed with that in mind. From 0982430921084879e5e71a881079c2864a267e02 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 04:55:31 -0500 Subject: [PATCH 02/36] docs(plan): settle six plan defects found before the teach recorder build The combined-credential search had nowhere to record a hit in a JSON, urlencoded or multipart body, so shape_of_body and reduce_authorization now take the pair and record it on the value that holds it. Test commands with two filters are written the way cargo accepts them. Source scans cover shipped code only. The session gets the timestamp helper it needs before the command line exists. The public suffix snapshot is shared by path. The manual run waits for a person. Refs #361 Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- .../plans/2026-11-18-361-teach-recorder.md | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/superpowers/plans/2026-11-18-361-teach-recorder.md b/docs/superpowers/plans/2026-11-18-361-teach-recorder.md index 18dc6ba..a73d1fe 100644 --- a/docs/superpowers/plans/2026-11-18-361-teach-recorder.md +++ b/docs/superpowers/plans/2026-11-18-361-teach-recorder.md @@ -27,6 +27,8 @@ - The binary listens on no socket and opens no connection except loopback CDP to the browser it launched. - Real-browser tests follow the existing convention: find the binary with the same path list as `chromium_bin` in `crates/kharond/tests/req_arch_003_org_chromium.rs`, honor `KHARON_CHROMIUM`, skip when none is found unless `KHARON_REQUIRE_CHROMIUM` is set, in which case panic. - Test files that prove a SPEC row are named for it: `req_tch_00N_.rs`. +- A source scan in a test covers shipped code only: for each file under `crates/kharon-teach/src`, the text before its first `#[cfg(test)]`. Unit-test modules sit at the bottom of their file, so a test server or a test URL there does not trip a scan. +- `cargo-deny` is not installed on the dev machine. Where a task says to run `cargo deny check`, CI runs it on the PR. - Linux only is built and tested. Platform-specific code sits behind `cfg`. ## Review Focus @@ -360,15 +362,17 @@ pub struct Stripped { pub value: T, pub query_values_removed: u64, pub path_s pub fn strip_url(url: &str, markers: &[Marker]) -> Option>; // None when `url` does not parse /// scheme = first token, lowercased. For `basic`, the credentials are base64-decoded and the decoded bytes /// are searched; a marker found there is reported with `Encoding::Base64`. Other schemes are searched as sent. -pub fn reduce_authorization(header_value: &[u8], markers: &[Marker]) -> AuthShape; +/// When `pair` is given the value is also searched with `search_pair`. No (role, encoding) is listed twice. +pub fn reduce_authorization(header_value: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> AuthShape; // body.rs pub const MAX_BODY_BYTES: usize = 1 << 20; /// `content_type` picks the reader: application/x-www-form-urlencoded, application/json (and any +json), /// multipart/form-data; anything else, or a body that fails to parse as its declared type, is Opaque. -pub fn shape_of_body(content_type: Option<&str>, body: &[u8], markers: &[Marker]) -> BodyShape; +/// `pair` is (username, password) once both are known; see below. +pub fn shape_of_body(content_type: Option<&str>, body: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> BodyShape; ``` -A multipart part with a `filename` is `ValueKind::File` with its length and no marker search. Every other value is searched with every marker. `shape_of_body` never returns `None` or `Unavailable`; the caller (Task 5) chooses those. +A multipart part with a `filename` is `ValueKind::File` with its length and no marker search. Every other value is searched with every marker and, when `pair` is given, with `search_pair`, so a combined credential is recorded on the value that holds it (an `Opaque` body holds it at whole-body level). A place never lists the same role and encoding twice. `shape_of_body` never returns `None` or `Unavailable`; the caller (Task 5) chooses those. - [ ] **Step 1: Write the failing tests** @@ -397,10 +401,12 @@ A multipart part with a `filename` is `ValueKind::File` with its length and no m #[test] fn multipart_text_and_file() { /* a text part "password" with a hit, a file part kind File with no hits */ } #[test] fn unknown_type_is_opaque_with_whole_body_hit() { /* content type application/octet-stream holding the password bytes -> Opaque { len, markers: [AsTyped] } */ } #[test] fn bad_json_falls_back_to_opaque() {} +#[test] fn a_combined_credential_lands_on_the_value_that_holds_it() { /* {"auth":"","n":1} with pair -> the auth value has [Username Base64, Password Base64]; without pair -> no hits */ } +#[test] fn basic_auth_with_the_pair_lists_each_hit_once() { /* reduce_authorization on a Basic header with pair given -> still exactly [Username Base64, Password Base64] */ } #[test] fn serialized_shapes_never_contain_a_marker() { /* for each case above: serde_json::to_string(&shape) contains neither "Tr0ub4dor-9x" nor "kh.throwaway7" */ } ``` -- [ ] **Step 2: Run** `cargo test -p kharon-teach strip body`. Expected: FAIL. +- [ ] **Step 2: Run** `cargo test -p kharon-teach -- strip body` (cargo takes one filter before `--`; several go after it). Expected: FAIL. - [ ] **Step 3: Implement.** Add `url = "2"`, `serde_json`. The JSON body is the one place a site payload is read as `serde_json::Value`; say so in a comment. Urlencoded keys and JSON keys are kept as they are; values never are. - [ ] **Step 4: Run** tests, clippy, fmt. Expected: pass. - [ ] **Step 5: Commit** `feat(teach): strip URLs and reduce request bodies to shapes (#361)` @@ -455,7 +461,7 @@ impl Capture { Decisions `finish` implements, beyond the spec's text: -- Once the username is resolved, every buffered request's URL, `Authorization` header and body are also searched with `search_pair`; its hits are added to that request's hits (at whole-body level for a body). +- Username candidates are tried against the raw buffered bytes first. Once the username is resolved, every buffered request is reduced with the pair: `search_pair` over the URL bytes goes into `url_markers`, and `reduce_authorization` and `shape_of_body` are called with `pair = Some((username, password))`, so a combined credential is recorded where it sits. - Elements are keyed by `(document_id, element_id)`. One `Marker` per element holds its latest value. An element is a password element if any of its reports had `is_password_now`. - Password markers: the latest values of password elements, with equal values merged. More than one distinct value sets `multiple_passwords`. - Username: among non-password elements with a non-empty value, prefer one in the same form as a password element; else one whose value is found in a request that also carries the password; else one whose value is found in any request; else the last one typed in before the first password input. Other typed values are dropped. @@ -492,8 +498,8 @@ Decisions `finish` implements, beyond the spec's text: #[test] fn registrable_domains() { /* login.ex.co.uk -> ex.co.uk; 127.0.0.1 and localhost unchanged */ } ``` -- [ ] **Step 2: Run** `cargo test -p kharon-teach capture site`. Expected: FAIL. -- [ ] **Step 3: Implement.** Load the public suffix list the way `crates/kharond/src/origin.rs` does. Requests are buffered with their bodies in `Zeroizing` buffers and analyzed in `finish`, after the markers are known. +- [ ] **Step 2: Run** `cargo test -p kharon-teach -- capture site`. Expected: FAIL. +- [ ] **Step 3: Implement.** Load the public suffix list the way `crates/kharond/src/origin.rs` does: `include_bytes!` of the same vendored snapshot, `crates/kharond/data/public_suffix_list.dat`, by relative path. That is one data file shared, not a crate dependency; do not copy it. Requests are buffered with their bodies in `Zeroizing` buffers and analyzed in `finish`, after the markers are known. - [ ] **Step 4: Run** tests, clippy, fmt. Expected: pass. - [ ] **Step 5: Commit** `feat(teach): capture state machine from events to a stripped recording (#361)` @@ -611,7 +617,7 @@ impl Cdp { #[test] fn headers_are_read_case_insensitively() ``` -- [ ] **Step 2: Run** `cargo test -p kharon-teach cdp events`. Expected: FAIL. +- [ ] **Step 2: Run** `cargo test -p kharon-teach -- cdp events`. Expected: FAIL. - [ ] **Step 3: Implement.** Add `tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "handshake"] }`, `futures-util`. One reader task routes replies by `id` to `oneshot` senders and everything else to the event channel. Outgoing frames are typed structs, never `json!`. - [ ] **Step 4: Run** tests, clippy, fmt. Expected: pass. - [ ] **Step 5: Commit** `feat(teach): a CDP client that subscribes to events (#361)` @@ -646,6 +652,8 @@ pub struct Ready { pub devtools_port: u16, pub profile_dir: std::path::PathBuf, pub enum SessionError { #[error(transparent)] Browser(#[from] BrowserError), #[error(transparent)] Cdp(#[from] CdpError), #[error(transparent)] Io(#[from] std::io::Error) } pub enum Outcome { Recorded(Box), Discarded, NeverStarted } +/// RFC 3339 UTC, with the `time` crate as `crates/kharond/src/audit/log.rs` does. Fills `EndInfo`; Task 9 reuses it. +pub fn utc_rfc3339(t: std::time::SystemTime) -> String; /// Sweeps stale profiles, launches, records until `end` fires or the window closes, cleans up. /// `ready` fires once the browser is up. Dropping the future (Ctrl-C) still removes the profile. pub async fn record(opts: RecordOptions<'_>, ready: tokio::sync::oneshot::Sender, @@ -778,8 +786,8 @@ Exit codes: `0` a file was written, or the recording was discarded; `2` a refusa ``` - [ ] **Step 2: Run** `cargo test -p kharon-teach`. Expected: the new tests FAIL. -- [ ] **Step 3: Implement.** Add `clap` (derive) and, Linux only, `rustix` with the `process` feature. For the UTC timestamp use whatever `crates/kharond/src/audit/log.rs` uses for its `ts` field; do not add a second date crate. -- [ ] **Step 4: Run** `KHARON_REQUIRE_CHROMIUM=1 cargo test -p kharon-teach`, clippy, fmt, `cargo deny check`. Then one manual run on this machine: `cargo run -p kharon-teach -- https://login.page.twkr.io/login.html`, log in with made-up values, press Enter, and read the file. Paste the printed summary into the commit body. +- [ ] **Step 3: Implement.** Add `clap` (derive) and, Linux only, `rustix` with the `process` feature. For the file name's UTC timestamp use the `time` crate, already a dependency since Task 8's `utc_rfc3339`; do not add a second date crate. +- [ ] **Step 4: Run** `KHARON_REQUIRE_CHROMIUM=1 cargo test -p kharon-teach`, clippy, fmt, `cargo deny check`. The manual run (`cargo run -p kharon-teach -- https://login.page.twkr.io/login.html`, log in with made-up values, press Enter, read the file) needs a person at a display. The implementer does not do it; the controller arranges it with Lothrop after this task and puts the printed summary in the PR body. - [ ] **Step 5: Commit** `feat(teach): kharon-teach command line, consent and the recording file (#361)` --- From c659227166fed16b1992b4f10cc0477f165e8a77 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 04:57:21 -0500 Subject: [PATCH 03/36] feat(recording): typed recording format for teach mode (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- Cargo.lock | 9 + Cargo.toml | 1 + crates/kharon-recording/Cargo.toml | 13 + crates/kharon-recording/src/classify.rs | 49 +++ crates/kharon-recording/src/format.rs | 397 ++++++++++++++++++++++++ crates/kharon-recording/src/lib.rs | 15 + crates/kharon-recording/src/request.rs | 168 ++++++++++ 7 files changed, 652 insertions(+) create mode 100644 crates/kharon-recording/Cargo.toml create mode 100644 crates/kharon-recording/src/classify.rs create mode 100644 crates/kharon-recording/src/format.rs create mode 100644 crates/kharon-recording/src/lib.rs create mode 100644 crates/kharon-recording/src/request.rs diff --git a/Cargo.lock b/Cargo.lock index 5b8b951..9a69396 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1070,6 +1070,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "kharon-recording" +version = "0.0.0" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "kharond" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index ee2fa89..e04dbec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/kharon-cli", "crates/kharon-connector", "crates/kharon-agent-connector", + "crates/kharon-recording", ] # SPEC §8 process hardening: a release build aborts on panic instead of diff --git a/crates/kharon-recording/Cargo.toml b/crates/kharon-recording/Cargo.toml new file mode 100644 index 0000000..f4d4d2f --- /dev/null +++ b/crates/kharon-recording/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "kharon-recording" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true +description = "Typed recording format for kharon teach mode (#361)" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "2" diff --git a/crates/kharon-recording/src/classify.rs b/crates/kharon-recording/src/classify.rs new file mode 100644 index 0000000..cb3349b --- /dev/null +++ b/crates/kharon-recording/src/classify.rs @@ -0,0 +1,49 @@ +//! Classification types. Task 2 adds the classifier functions. + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct Flags { + pub captcha: bool, + pub otp_step: bool, + pub cross_site: bool, + pub popup: bool, + pub framed_login: bool, + pub shadow_dom: bool, + pub multiple_passwords: bool, + pub body_unavailable: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PasswordLeg { + FormPost, + ScriptPost, + Transformed, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "shape", rename_all = "snake_case")] +pub enum Shape { + FormPost, + ScriptPost, + TwoStep { password_leg: PasswordLeg }, + Transformed { digest: Option }, + Unrecognized, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RungCeiling { + Milliseconds, + AgentContinues, + NeedsPerson, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Classification { + pub shape: Shape, + pub rung_ceiling: RungCeiling, + pub network_swap_candidate: bool, + pub notes: Vec, +} diff --git a/crates/kharon-recording/src/format.rs b/crates/kharon-recording/src/format.rs new file mode 100644 index 0000000..ed29fca --- /dev/null +++ b/crates/kharon-recording/src/format.rs @@ -0,0 +1,397 @@ +//! The recording file: a typed, value-free record of a demonstrated login. + +use serde::{Deserialize, Serialize}; + +use crate::classify::{Classification, Flags}; +use crate::request::{Role, StoredRequest}; + +pub const FORMAT_VERSION: u32 = 1; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StrippedUrl { + pub origin: String, + pub path: String, + pub query_keys: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct FrameRef { + pub main: bool, + pub origin: String, + pub cross_origin: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct FormRef { + pub index: u32, + pub action: Option, + pub method: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ElementDescriptor { + pub element_id: u32, + pub tag: String, + pub input_type: Option, + pub id: Option, + pub name: Option, + pub autocomplete: Option, + pub placeholder: Option, + pub aria_label: Option, + pub was_password: bool, + pub form: Option, + pub selectors: Vec, + pub in_shadow_root: bool, +} + +/// An `Input` step is recorded once per element, at its first input event, and +/// its `value_len` is updated in place as the value changes. `change` events +/// add no step. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Step { + Navigation { + t_ms: u64, + frame: FrameRef, + url: StrippedUrl, + }, + Input { + t_ms: u64, + frame: FrameRef, + element: ElementDescriptor, + value_len: u32, + role: Role, + }, + Click { + t_ms: u64, + frame: FrameRef, + element: ElementDescriptor, + }, + Submit { + t_ms: u64, + frame: FrameRef, + form: FormRef, + }, + EnterKey { + t_ms: u64, + frame: FrameRef, + element: ElementDescriptor, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EndedBy { + Confirmed, + WindowClosed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ListenerWorld { + Isolated, + Main, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Viewport { + pub width: u32, + pub height: u32, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct BrowserInfo { + pub product: String, + pub user_agent: String, + pub viewport: Viewport, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StrippedCounts { + pub requests_seen: u64, + pub requests_stored: u64, + pub request_bodies_reduced: u64, + pub query_values_removed: u64, + pub path_segments_replaced: u64, + pub typed_values_removed: u64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Recording { + pub format_version: u32, + pub tool_version: String, + pub browser: BrowserInfo, + pub listener_world: ListenerWorld, + pub started_at: String, + pub ended_at: String, + pub ended_by: EndedBy, + pub success_confirmed: bool, + pub start_url: StrippedUrl, + pub final_url: Option, + pub steps: Vec, + pub requests: Vec, + pub flags: Flags, + pub classification: Classification, + pub stripped: StrippedCounts, +} + +#[derive(Debug, thiserror::Error)] +pub enum FormatError { + #[error("unsupported recording format_version {0}")] + UnsupportedVersion(u32), + #[error("not a kharon recording: {0}")] + Parse(#[from] serde_json::Error), +} + +#[derive(Deserialize)] +struct VersionProbe { + format_version: u32, +} + +impl Recording { + /// Pretty JSON with a trailing newline. + pub fn to_json(&self) -> String { + let mut text = serde_json::to_string_pretty(self) + .expect("a Recording is plain data and always serializes"); + text.push('\n'); + text + } + + /// The version is checked before the full parse. + pub fn from_json(text: &str) -> Result { + let probe: VersionProbe = serde_json::from_str(text)?; + if probe.format_version != FORMAT_VERSION { + return Err(FormatError::UnsupportedVersion(probe.format_version)); + } + Ok(serde_json::from_str(text)?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::classify::{PasswordLeg, RungCeiling, Shape}; + use crate::request::{ + AuthShape, BodyShape, Encoding, FieldShape, JsonField, JsonShape, MarkerHit, ResourceKind, + ValueKind, ValueShape, + }; + + fn url(path: &str) -> StrippedUrl { + StrippedUrl { + origin: "https://example.test".into(), + path: path.into(), + query_keys: vec![], + } + } + + fn frame() -> FrameRef { + FrameRef { + main: true, + origin: "https://example.test".into(), + cross_origin: false, + } + } + + fn hit(role: Role) -> MarkerHit { + MarkerHit { + role, + encoding: Encoding::AsTyped, + low_confidence: false, + } + } + + fn value(markers: Vec) -> ValueShape { + ValueShape { + kind: ValueKind::String, + len: 8, + markers, + } + } + + fn request(body: BodyShape) -> StoredRequest { + StoredRequest { + t_ms: 120, + method: "POST".into(), + url: url("/login"), + resource: ResourceKind::Document, + main_frame_navigation: true, + content_type: Some("application/x-www-form-urlencoded".into()), + authorization: None, + url_markers: vec![], + body, + response: None, + } + } + + fn sample() -> Recording { + let element = ElementDescriptor { + element_id: 2, + tag: "input".into(), + input_type: Some("password".into()), + id: Some("pw".into()), + name: Some("password".into()), + autocomplete: None, + placeholder: None, + aria_label: None, + was_password: true, + form: Some(FormRef { + index: 0, + action: Some(url("/login")), + method: "post".into(), + }), + selectors: vec!["#pw".into()], + in_shadow_root: false, + }; + let body = BodyShape::Urlencoded { + fields: vec![FieldShape { + key: "password".into(), + value: value(vec![hit(Role::Password)]), + }], + }; + Recording { + format_version: FORMAT_VERSION, + tool_version: "0.0.0".into(), + browser: BrowserInfo { + product: "Chrome/1".into(), + user_agent: "UA".into(), + viewport: Viewport { + width: 1280, + height: 800, + }, + }, + listener_world: ListenerWorld::Isolated, + started_at: "2026-11-18T00:00:00Z".into(), + ended_at: "2026-11-18T00:01:00Z".into(), + ended_by: EndedBy::Confirmed, + success_confirmed: true, + start_url: url("/login"), + final_url: Some(url("/home")), + steps: vec![ + Step::Navigation { + t_ms: 0, + frame: frame(), + url: url("/login"), + }, + Step::Input { + t_ms: 50, + frame: frame(), + element, + value_len: 8, + role: Role::Password, + }, + ], + requests: vec![request(body)], + flags: Flags::default(), + classification: Classification { + shape: Shape::TwoStep { + password_leg: PasswordLeg::FormPost, + }, + rung_ceiling: RungCeiling::Milliseconds, + network_swap_candidate: false, + notes: vec![], + }, + stripped: StrippedCounts { + requests_seen: 3, + requests_stored: 1, + request_bodies_reduced: 1, + query_values_removed: 0, + path_segments_replaced: 0, + typed_values_removed: 2, + }, + } + } + + fn request_with_three_hits() -> StoredRequest { + let nested = JsonShape::Object { + fields: vec![JsonField { + key: "auth".into(), + value: JsonShape::Array { + items: vec![JsonShape::Object { + fields: vec![JsonField { + key: "p".into(), + value: JsonShape::Value { + value: value(vec![hit(Role::Password)]), + }, + }], + }], + }, + }], + }; + let mut r = request(BodyShape::Json { root: nested }); + r.url_markers = vec![hit(Role::Username)]; + r.authorization = Some(AuthShape { + scheme: "Bearer".into(), + markers: vec![hit(Role::Password)], + }); + r + } + + #[test] + fn round_trips() { + let r = sample(); + assert_eq!(Recording::from_json(&r.to_json()).unwrap(), r); + } + + #[test] + fn round_trips_nested_classification_shape() { + let c = Classification { + shape: Shape::Transformed { + digest: Some("sha256".into()), + }, + rung_ceiling: RungCeiling::NeedsPerson, + network_swap_candidate: true, + notes: vec!["n".into()], + }; + let text = serde_json::to_string(&c).unwrap(); + assert!( + text.contains("\"shape\":{\"shape\":\"transformed\""), + "{text}" + ); + assert_eq!(serde_json::from_str::(&text).unwrap(), c); + } + + #[test] + fn refuses_an_unknown_version() { + let text = sample() + .to_json() + .replace("\"format_version\": 1", "\"format_version\": 2"); + assert!(matches!( + Recording::from_json(&text), + Err(FormatError::UnsupportedVersion(2)) + )); + } + + #[test] + fn refuses_a_file_with_no_version() { + assert!(matches!( + Recording::from_json("{}"), + Err(FormatError::Parse(_)) + )); + } + + #[test] + fn wire_names_are_snake_case_and_tagged() { + let j = sample().to_json(); + for needle in [ + "\"type\": \"navigation\"", + "\"role\": \"password\"", + "\"kind\": \"urlencoded\"", + "\"encoding\": \"as_typed\"", + "\"ended_by\": \"confirmed\"", + ] { + assert!(j.contains(needle), "missing {needle}"); + } + } + + #[test] + fn hits_walks_url_authorization_and_nested_json() { + assert_eq!(request_with_three_hits().hits().len(), 3); + } + + #[test] + fn digest_names() { + assert!(Encoding::Sha256Hex.is_digest()); + assert_eq!(Encoding::Md5Base64.digest_name(), Some("md5")); + assert!(!Encoding::Hex.is_digest()); + } +} diff --git a/crates/kharon-recording/src/lib.rs b/crates/kharon-recording/src/lib.rs new file mode 100644 index 0000000..da8c362 --- /dev/null +++ b/crates/kharon-recording/src/lib.rs @@ -0,0 +1,15 @@ +//! Typed recording format for kharon teach mode (#361). + +pub mod classify; +pub mod format; +pub mod request; + +pub use classify::{Classification, Flags, PasswordLeg, RungCeiling, Shape}; +pub use format::{ + BrowserInfo, ElementDescriptor, EndedBy, FORMAT_VERSION, FormRef, FormatError, FrameRef, + ListenerWorld, Recording, Step, StrippedCounts, StrippedUrl, Viewport, +}; +pub use request::{ + AuthShape, BodyShape, Encoding, FieldShape, JsonField, JsonShape, MarkerHit, ResourceKind, + ResponseShape, Role, StoredRequest, ValueKind, ValueShape, +}; diff --git a/crates/kharon-recording/src/request.rs b/crates/kharon-recording/src/request.rs new file mode 100644 index 0000000..9c9b210 --- /dev/null +++ b/crates/kharon-recording/src/request.rs @@ -0,0 +1,168 @@ +//! Reduced request shapes. Values are never stored, only lengths and marker hits. + +use serde::{Deserialize, Serialize}; + +use crate::format::StrippedUrl; + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Role { + None, + Username, + Password, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Encoding { + AsTyped, + Percent, + JsonEscaped, + Base64, + Base64Url, + Hex, + Md5Hex, + Md5Base64, + Sha1Hex, + Sha1Base64, + Sha256Hex, + Sha256Base64, + Sha512Hex, + Sha512Base64, +} + +impl Encoding { + pub fn is_digest(self) -> bool { + self.digest_name().is_some() + } + + pub fn digest_name(self) -> Option<&'static str> { + match self { + Self::Md5Hex | Self::Md5Base64 => Some("md5"), + Self::Sha1Hex | Self::Sha1Base64 => Some("sha1"), + Self::Sha256Hex | Self::Sha256Base64 => Some("sha256"), + Self::Sha512Hex | Self::Sha512Base64 => Some("sha512"), + _ => None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct MarkerHit { + pub role: Role, + pub encoding: Encoding, + pub low_confidence: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ValueKind { + String, + Number, + Bool, + Null, + File, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ValueShape { + pub kind: ValueKind, + pub len: u32, + pub markers: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct FieldShape { + pub key: String, + pub value: ValueShape, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum JsonShape { + Object { fields: Vec }, + Array { items: Vec }, + Value { value: ValueShape }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct JsonField { + pub key: String, + pub value: JsonShape, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum BodyShape { + None, + Unavailable, + Urlencoded { fields: Vec }, + Json { root: JsonShape }, + Multipart { parts: Vec }, + Opaque { len: u64, markers: Vec }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AuthShape { + pub scheme: String, + pub markers: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ResourceKind { + Document, + Xhr, + Fetch, + Other, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ResponseShape { + pub status: u16, + pub content_type: Option, + pub set_cookie: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StoredRequest { + pub t_ms: u64, + pub method: String, + pub url: StrippedUrl, + pub resource: ResourceKind, + pub main_frame_navigation: bool, + pub content_type: Option, + pub authorization: Option, + pub url_markers: Vec, + pub body: BodyShape, + pub response: Option, +} + +impl JsonShape { + fn collect<'a>(&'a self, out: &mut Vec<&'a MarkerHit>) { + match self { + Self::Object { fields } => fields.iter().for_each(|f| f.value.collect(out)), + Self::Array { items } => items.iter().for_each(|i| i.collect(out)), + Self::Value { value } => out.extend(&value.markers), + } + } +} + +impl StoredRequest { + /// Every marker hit in the URL, the authorization header and every level of the body. + pub fn hits(&self) -> Vec<&MarkerHit> { + let mut out: Vec<&MarkerHit> = self.url_markers.iter().collect(); + if let Some(auth) = &self.authorization { + out.extend(&auth.markers); + } + match &self.body { + BodyShape::None | BodyShape::Unavailable => {} + BodyShape::Urlencoded { fields } | BodyShape::Multipart { parts: fields } => { + out.extend(fields.iter().flat_map(|f| &f.value.markers)); + } + BodyShape::Json { root } => root.collect(&mut out), + BodyShape::Opaque { markers, .. } => out.extend(markers), + } + out + } +} From 88c93148af88ad9aa61303092ce15cfa1bedd431 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 04:58:58 -0500 Subject: [PATCH 04/36] docs(plan): tag Shape with kind, like the other tagged enums The task 1 review found that tagging Shape with "shape" makes the file read "shape": {"shape": ...}. Refs #361 Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- docs/superpowers/plans/2026-11-18-361-teach-recorder.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/superpowers/plans/2026-11-18-361-teach-recorder.md b/docs/superpowers/plans/2026-11-18-361-teach-recorder.md index a73d1fe..cd300b1 100644 --- a/docs/superpowers/plans/2026-11-18-361-teach-recorder.md +++ b/docs/superpowers/plans/2026-11-18-361-teach-recorder.md @@ -220,7 +220,7 @@ fn hits_walks_url_authorization_and_nested_json() { pub struct Flags { pub captcha: bool, pub otp_step: bool, pub cross_site: bool, pub popup: bool, pub framed_login: bool, pub shadow_dom: bool, pub multiple_passwords: bool, pub body_unavailable: bool } #[serde(rename_all = "snake_case")] pub enum PasswordLeg { FormPost, ScriptPost, Transformed } -#[serde(tag = "shape", rename_all = "snake_case")] +#[serde(tag = "kind", rename_all = "snake_case")] // on the wire: "shape": {"kind": "two_step", "password_leg": "form_post"} pub enum Shape { FormPost, ScriptPost, TwoStep { password_leg: PasswordLeg }, Transformed { digest: Option }, Unrecognized } #[serde(rename_all = "snake_case")] pub enum RungCeiling { Milliseconds, AgentContinues, NeedsPerson } From 9c63670c63d2d81c666482da526c68f0098af01c Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 04:59:43 -0500 Subject: [PATCH 05/36] refactor(recording): tag Shape with kind and round-trip every variant (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-recording/src/classify.rs | 2 +- crates/kharon-recording/src/format.rs | 130 +++++++++++++++++++++++- crates/kharon-recording/src/request.rs | 28 ++++- 3 files changed, 152 insertions(+), 8 deletions(-) diff --git a/crates/kharon-recording/src/classify.rs b/crates/kharon-recording/src/classify.rs index cb3349b..80cb77e 100644 --- a/crates/kharon-recording/src/classify.rs +++ b/crates/kharon-recording/src/classify.rs @@ -23,7 +23,7 @@ pub enum PasswordLeg { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "shape", rename_all = "snake_case")] +#[serde(tag = "kind", rename_all = "snake_case")] pub enum Shape { FormPost, ScriptPost, diff --git a/crates/kharon-recording/src/format.rs b/crates/kharon-recording/src/format.rs index ed29fca..cc7e74b 100644 --- a/crates/kharon-recording/src/format.rs +++ b/crates/kharon-recording/src/format.rs @@ -9,6 +9,7 @@ pub const FORMAT_VERSION: u32 = 1; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct StrippedUrl { + /// scheme://host[:port] pub origin: String, pub path: String, pub query_keys: Vec, @@ -38,12 +39,16 @@ pub struct ElementDescriptor { pub autocomplete: Option, pub placeholder: Option, pub aria_label: Option, + /// True when the element was a password input (type=password) when recorded. pub was_password: bool, pub form: Option, + /// Candidate CSS selectors, most stable first. pub selectors: Vec, pub in_shadow_root: bool, } +/// Every `t_ms` is milliseconds since the recording started. +/// /// An `Input` step is recorded once per element, at its first input event, and /// its `value_len` is updated in place as the value changes. `change` events /// add no step. @@ -59,6 +64,7 @@ pub enum Step { t_ms: u64, frame: FrameRef, element: ElementDescriptor, + /// Characters typed so far. value_len: u32, role: Role, }, @@ -108,21 +114,30 @@ pub struct BrowserInfo { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct StrippedCounts { + /// Requests observed on the wire. pub requests_seen: u64, + /// Requests kept in the file. pub requests_stored: u64, + /// Request bodies replaced by a value-free shape. pub request_bodies_reduced: u64, + /// URL query values dropped (keys are kept). pub query_values_removed: u64, + /// URL path segments replaced by a placeholder. pub path_segments_replaced: u64, + /// Typed input values discarded (only lengths are kept). pub typed_values_removed: u64, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Recording { + /// `to_json` always writes `FORMAT_VERSION`, whatever this field holds. pub format_version: u32, pub tool_version: String, pub browser: BrowserInfo, pub listener_world: ListenerWorld, + /// RFC 3339, UTC. pub started_at: String, + /// RFC 3339, UTC. pub ended_at: String, pub ended_by: EndedBy, pub success_confirmed: bool, @@ -151,7 +166,9 @@ struct VersionProbe { impl Recording { /// Pretty JSON with a trailing newline. pub fn to_json(&self) -> String { - let mut text = serde_json::to_string_pretty(self) + let mut copy = self.clone(); + copy.format_version = FORMAT_VERSION; + let mut text = serde_json::to_string_pretty(©) .expect("a Recording is plain data and always serializes"); text.push('\n'); text @@ -344,7 +361,7 @@ mod tests { }; let text = serde_json::to_string(&c).unwrap(); assert!( - text.contains("\"shape\":{\"shape\":\"transformed\""), + text.contains("\"shape\":{\"kind\":\"transformed\""), "{text}" ); assert_eq!(serde_json::from_str::(&text).unwrap(), c); @@ -394,4 +411,113 @@ mod tests { assert_eq!(Encoding::Md5Base64.digest_name(), Some("md5")); assert!(!Encoding::Hex.is_digest()); } + + #[test] + fn to_json_always_writes_the_current_version() { + let mut r = sample(); + r.format_version = 99; + let back = Recording::from_json(&r.to_json()).unwrap(); + assert_eq!(back.format_version, FORMAT_VERSION); + } + + #[test] + fn every_variant_round_trips() { + let mut r = sample(); + let Step::Input { element, .. } = r.steps[1].clone() else { + unreachable!() + }; + let form = element.form.clone().unwrap(); + r.steps.extend([ + Step::Click { + t_ms: 60, + frame: frame(), + element: element.clone(), + }, + Step::Submit { + t_ms: 70, + frame: frame(), + form, + }, + Step::EnterKey { + t_ms: 80, + frame: frame(), + element, + }, + ]); + let json_root = JsonShape::Object { + fields: vec![JsonField { + key: "p".into(), + value: JsonShape::Value { + value: value(vec![hit(Role::Password)]), + }, + }], + }; + let bodies = [ + BodyShape::None, + BodyShape::Unavailable, + BodyShape::Json { root: json_root }, + BodyShape::Multipart { + parts: vec![FieldShape { + key: "f".into(), + value: value(vec![hit(Role::Username)]), + }], + }, + BodyShape::Opaque { + len: 9, + markers: vec![hit(Role::Password)], + }, + ]; + r.requests = bodies.into_iter().map(request).collect(); + let shapes = [ + Shape::FormPost, + Shape::ScriptPost, + Shape::TwoStep { + password_leg: PasswordLeg::ScriptPost, + }, + Shape::TwoStep { + password_leg: PasswordLeg::Transformed, + }, + Shape::Transformed { digest: None }, + Shape::Transformed { + digest: Some("md5".into()), + }, + Shape::Unrecognized, + ]; + for shape in shapes { + r.classification.shape = shape; + assert_eq!(Recording::from_json(&r.to_json()).unwrap(), r); + } + } + + #[test] + fn two_step_wire_form() { + let c = sample().classification; + let text = serde_json::to_string(&c).unwrap(); + assert!( + text.contains(r#""shape":{"kind":"two_step","password_leg":"form_post"}"#), + "{text}" + ); + } + + #[test] + fn hits_covers_multipart_and_opaque() { + let m = request(BodyShape::Multipart { + parts: vec![ + FieldShape { + key: "a".into(), + value: value(vec![hit(Role::Username)]), + }, + FieldShape { + key: "b".into(), + value: value(vec![hit(Role::Password)]), + }, + ], + }); + assert_eq!(m.hits().len(), 2); + let o = request(BodyShape::Opaque { + len: 4, + markers: vec![hit(Role::Password)], + }); + assert_eq!(o.hits().len(), 1); + } } diff --git a/crates/kharon-recording/src/request.rs b/crates/kharon-recording/src/request.rs index 9c9b210..a1146f6 100644 --- a/crates/kharon-recording/src/request.rs +++ b/crates/kharon-recording/src/request.rs @@ -42,7 +42,12 @@ impl Encoding { Self::Sha1Hex | Self::Sha1Base64 => Some("sha1"), Self::Sha256Hex | Self::Sha256Base64 => Some("sha256"), Self::Sha512Hex | Self::Sha512Base64 => Some("sha512"), - _ => None, + Self::AsTyped + | Self::Percent + | Self::JsonEscaped + | Self::Base64 + | Self::Base64Url + | Self::Hex => None, } } } @@ -51,6 +56,7 @@ impl Encoding { pub struct MarkerHit { pub role: Role, pub encoding: Encoding, + /// True when the match is weak (short value, or a common substring), so a reader treats it as a hint. pub low_confidence: bool, } @@ -67,6 +73,7 @@ pub enum ValueKind { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ValueShape { pub kind: ValueKind, + /// Characters for a string value, bytes for a file. pub len: u32, pub markers: Vec, } @@ -96,10 +103,20 @@ pub struct JsonField { pub enum BodyShape { None, Unavailable, - Urlencoded { fields: Vec }, - Json { root: JsonShape }, - Multipart { parts: Vec }, - Opaque { len: u64, markers: Vec }, + Urlencoded { + fields: Vec, + }, + Json { + root: JsonShape, + }, + Multipart { + parts: Vec, + }, + Opaque { + /// Bytes. + len: u64, + markers: Vec, + }, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -126,6 +143,7 @@ pub struct ResponseShape { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct StoredRequest { + /// Milliseconds since the recording started. pub t_ms: u64, pub method: String, pub url: StrippedUrl, From c6a265c2604c6a3c7161474c665340edc82184c2 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:01:54 -0500 Subject: [PATCH 06/36] feat(recording): classify a recorded login by shape and rung ceiling (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-recording/src/classify.rs | 634 +++++++++++++++++++++++- crates/kharon-recording/src/lib.rs | 4 +- crates/kharond/src/cdp/page_js.rs | 2 + 3 files changed, 638 insertions(+), 2 deletions(-) diff --git a/crates/kharon-recording/src/classify.rs b/crates/kharon-recording/src/classify.rs index 80cb77e..3d3bff5 100644 --- a/crates/kharon-recording/src/classify.rs +++ b/crates/kharon-recording/src/classify.rs @@ -1,7 +1,10 @@ -//! Classification types. Task 2 adds the classifier functions. +//! Classification of a recorded login: its shape and the strongest rung it can take. use serde::{Deserialize, Serialize}; +use crate::format::{ElementDescriptor, Step}; +use crate::request::{ResourceKind, Role, StoredRequest}; + #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct Flags { pub captcha: bool, @@ -47,3 +50,632 @@ pub struct Classification { pub network_swap_candidate: bool, pub notes: Vec, } + +fn plain_hit(r: &StoredRequest) -> bool { + r.hits() + .iter() + .any(|h| h.role == Role::Password && !h.encoding.is_digest()) +} + +fn has_hit(r: &StoredRequest, role: Role) -> bool { + r.hits().iter().any(|h| h.role == role) +} + +/// Restates the three TOTP tests in `resolveField` in +/// `crates/kharond/src/cdp/page_js.rs`; keep the two in step. +pub fn is_otp_descriptor(e: &ElementDescriptor) -> bool { + e.autocomplete + .as_deref() + .is_some_and(|a| a.contains("one-time")) + || e.name + .as_deref() + .is_some_and(|n| n.to_lowercase().contains("otp")) + || e.input_type.as_deref() == Some("tel") +} + +fn t_ms(step: &Step) -> u64 { + match step { + Step::Navigation { t_ms, .. } + | Step::Input { t_ms, .. } + | Step::Click { t_ms, .. } + | Step::Submit { t_ms, .. } + | Step::EnterKey { t_ms, .. } => *t_ms, + } +} + +fn unrecognized(note: &str) -> Classification { + Classification { + shape: Shape::Unrecognized, + rung_ceiling: RungCeiling::NeedsPerson, + network_swap_candidate: false, + notes: vec![note.to_string()], + } +} + +/// Names the shape of a recorded login and the strongest rung it can take. +pub fn classify(steps: &[Step], requests: &[StoredRequest], flags: &Flags) -> Classification { + let swap_candidate = requests.iter().filter(|r| plain_hit(r)).count() == 1; + + // Rule 1. + let blocking = [ + ( + flags.cross_site, + "The login goes to another site partway through, so a person has to look at it.", + ), + ( + flags.popup, + "The login opens a pop-up window, so a person has to look at it.", + ), + ( + flags.framed_login, + "The login form sits inside a frame, so a person has to look at it.", + ), + ( + flags.multiple_passwords, + "The page has more than one password field, so a person has to look at it.", + ), + ]; + let notes: Vec = blocking + .iter() + .filter(|(on, _)| *on) + .map(|(_, n)| n.to_string()) + .collect(); + if !notes.is_empty() { + return Classification { + shape: Shape::Unrecognized, + rung_ceiling: RungCeiling::NeedsPerson, + network_swap_candidate: false, + notes, + }; + } + + // Rule 2. + let first_password = steps.iter().find_map(|s| match s { + Step::Input { + t_ms, + role: Role::Password, + element, + .. + } => Some((*t_ms, element)), + _ => None, + }); + let Some((p, password_element)) = first_password else { + return unrecognized("no password field was used"); + }; + + // Rule 3. + let username_t = steps.iter().find_map(|s| match s { + Step::Input { + t_ms, + role: Role::Username, + .. + } => Some(*t_ms), + _ => None, + }); + let navigated = username_t.is_some_and(|u| { + steps.iter().any(|s| { + matches!(s, Step::Navigation { frame, .. } if frame.main) && t_ms(s) > u && t_ms(s) < p + }) + }); + let username_only_request = requests + .iter() + .any(|r| r.t_ms < p && has_hit(r, Role::Username) && !has_hit(r, Role::Password)); + let two_step = navigated || username_only_request; + + // Rule 4. + let mut notes = Vec::new(); + let earliest_plain = requests + .iter() + .filter(|r| plain_hit(r)) + .min_by_key(|r| r.t_ms); + let leg = match earliest_plain { + Some(r) => match r.resource { + ResourceKind::Document if r.main_frame_navigation => PasswordLeg::FormPost, + ResourceKind::Xhr | ResourceKind::Fetch => PasswordLeg::ScriptPost, + _ => { + notes.push( + "The password was sent in a way this tool does not know, so a person has to look at it." + .to_string(), + ); + return finish_unrecognized(notes); + } + }, + None => { + if flags.body_unavailable { + notes.push( + "The browser would not give up the request bodies, so the password could not be found in them." + .to_string(), + ); + return finish_unrecognized(notes); + } + PasswordLeg::Transformed + } + }; + let digest = requests + .iter() + .flat_map(|r| r.hits()) + .find(|h| h.role == Role::Password && h.encoding.is_digest()) + .and_then(|h| h.encoding.digest_name()) + .map(str::to_string); + + // Rule 5. + let shape = if two_step { + Shape::TwoStep { password_leg: leg } + } else { + match leg { + PasswordLeg::FormPost => Shape::FormPost, + PasswordLeg::ScriptPost => Shape::ScriptPost, + PasswordLeg::Transformed => Shape::Transformed { digest }, + } + }; + + // Rule 6. + let mut ceiling = RungCeiling::Milliseconds; + let mut lower = |to: RungCeiling, note: &str| { + if to == RungCeiling::NeedsPerson || ceiling == RungCeiling::Milliseconds { + ceiling = to; + } + notes.push(note.to_string()); + }; + if matches!(shape, Shape::TwoStep { .. }) { + lower( + RungCeiling::AgentContinues, + "The username and password are entered on separate steps, so the agent has to carry on after the first.", + ); + } + if flags.captcha { + lower( + RungCeiling::AgentContinues, + "The login shows a CAPTCHA, so the agent has to carry on from there.", + ); + } + if flags.otp_step { + lower( + RungCeiling::AgentContinues, + "The login asks for a one-time code, so the agent has to carry on from there.", + ); + } + let one_page = !matches!(shape, Shape::TwoStep { .. }); + if one_page && password_element.form.is_none() { + let can_submit = steps.iter().any(|s| { + t_ms(s) >= p + && match s { + Step::EnterKey { .. } => true, + Step::Click { element, .. } => { + element.tag.eq_ignore_ascii_case("button") + || element.input_type.as_deref() == Some("submit") + } + _ => false, + } + }); + if !can_submit { + lower( + RungCeiling::AgentContinues, + "The password field is not in a form and nothing was pressed to send it, so the agent has to find a way to submit.", + ); + } + } + if matches!(shape, Shape::Transformed { digest: None }) { + lower( + RungCeiling::NeedsPerson, + "The password is changed before it is sent in a way this tool cannot name, so a person has to look at it.", + ); + } + + Classification { + shape, + rung_ceiling: ceiling, + network_swap_candidate: swap_candidate, + notes, + } +} + +fn finish_unrecognized(notes: Vec) -> Classification { + Classification { + shape: Shape::Unrecognized, + rung_ceiling: RungCeiling::NeedsPerson, + network_swap_candidate: false, + notes, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::{FormRef, FrameRef, StrippedUrl}; + use crate::request::{BodyShape, Encoding, FieldShape, MarkerHit, ValueKind, ValueShape}; + + fn frame() -> FrameRef { + FrameRef { + main: true, + origin: "https://example.test".into(), + cross_origin: false, + } + } + + fn url() -> StrippedUrl { + StrippedUrl { + origin: "https://example.test".into(), + path: "/login".into(), + query_keys: vec![], + } + } + + fn element(id: u32, input_type: &str, in_form: bool) -> ElementDescriptor { + ElementDescriptor { + element_id: id, + tag: "input".into(), + input_type: Some(input_type.into()), + id: None, + name: None, + autocomplete: None, + placeholder: None, + aria_label: None, + was_password: input_type == "password", + form: in_form.then(|| FormRef { + index: 0, + action: None, + method: "post".into(), + }), + selectors: vec![], + in_shadow_root: false, + } + } + + fn input(t: u64, role: Role, in_form: bool) -> Step { + let ty = if role == Role::Password { + "password" + } else { + "text" + }; + Step::Input { + t_ms: t, + frame: frame(), + element: element(t as u32, ty, in_form), + value_len: 6, + role, + } + } + + fn nav(t: u64) -> Step { + Step::Navigation { + t_ms: t, + frame: frame(), + url: url(), + } + } + + fn hit(role: Role, encoding: Encoding) -> MarkerHit { + MarkerHit { + role, + encoding, + low_confidence: false, + } + } + + fn req(t: u64, resource: ResourceKind, hits: Vec) -> StoredRequest { + StoredRequest { + t_ms: t, + method: "POST".into(), + url: url(), + resource, + main_frame_navigation: resource == ResourceKind::Document, + content_type: None, + authorization: None, + url_markers: vec![], + body: BodyShape::Urlencoded { + fields: vec![FieldShape { + key: "k".into(), + value: ValueShape { + kind: ValueKind::String, + len: 6, + markers: hits, + }, + }], + }, + response: None, + } + } + + fn plain_pw() -> MarkerHit { + hit(Role::Password, Encoding::AsTyped) + } + + fn one_page_steps() -> Vec { + vec![ + nav(0), + input(10, Role::Username, true), + input(20, Role::Password, true), + ] + } + + fn clean_form_post() -> Classification { + classify( + &one_page_steps(), + &[req(100, ResourceKind::Document, vec![plain_pw()])], + &Flags::default(), + ) + } + + #[test] + fn form_post() { + let c = clean_form_post(); + assert_eq!(c.shape, Shape::FormPost); + assert_eq!(c.rung_ceiling, RungCeiling::Milliseconds); + assert!(c.network_swap_candidate); + } + + #[test] + fn script_post() { + let c = classify( + &one_page_steps(), + &[req( + 100, + ResourceKind::Fetch, + vec![hit(Role::Password, Encoding::JsonEscaped)], + )], + &Flags::default(), + ); + assert_eq!(c.shape, Shape::ScriptPost); + assert_eq!(c.rung_ceiling, RungCeiling::Milliseconds); + assert!(c.network_swap_candidate); + } + + #[test] + fn two_step_by_navigation() { + let steps = vec![ + nav(0), + input(10, Role::Username, true), + nav(50), + input(60, Role::Password, true), + ]; + let c = classify( + &steps, + &[req(100, ResourceKind::Document, vec![plain_pw()])], + &Flags::default(), + ); + assert_eq!( + c.shape, + Shape::TwoStep { + password_leg: PasswordLeg::FormPost + } + ); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); + assert!(!c.notes.is_empty()); + } + + #[test] + fn two_step_by_request() { + let steps = vec![ + nav(0), + input(10, Role::Username, true), + input(60, Role::Password, true), + ]; + let reqs = [ + req( + 30, + ResourceKind::Xhr, + vec![hit(Role::Username, Encoding::AsTyped)], + ), + req(100, ResourceKind::Fetch, vec![plain_pw()]), + ]; + let c = classify(&steps, &reqs, &Flags::default()); + assert_eq!( + c.shape, + Shape::TwoStep { + password_leg: PasswordLeg::ScriptPost + } + ); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); + } + + #[test] + fn two_step_with_a_transformed_leg_is_agent_continues() { + let steps = vec![ + nav(0), + input(10, Role::Username, true), + nav(50), + input(60, Role::Password, true), + ]; + let c = classify( + &steps, + &[req( + 100, + ResourceKind::Fetch, + vec![hit(Role::Password, Encoding::Sha256Hex)], + )], + &Flags::default(), + ); + assert_eq!( + c.shape, + Shape::TwoStep { + password_leg: PasswordLeg::Transformed + } + ); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); + } + + #[test] + fn transformed_with_digest() { + let c = classify( + &one_page_steps(), + &[req( + 100, + ResourceKind::Fetch, + vec![hit(Role::Password, Encoding::Sha256Hex)], + )], + &Flags::default(), + ); + assert_eq!( + c.shape, + Shape::Transformed { + digest: Some("sha256".into()) + } + ); + assert_eq!(c.rung_ceiling, RungCeiling::Milliseconds); + assert!(!c.network_swap_candidate); + } + + #[test] + fn transformed_unknown() { + let c = classify( + &one_page_steps(), + &[req(100, ResourceKind::Fetch, vec![])], + &Flags::default(), + ); + assert_eq!(c.shape, Shape::Transformed { digest: None }); + assert_eq!(c.rung_ceiling, RungCeiling::NeedsPerson); + assert!(!c.notes.is_empty()); + } + + #[test] + fn password_nowhere_and_body_unavailable_is_unrecognized() { + let flags = Flags { + body_unavailable: true, + ..Flags::default() + }; + let c = classify( + &one_page_steps(), + &[req(100, ResourceKind::Fetch, vec![])], + &flags, + ); + assert_eq!(c.shape, Shape::Unrecognized); + assert_eq!(c.rung_ceiling, RungCeiling::NeedsPerson); + assert!(!c.notes.is_empty()); + } + + #[test] + fn each_blocking_flag_wins() { + let setters: [fn(&mut Flags); 4] = [ + |f| f.cross_site = true, + |f| f.popup = true, + |f| f.framed_login = true, + |f| f.multiple_passwords = true, + ]; + for set in setters { + let mut flags = Flags::default(); + set(&mut flags); + let c = classify( + &one_page_steps(), + &[req(100, ResourceKind::Document, vec![plain_pw()])], + &flags, + ); + assert_eq!(c.shape, Shape::Unrecognized); + assert_eq!(c.rung_ceiling, RungCeiling::NeedsPerson); + assert!(!c.network_swap_candidate); + assert!(!c.notes.is_empty()); + } + } + + #[test] + fn captcha_caps_a_form_post_at_agent_continues() { + let flags = Flags { + captcha: true, + ..Flags::default() + }; + let c = classify( + &one_page_steps(), + &[req(100, ResourceKind::Document, vec![plain_pw()])], + &flags, + ); + assert_eq!(c.shape, Shape::FormPost); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); + assert!(!c.notes.is_empty()); + } + + #[test] + fn otp_step_caps_at_agent_continues() { + let flags = Flags { + otp_step: true, + ..Flags::default() + }; + let c = classify( + &one_page_steps(), + &[req(100, ResourceKind::Document, vec![plain_pw()])], + &flags, + ); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); + assert!(!c.notes.is_empty()); + } + + fn formless_steps() -> Vec { + vec![ + nav(0), + input(10, Role::Username, false), + input(20, Role::Password, false), + ] + } + + #[test] + fn no_way_to_submit_caps_at_agent_continues() { + let c = classify( + &formless_steps(), + &[req(100, ResourceKind::Fetch, vec![plain_pw()])], + &Flags::default(), + ); + assert_eq!(c.shape, Shape::ScriptPost); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); + assert!(!c.notes.is_empty()); + } + + #[test] + fn enter_key_keeps_milliseconds_without_a_form() { + let mut steps = formless_steps(); + steps.push(Step::EnterKey { + t_ms: 30, + frame: frame(), + element: element(2, "password", false), + }); + let c = classify( + &steps, + &[req(100, ResourceKind::Fetch, vec![plain_pw()])], + &Flags::default(), + ); + assert_eq!(c.rung_ceiling, RungCeiling::Milliseconds); + assert!(c.notes.is_empty()); + } + + #[test] + fn two_plain_hits_is_not_a_swap_candidate() { + let c = classify( + &one_page_steps(), + &[ + req(100, ResourceKind::Document, vec![plain_pw()]), + req(110, ResourceKind::Fetch, vec![plain_pw()]), + ], + &Flags::default(), + ); + assert!(!c.network_swap_candidate); + } + + #[test] + fn no_password_step_is_unrecognized() { + let steps = vec![nav(0), input(10, Role::Username, true)]; + let c = classify( + &steps, + &[req(100, ResourceKind::Document, vec![plain_pw()])], + &Flags::default(), + ); + assert_eq!(c.shape, Shape::Unrecognized); + assert_eq!(c.rung_ceiling, RungCeiling::NeedsPerson); + assert!(c.notes.iter().any(|n| n == "no password field was used")); + } + + #[test] + fn otp_descriptor_matches_the_three_tests_and_nothing_else() { + let base = element(1, "text", true); + assert!(!is_otp_descriptor(&base)); + let mut a = base.clone(); + a.autocomplete = Some("one-time-code".into()); + assert!(is_otp_descriptor(&a)); + let mut n = base.clone(); + n.name = Some("User_OTP".into()); + assert!(is_otp_descriptor(&n)); + let t = element(1, "tel", true); + assert!(is_otp_descriptor(&t)); + let mut other = base; + other.autocomplete = Some("username".into()); + other.name = Some("email".into()); + assert!(!is_otp_descriptor(&other)); + } +} diff --git a/crates/kharon-recording/src/lib.rs b/crates/kharon-recording/src/lib.rs index da8c362..87646db 100644 --- a/crates/kharon-recording/src/lib.rs +++ b/crates/kharon-recording/src/lib.rs @@ -4,7 +4,9 @@ pub mod classify; pub mod format; pub mod request; -pub use classify::{Classification, Flags, PasswordLeg, RungCeiling, Shape}; +pub use classify::{ + Classification, Flags, PasswordLeg, RungCeiling, Shape, classify, is_otp_descriptor, +}; pub use format::{ BrowserInfo, ElementDescriptor, EndedBy, FORMAT_VERSION, FormRef, FormatError, FrameRef, ListenerWorld, Recording, Step, StrippedCounts, StrippedUrl, Viewport, diff --git a/crates/kharond/src/cdp/page_js.rs b/crates/kharond/src/cdp/page_js.rs index 6e644cb..4d881a0 100644 --- a/crates/kharond/src/cdp/page_js.rs +++ b/crates/kharond/src/cdp/page_js.rs @@ -121,6 +121,7 @@ function fillInPage(specs, fields, expectedTopOrigin, submitRequested, budgetMs) function resolveField(kind, selector) { if (selector) return document.querySelector(selector); if (kind === "password") return document.querySelector('input[type="password"]'); + // Keep in step with is_otp_descriptor in crates/kharon-recording/src/classify.rs. if (kind === "totp") { return document.querySelector( 'input[autocomplete*="one-time"], input[name*="otp" i], input[type="tel"]' @@ -234,6 +235,7 @@ function clearInPage(fields) { function resolveField(kind, selector) { if (selector) return document.querySelector(selector); if (kind === "password") return document.querySelector('input[type="password"]'); + // Keep in step with is_otp_descriptor in crates/kharon-recording/src/classify.rs. if (kind === "totp") { return document.querySelector( 'input[autocomplete*="one-time"], input[name*="otp" i], input[type="tel"]' From 434073344537d0611e5a9a7f4f1c687cadb38bcc Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:04:35 -0500 Subject: [PATCH 07/36] feat(teach): markers and their search over request bytes (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- Cargo.lock | 24 ++ Cargo.toml | 1 + crates/kharon-recording/src/request.rs | 2 +- crates/kharon-teach/Cargo.toml | 26 ++ crates/kharon-teach/src/lib.rs | 3 + crates/kharon-teach/src/main.rs | 6 + crates/kharon-teach/src/marker.rs | 522 +++++++++++++++++++++++++ 7 files changed, 583 insertions(+), 1 deletion(-) create mode 100644 crates/kharon-teach/Cargo.toml create mode 100644 crates/kharon-teach/src/lib.rs create mode 100644 crates/kharon-teach/src/main.rs create mode 100644 crates/kharon-teach/src/marker.rs diff --git a/Cargo.lock b/Cargo.lock index 9a69396..6e85aaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1079,6 +1079,20 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "kharon-teach" +version = "0.0.0" +dependencies = [ + "base64ct", + "hex", + "kharon-recording", + "md-5", + "percent-encoding", + "sha1", + "sha2", + "zeroize", +] + [[package]] name = "kharond" version = "0.0.0" @@ -1175,6 +1189,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.3" diff --git a/Cargo.toml b/Cargo.toml index e04dbec..60369e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/kharon-connector", "crates/kharon-agent-connector", "crates/kharon-recording", + "crates/kharon-teach", ] # SPEC §8 process hardening: a release build aborts on panic instead of diff --git a/crates/kharon-recording/src/request.rs b/crates/kharon-recording/src/request.rs index a1146f6..d0f850a 100644 --- a/crates/kharon-recording/src/request.rs +++ b/crates/kharon-recording/src/request.rs @@ -56,7 +56,7 @@ impl Encoding { pub struct MarkerHit { pub role: Role, pub encoding: Encoding, - /// True when the match is weak (short value, or a common substring), so a reader treats it as a hint. + /// True when the match is weak (the marker is shorter than `LOW_CONFIDENCE_BELOW` characters), so a reader treats it as a hint. pub low_confidence: bool, } diff --git a/crates/kharon-teach/Cargo.toml b/crates/kharon-teach/Cargo.toml new file mode 100644 index 0000000..fb23020 --- /dev/null +++ b/crates/kharon-teach/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "kharon-teach" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true +description = "Teach-mode recorder for kharon (#361)" + +[lib] +name = "kharon_teach" +path = "src/lib.rs" + +[[bin]] +name = "kharon-teach" +path = "src/main.rs" + +[dependencies] +kharon-recording = { path = "../kharon-recording" } +zeroize = "1" +sha1 = "0.10" +sha2 = "0.10" +md-5 = "0.10" +base64ct = { version = "1", features = ["alloc"] } +hex = "0.4" +percent-encoding = "2" diff --git a/crates/kharon-teach/src/lib.rs b/crates/kharon-teach/src/lib.rs new file mode 100644 index 0000000..c07839b --- /dev/null +++ b/crates/kharon-teach/src/lib.rs @@ -0,0 +1,3 @@ +//! Teach-mode recorder for kharon (#361). + +pub mod marker; diff --git a/crates/kharon-teach/src/main.rs b/crates/kharon-teach/src/main.rs new file mode 100644 index 0000000..5e58dc9 --- /dev/null +++ b/crates/kharon-teach/src/main.rs @@ -0,0 +1,6 @@ +use std::process::ExitCode; + +fn main() -> ExitCode { + eprintln!("kharon-teach: not implemented"); + ExitCode::from(1) +} diff --git a/crates/kharon-teach/src/marker.rs b/crates/kharon-teach/src/marker.rs new file mode 100644 index 0000000..6ed027c --- /dev/null +++ b/crates/kharon-teach/src/marker.rs @@ -0,0 +1,522 @@ +//! Typed markers: a value the user typed, held only in memory, searched for in request bytes +//! under the encodings a site might apply. A `Marker` can never be printed. + +use base64ct::{Base64, Base64Unpadded, Base64Url, Base64UrlUnpadded, Encoding as B64}; +use kharon_recording::{Encoding, MarkerHit, Role}; +use md5::Md5; +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode}; +use sha1::Sha1; +use sha2::{Digest, Sha256, Sha512}; +use zeroize::Zeroizing; + +/// Markers shorter than this many characters produce low-confidence hits. +pub const LOW_CONFIDENCE_BELOW: u32 = 6; + +/// A typed value and its role. It implements none of `Debug`, `Display`, `Serialize` or `Clone`. +/// +/// ```compile_fail +/// let m = kharon_teach::marker::Marker::new(zeroize::Zeroizing::new(String::from("x")), kharon_recording::Role::Password); +/// println!("{:?}", m); +/// ``` +pub struct Marker { + value: Zeroizing, + role: Role, +} + +impl Marker { + pub fn new(value: Zeroizing, role: Role) -> Self { + Self { value, role } + } + + pub fn role(&self) -> Role { + self.role + } + + /// Length in characters. + pub fn len(&self) -> u32 { + u32::try_from(self.value.chars().count()).unwrap_or(u32::MAX) + } + + pub fn is_empty(&self) -> bool { + self.value.is_empty() + } + + /// Whether both markers hold the same value (role is ignored). Not constant time. + pub fn same_value(&self, other: &Marker) -> bool { + self.value.as_str() == other.value.as_str() + } + + pub fn with_role(self, role: Role) -> Self { + Self { role, ..self } + } +} + +type Needle = Zeroizing>; + +/// RFC 3986 unreserved characters stay as they are; every other byte is percent-encoded. +const UNRESERVED: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~'); + +/// Every way `marker` occurs in `haystack`. Digests are searched only for Role::Password. +/// +/// At most one hit per `Encoding`, in the order of the enum's variants. An encoded form that is +/// byte-identical to the as-typed form is not searched, so a plain value reports only `AsTyped`. +pub fn search(haystack: &[u8], marker: &Marker) -> Vec { + if marker.is_empty() { + return Vec::new(); + } + let low_confidence = marker.len() < LOW_CONFIDENCE_BELOW; + needles(marker) + .into_iter() + .filter(|(_, forms)| forms.iter().any(|n| contains(haystack, n))) + .map(|(encoding, _)| MarkerHit { + role: marker.role, + encoding, + low_confidence, + }) + .collect() +} + +/// The base64 and base64url (padded or not) of `username:password`. A match returns two hits, +/// one per role, both `Encoding::Base64` (or `Base64Url`). +/// +/// Nothing is returned when either marker is empty. `low_confidence` is set when the combined +/// length of both values is under `LOW_CONFIDENCE_BELOW` characters. +pub fn search_pair(haystack: &[u8], username: &Marker, password: &Marker) -> Vec { + if username.is_empty() || password.is_empty() { + return Vec::new(); + } + let mut joined = Zeroizing::new(String::with_capacity( + username.value.len() + password.value.len() + 1, + )); + joined.push_str(&username.value); + joined.push(':'); + joined.push_str(&password.value); + let low_confidence = username.len().saturating_add(password.len()) < LOW_CONFIDENCE_BELOW; + let (standard, url_safe) = base64_forms(joined.as_bytes()); + let url_safe = { + let seen: Vec<&[u8]> = standard.iter().map(|n| n.as_slice()).collect(); + dedup(url_safe, &seen) + }; + let mut hits = Vec::new(); + for (encoding, forms) in [ + (Encoding::Base64, standard), + (Encoding::Base64Url, url_safe), + ] { + if forms.iter().any(|n| contains(haystack, n)) { + for role in [username.role, password.role] { + hits.push(MarkerHit { + role, + encoding, + low_confidence, + }); + } + } + } + hits +} + +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() && haystack.windows(needle.len()).any(|w| w == needle) +} + +/// The search forms per encoding, in `Encoding` order. +fn needles(marker: &Marker) -> Vec<(Encoding, Vec)> { + let value = marker.value.as_bytes(); + let (standard, url_safe) = base64_forms(value); + let standard = dedup(standard, &[value]); + let url_safe = { + let mut seen: Vec<&[u8]> = vec![value]; + seen.extend(standard.iter().map(|n| n.as_slice())); + dedup(url_safe, &seen) + }; + let mut out = vec![ + (Encoding::AsTyped, vec![Zeroizing::new(value.to_vec())]), + (Encoding::Percent, dedup(percent_forms(value), &[value])), + ( + Encoding::JsonEscaped, + dedup(json_forms(&marker.value), &[value]), + ), + (Encoding::Base64, standard), + (Encoding::Base64Url, url_safe), + (Encoding::Hex, hex_forms(value)), + ]; + if marker.role == Role::Password { + let digests: [(Encoding, Encoding, Vec); 4] = [ + ( + Encoding::Md5Hex, + Encoding::Md5Base64, + Md5::digest(value).to_vec(), + ), + ( + Encoding::Sha1Hex, + Encoding::Sha1Base64, + Sha1::digest(value).to_vec(), + ), + ( + Encoding::Sha256Hex, + Encoding::Sha256Base64, + Sha256::digest(value).to_vec(), + ), + ( + Encoding::Sha512Hex, + Encoding::Sha512Base64, + Sha512::digest(value).to_vec(), + ), + ]; + for (hex_encoding, b64_encoding, digest) in digests { + out.push((hex_encoding, hex_forms(&digest))); + out.push((b64_encoding, base64_forms(&digest).0)); + } + } + out +} + +/// Drops empty forms, forms equal to one in `seen`, and repeats. +fn dedup(forms: Vec, seen: &[&[u8]]) -> Vec { + let mut kept: Vec = Vec::new(); + for form in forms { + let known = form.is_empty() + || seen.iter().any(|s| *s == form.as_slice()) + || kept.iter().any(|k| k.as_slice() == form.as_slice()); + if !known { + kept.push(form); + } + } + kept +} + +fn encode_b64(bytes: &[u8]) -> Needle { + let mut buf = Zeroizing::new(vec![0u8; E::encoded_len(bytes)]); + if E::encode(bytes, &mut buf).is_err() { + // The buffer is sized by `encoded_len`, so this cannot happen; match nothing if it does. + buf.clear(); + } + buf +} + +/// (standard, URL-safe), each padded then unpadded. +fn base64_forms(bytes: &[u8]) -> (Vec, Vec) { + ( + vec![ + encode_b64::(bytes), + encode_b64::(bytes), + ], + vec![ + encode_b64::(bytes), + encode_b64::(bytes), + ], + ) +} + +/// Lowercase then uppercase. +fn hex_forms(bytes: &[u8]) -> Vec { + let mut lower = Zeroizing::new(vec![0u8; bytes.len() * 2]); + if hex::encode_to_slice(bytes, &mut lower).is_err() { + lower.clear(); + } + let mut upper = lower.clone(); + upper.make_ascii_uppercase(); + vec![lower, upper] +} + +/// Uppercase and lowercase percent hex, each with space as `%20` and as `+`. +fn percent_forms(bytes: &[u8]) -> Vec { + let mut forms = Vec::new(); + for plus in [false, true] { + for lower in [false, true] { + let mut out = Zeroizing::new(Vec::with_capacity(bytes.len() * 3)); + for chunk in percent_encode(bytes, UNRESERVED) { + if plus && chunk == "%20" { + out.push(b'+'); + } else if lower && chunk.starts_with('%') { + out.extend(chunk.bytes().map(|b| b.to_ascii_lowercase())); + } else { + out.extend_from_slice(chunk.as_bytes()); + } + } + forms.push(out); + } + } + forms +} + +#[derive(Clone, Copy)] +enum JsonMode { + /// Non-ASCII characters stay as UTF-8. + Raw, + /// Non-ASCII characters become backslash-u escapes, with hex digits in the given case. + Ascii { upper: bool }, +} + +/// The content between the quotes as `serde_json` writes it (raw non-ASCII), and the form with +/// every non-ASCII character as a backslash-u escape in each hex case. +fn json_forms(value: &str) -> Vec { + [ + JsonMode::Raw, + JsonMode::Ascii { upper: false }, + JsonMode::Ascii { upper: true }, + ] + .into_iter() + .map(|mode| json_escape(value, mode)) + .collect() +} + +fn push_u_escape(out: &mut Vec, unit: u16, upper: bool) { + let digits: &[u8; 16] = if upper { + b"0123456789ABCDEF" + } else { + b"0123456789abcdef" + }; + out.extend_from_slice(b"\\u"); + for shift in [12, 8, 4, 0] { + out.push(digits[usize::from((unit >> shift) & 0xf)]); + } +} + +fn json_escape(value: &str, mode: JsonMode) -> Needle { + let upper = matches!(mode, JsonMode::Ascii { upper: true }); + let mut out = Zeroizing::new(Vec::with_capacity(value.len() * 6)); + for c in value.chars() { + match c { + '"' => out.extend_from_slice(b"\\\""), + '\\' => out.extend_from_slice(b"\\\\"), + '\n' => out.extend_from_slice(b"\\n"), + '\r' => out.extend_from_slice(b"\\r"), + '\t' => out.extend_from_slice(b"\\t"), + '\u{8}' => out.extend_from_slice(b"\\b"), + '\u{c}' => out.extend_from_slice(b"\\f"), + c if u32::from(c) < 0x20 => push_u_escape(&mut out, u16::from(c as u8), upper), + c if c.is_ascii() => out.push(c as u8), + c => match mode { + JsonMode::Raw => { + let mut buf = [0u8; 4]; + out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes()); + } + JsonMode::Ascii { .. } => { + let mut units = [0u16; 2]; + for unit in c.encode_utf16(&mut units) { + push_u_escape(&mut out, *unit, upper); + } + } + }, + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pw(s: &str) -> Marker { + Marker::new(Zeroizing::new(s.to_owned()), Role::Password) + } + fn user(s: &str) -> Marker { + Marker::new(Zeroizing::new(s.to_owned()), Role::Username) + } + /// `@` stands for a backslash, so escape sequences stay literal in the source. + fn bs(s: &str) -> String { + s.replace('@', "\\") + } + fn encodings(h: &[u8], m: &Marker) -> Vec { + search(h, m).into_iter().map(|x| x.encoding).collect() + } + + #[test] + fn as_typed() { + assert_eq!( + encodings(b"user=a&password=Tr0ub4dor-9x&x=1", &pw("Tr0ub4dor-9x")), + [Encoding::AsTyped] + ); + } + + #[test] + fn percent_encoded() { + assert_eq!( + encodings(b"p=a%20b%26c%2Bd12", &pw("a b&c+d12")), + [Encoding::Percent] + ); + } + + #[test] + fn percent_encoded_lowercase_hex() { + assert_eq!( + encodings(b"p=a%2fb%26c%2bd12", &pw("a/b&c+d12")), + [Encoding::Percent] + ); + } + + #[test] + fn percent_encoded_plus_for_space() { + assert_eq!( + encodings(b"p=a+b%26c%2Bd12", &pw("a b&c+d12")), + [Encoding::Percent] + ); + } + + #[test] + fn json_escaped() { + assert_eq!( + encodings(br#"{"p":"a\"b\\c12"}"#, &pw(r#"a"b\c12"#)), + [Encoding::JsonEscaped] + ); + } + + #[test] + fn json_escaped_control_characters() { + assert_eq!( + encodings(br#"{"p":"a\nb\u0001c12"}"#, &pw("a\nb\u{1}c12")), + [Encoding::JsonEscaped] + ); + } + + #[test] + fn base64_and_base64url() { + assert_eq!( + encodings(b"x=Pj4+Pz8/MTI=", &pw(">>>???12")), + [Encoding::Base64] + ); + // unpadded is found too + assert_eq!( + encodings(b"x=Pj4-Pz8_MTI", &pw(">>>???12")), + [Encoding::Base64Url] + ); + } + + #[test] + fn hex_either_case() { + let lower = hex::encode("Tr0ub4dor-9x"); + let upper = lower.to_uppercase(); + assert_eq!( + encodings(lower.as_bytes(), &pw("Tr0ub4dor-9x")), + [Encoding::Hex] + ); + assert_eq!( + encodings(upper.as_bytes(), &pw("Tr0ub4dor-9x")), + [Encoding::Hex] + ); + } + + #[test] + fn sha256_hex_of_a_password() { + let digest = hex::encode(Sha256::digest(b"Tr0ub4dor-9x")); + let haystack = format!("h={digest}"); + assert_eq!( + encodings(haystack.as_bytes(), &pw("Tr0ub4dor-9x")), + [Encoding::Sha256Hex] + ); + } + + #[test] + fn digests_are_not_searched_for_a_username() { + let digest = hex::encode(Sha256::digest(b"Tr0ub4dor-9x")); + let haystack = format!("h={digest}"); + assert!(search(haystack.as_bytes(), &user("Tr0ub4dor-9x")).is_empty()); + } + + #[test] + fn short_marker_is_low_confidence() { + assert!(search(b"x=abc12", &pw("abc12"))[0].low_confidence); + assert!(!search(b"x=abc123", &pw("abc123"))[0].low_confidence); + } + + #[test] + fn empty_marker_matches_nothing() { + assert!(search(b"anything", &pw("")).is_empty()); + } + + #[test] + fn hit_carries_the_marker_role() { + let hits = search(b"u=kh.throwaway7", &user("kh.throwaway7")); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].role, Role::Username); + } + + #[test] + fn non_ascii_percent_encoded() { + assert_eq!( + encodings( + b"p=p%C3%A4ssw%C3%B6rd%E2%9C%9312", + &pw("p\u{e4}ssw\u{f6}rd\u{2713}12") + ), + [Encoding::Percent] + ); + } + + #[test] + fn non_ascii_json_escaped_both_forms() { + let m = pw("p\u{e4}ssw\u{f6}rd\u{2713}12"); + let raw = "{\"p\":\"p\u{e4}ssw\u{f6}rd\u{2713}12\"}"; + assert_eq!(encodings(raw.as_bytes(), &m), [Encoding::AsTyped]); + assert_eq!( + encodings(bs("{\"p\":\"p@u00e4ssw@u00f6rd@u271312\"}").as_bytes(), &m), + [Encoding::JsonEscaped] + ); + assert_eq!( + encodings(bs("{\"p\":\"p@u00E4ssw@u00F6rd@u271312\"}").as_bytes(), &m), + [Encoding::JsonEscaped] + ); + } + + #[test] + fn json_surrogate_pair() { + assert_eq!( + encodings( + bs("{\"p\":\"pw@ud83d@ude00xyz\"}").as_bytes(), + &pw("pw\u{1f600}xyz") + ), + [Encoding::JsonEscaped] + ); + } + + #[test] + fn two_hundred_characters() { + let value = "Zq9".repeat(66) + "ab"; + assert_eq!(value.chars().count(), 200); + let haystack = format!("a=1&p={value}&b=2"); + assert_eq!( + encodings(haystack.as_bytes(), &pw(&value)), + [Encoding::AsTyped] + ); + } + + #[test] + fn pair_is_found_where_neither_value_alone_is() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let haystack = format!(r#"{{"auth":"{b64}"}}"#); + let u = user("kh.throwaway7"); + let p = pw("Tr0ub4dor-9x"); + assert!(search(haystack.as_bytes(), &u).is_empty()); + assert!(search(haystack.as_bytes(), &p).is_empty()); + let hits = search_pair(haystack.as_bytes(), &u, &p); + assert_eq!( + hits.iter() + .map(|h| (h.role, h.encoding)) + .collect::>(), + [ + (Role::Username, Encoding::Base64), + (Role::Password, Encoding::Base64) + ] + ); + } + + #[test] + fn pair_absent_gives_nothing() { + assert!( + search_pair(b"nothing here", &user("kh.throwaway7"), &pw("Tr0ub4dor-9x")).is_empty() + ); + } + + #[test] + fn same_value_and_with_role() { + let a = pw("abc"); + let b = user("abc"); + assert!(a.same_value(&b)); + assert_eq!(a.with_role(Role::None).role(), Role::None); + assert_eq!(pw("h\u{e4}").len(), 2); + } +} From 056ca5a565f2f64b5fb06000e3fdf23b3939bcf1 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:07:47 -0500 Subject: [PATCH 08/36] feat(teach): find a marker in a host name without regard to case (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-teach/src/marker.rs | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/crates/kharon-teach/src/marker.rs b/crates/kharon-teach/src/marker.rs index 6ed027c..d35e126 100644 --- a/crates/kharon-teach/src/marker.rs +++ b/crates/kharon-teach/src/marker.rs @@ -49,6 +49,24 @@ impl Marker { pub fn with_role(self, role: Role) -> Self { Self { role, ..self } } + + /// Byte range of the first place this marker's value occurs in `haystack`, comparing ASCII letters + /// without regard to case. `None` for an empty marker. Used for host names, which arrive lowercased. + /// + /// Compares byte windows in place, so no lowercase copy of the value is ever made. Non-ASCII + /// bytes must match exactly. The range starts and ends on character boundaries: a match + /// begins with an ASCII byte or a UTF-8 lead byte, and ends with a complete character. + pub fn find_ignore_ascii_case(&self, haystack: &str) -> Option> { + let needle = self.value.as_bytes(); + if needle.is_empty() { + return None; + } + haystack + .as_bytes() + .windows(needle.len()) + .position(|w| w.eq_ignore_ascii_case(needle)) + .map(|start| start..start + needle.len()) + } } type Needle = Zeroizing>; @@ -519,4 +537,33 @@ mod tests { assert_eq!(a.with_role(Role::None).role(), Role::None); assert_eq!(pw("h\u{e4}").len(), 2); } + + #[test] + fn find_ignore_ascii_case_in_a_lowercased_host() { + let m = user("Kh.Throwaway7"); + assert_eq!( + m.find_ignore_ascii_case("a.kh.throwaway7.ex.test"), + Some(2..15) + ); + } + + #[test] + fn find_ignore_ascii_case_no_match_and_empty() { + assert_eq!( + user("kh.throwaway7").find_ignore_ascii_case("ex.test"), + None + ); + assert_eq!(user("longer-than-host").find_ignore_ascii_case("a"), None); + assert_eq!(user("").find_ignore_ascii_case("ex.test"), None); + } + + #[test] + fn find_ignore_ascii_case_non_ascii_is_byte_exact() { + let m = user("M\u{fc}ller1"); + assert_eq!(m.find_ignore_ascii_case("x-m\u{fc}ller1"), Some(2..10)); + assert_eq!( + user("\u{c9}t\u{e9}12").find_ignore_ascii_case("\u{e9}t\u{e9}12"), + None + ); + } } From 64166461fae74f35455d40c6915711c2a7cfcaf1 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:10:07 -0500 Subject: [PATCH 09/36] feat(teach): strip URLs and reduce request bodies to shapes (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- Cargo.lock | 2 + crates/kharon-teach/Cargo.toml | 2 + crates/kharon-teach/src/body.rs | 796 +++++++++++++++++++++++++++++++ crates/kharon-teach/src/lib.rs | 2 + crates/kharon-teach/src/strip.rs | 433 +++++++++++++++++ 5 files changed, 1235 insertions(+) create mode 100644 crates/kharon-teach/src/body.rs create mode 100644 crates/kharon-teach/src/strip.rs diff --git a/Cargo.lock b/Cargo.lock index 6e85aaf..9fbe748 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1088,8 +1088,10 @@ dependencies = [ "kharon-recording", "md-5", "percent-encoding", + "serde_json", "sha1", "sha2", + "url", "zeroize", ] diff --git a/crates/kharon-teach/Cargo.toml b/crates/kharon-teach/Cargo.toml index fb23020..c8b1694 100644 --- a/crates/kharon-teach/Cargo.toml +++ b/crates/kharon-teach/Cargo.toml @@ -24,3 +24,5 @@ md-5 = "0.10" base64ct = { version = "1", features = ["alloc"] } hex = "0.4" percent-encoding = "2" +url = "2" +serde_json = "1" diff --git a/crates/kharon-teach/src/body.rs b/crates/kharon-teach/src/body.rs new file mode 100644 index 0000000..f62f44f --- /dev/null +++ b/crates/kharon-teach/src/body.rs @@ -0,0 +1,796 @@ +//! Request bodies reduced to shapes: where a typed value sat, never what it was. +//! +//! The site's JSON body is read as a `serde_json::Value` here, the one place a site payload may +//! be, while its shape is extracted. Known limit: the parsed `Value` holds typed values in +//! ordinary `String`s that serde_json does not zeroize. + +use kharon_recording::{BodyShape, FieldShape, JsonField, JsonShape, ValueKind, ValueShape}; +use percent_encoding::percent_decode; +use zeroize::Zeroizing; + +use crate::marker::Marker; +use crate::strip::{TYPED, any_hit, hits_in}; + +/// Bodies larger than this are not parsed: they are `Opaque`. +pub const MAX_BODY_BYTES: usize = 1 << 20; + +/// `content_type` picks the reader: application/x-www-form-urlencoded, application/json (and any +json), +/// multipart/form-data; anything else, or a body that fails to parse as its declared type, is Opaque. +/// `pair` is (username, password) once both are known; see below. +/// +/// Values are searched after decoding (percent-decoded urlencoded value with `+` as space, a JSON +/// string's content, a JSON number's text, a multipart text part's bytes). An `Opaque` body is +/// searched as raw bytes. A name (key, part name) holding a typed value is written as `{typed}`. +/// Never returns `None` or `Unavailable`. +pub fn shape_of_body( + content_type: Option<&str>, + body: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> BodyShape { + let parsed = match content_type.map(ContentType::parse) { + Some(ct) if body.len() <= MAX_BODY_BYTES => match ct.essence.as_str() { + "application/x-www-form-urlencoded" => Some(BodyShape::Urlencoded { + fields: urlencoded_fields(body, markers, pair), + }), + "multipart/form-data" => ct + .boundary + .as_deref() + .and_then(|b| multipart_parts(body, b, markers, pair)) + .map(|parts| BodyShape::Multipart { parts }), + e if e == "application/json" || e.ends_with("+json") => { + // The one permitted use of `serde_json::Value`: a site's JSON body while its shape is extracted. + serde_json::from_slice::(body) + .ok() + .map(|v| BodyShape::Json { + root: json_shape(&v, markers, pair), + }) + } + _ => None, + }, + _ => None, + }; + parsed.unwrap_or_else(|| BodyShape::Opaque { + len: u64::try_from(body.len()).unwrap_or(u64::MAX), + markers: hits_in(body, markers, pair), + }) +} + +struct ContentType { + /// Lowercased, without parameters. + essence: String, + boundary: Option, +} + +impl ContentType { + fn parse(raw: &str) -> Self { + let (essence, params) = raw.split_once(';').unwrap_or((raw, "")); + let boundary = parameters(params) + .into_iter() + .find(|(k, _)| k == "boundary") + .map(|(_, v)| v) + .filter(|v| !v.is_empty()); + Self { + essence: essence.trim().to_ascii_lowercase(), + boundary, + } + } +} + +/// `; key=value; key="quoted value"` pairs, keys lowercased. Quotes may hold `;` and `\`-escapes. +fn parameters(text: &str) -> Vec<(String, String)> { + let mut out = Vec::new(); + let mut chars = text.chars().peekable(); + loop { + let mut key = String::new(); + while let Some(c) = chars.next_if(|c| *c != '=' && *c != ';') { + key.push(c); + } + let mut value = String::new(); + if chars.next_if_eq(&'=').is_some() { + if chars.next_if_eq(&'"').is_some() { + while let Some(c) = chars.next() { + match c { + '"' => break, + '\\' => value.extend(chars.next()), + c => value.push(c), + } + } + // Anything up to the next `;` after the closing quote is ignored. + while chars.next_if(|c| *c != ';').is_some() {} + } else { + while let Some(c) = chars.next_if(|c| *c != ';') { + value.push(c); + } + } + } + let key = key.trim().to_ascii_lowercase(); + if !key.is_empty() { + out.push((key, value.trim().to_owned())); + } + if chars.next().is_none() { + return out; + } + } +} + +/// Characters in `bytes` read as UTF-8, each invalid run counting as one. Makes no copy. +fn char_len(bytes: &[u8]) -> u32 { + let n: usize = bytes + .utf8_chunks() + .map(|c| c.valid().chars().count() + usize::from(!c.invalid().is_empty())) + .sum(); + u32::try_from(n).unwrap_or(u32::MAX) +} + +fn byte_len(bytes: &[u8]) -> u32 { + u32::try_from(bytes.len()).unwrap_or(u32::MAX) +} + +fn value_shape( + kind: ValueKind, + len: u32, + searched: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> ValueShape { + ValueShape { + kind, + len, + markers: hits_in(searched, markers, pair), + } +} + +fn text_value(bytes: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> ValueShape { + value_shape(ValueKind::String, char_len(bytes), bytes, markers, pair) +} + +/// A name that is kept only when it holds no typed value. +fn name(bytes: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> String { + if any_hit(bytes, markers, pair) { + TYPED.to_owned() + } else { + String::from_utf8_lossy(bytes).into_owned() + } +} + +/// Form decoding: `+` is a space, then percent-decoding. A lone `%` stays as it is. +fn form_decode(raw: &[u8]) -> Zeroizing> { + let spaced: Zeroizing> = Zeroizing::new( + raw.iter() + .map(|b| if *b == b'+' { b' ' } else { *b }) + .collect(), + ); + Zeroizing::new(percent_decode(&spaced).collect()) +} + +fn urlencoded_fields( + body: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> Vec { + body.split(|b| *b == b'&') + .filter(|piece| !piece.is_empty()) + .map(|piece| { + let (k, v) = match piece.iter().position(|b| *b == b'=') { + Some(i) => (&piece[..i], &piece[i + 1..]), + None => (piece, &piece[..0]), + }; + let key = form_decode(k); + let value = form_decode(v); + FieldShape { + key: name(&key, markers, pair), + value: text_value(&value, markers, pair), + } + }) + .collect() +} + +fn json_shape( + v: &serde_json::Value, + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> JsonShape { + use serde_json::Value; + // Depth is bounded by serde_json's own recursion limit (128) at parse time. + match v { + Value::Object(map) => JsonShape::Object { + fields: map + .iter() + .map(|(k, v)| JsonField { + key: name(k.as_bytes(), markers, pair), + value: json_shape(v, markers, pair), + }) + .collect(), + }, + Value::Array(items) => JsonShape::Array { + items: items.iter().map(|i| json_shape(i, markers, pair)).collect(), + }, + Value::String(s) => JsonShape::Value { + value: text_value(s.as_bytes(), markers, pair), + }, + Value::Number(n) => { + let text = Zeroizing::new(n.to_string()); + JsonShape::Value { + value: value_shape( + ValueKind::Number, + char_len(text.as_bytes()), + text.as_bytes(), + markers, + pair, + ), + } + } + Value::Bool(b) => JsonShape::Value { + value: value_shape( + ValueKind::Bool, + 0, + if *b { b"true" } else { b"false" }, + markers, + pair, + ), + }, + Value::Null => JsonShape::Value { + value: value_shape(ValueKind::Null, 0, b"null", markers, pair), + }, + } +} + +fn find(haystack: &[u8], needle: &[u8], from: usize) -> Option { + if needle.is_empty() || from > haystack.len() { + return None; + } + haystack[from..] + .windows(needle.len()) + .position(|w| w == needle) + .map(|i| from + i) +} + +/// Parts of a `multipart/form-data` body, or `None` when it is not well formed (no opening +/// delimiter, a part with no header end, or a part not closed by a delimiter). +fn multipart_parts( + body: &[u8], + boundary: &str, + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> Option> { + let mut delim = b"--".to_vec(); + delim.extend_from_slice(boundary.as_bytes()); + // The delimiter that closes a part is preceded by a line break. + let mut crlf_delim = b"\r\n".to_vec(); + crlf_delim.extend_from_slice(&delim); + let mut lf_delim = b"\n".to_vec(); + lf_delim.extend_from_slice(&delim); + + let mut pos = find(body, &delim, 0)? + delim.len(); + let mut parts = Vec::new(); + loop { + if body[pos..].starts_with(b"--") { + return Some(parts); + } + // Rest of the delimiter line (transport padding), then the headers. + let headers_start = pos + body[pos..].iter().position(|b| *b == b'\n')? + 1; + let (headers_end, content_start) = match ( + find(body, b"\r\n\r\n", headers_start), + find(body, b"\n\n", headers_start), + ) { + (Some(a), Some(b)) if b < a => (b, b + 2), + (Some(a), _) => (a, a + 4), + (None, Some(b)) => (b, b + 2), + (None, None) => return None, + }; + let headers = &body[headers_start..headers_end]; + let (content_end, next) = match ( + find(body, &crlf_delim, content_start.saturating_sub(2)), + find(body, &lf_delim, content_start.saturating_sub(1)), + ) { + (Some(a), Some(b)) if b < a => (b, b + 1), + (Some(a), _) => (a, a + 2), + (None, Some(b)) => (b, b + 1), + (None, None) => return None, + }; + let content = body.get(content_start..content_end).unwrap_or(&[]); + parts.push(part_shape(headers, content, markers, pair)); + pos = next + delim.len(); + } +} + +fn part_shape( + headers: &[u8], + content: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> FieldShape { + let mut part_name = Vec::new(); + let mut is_file = false; + for line in headers.split(|b| *b == b'\n') { + let line = String::from_utf8_lossy(line); + let Some((header, rest)) = line.split_once(':') else { + continue; + }; + if !header.trim().eq_ignore_ascii_case("content-disposition") { + continue; + } + for (key, value) in parameters(rest) { + match key.as_str() { + "name" => part_name = value.into_bytes(), + "filename" | "filename*" => is_file = true, + _ => {} + } + } + } + let value = if is_file { + ValueShape { + kind: ValueKind::File, + len: byte_len(content), + markers: Vec::new(), + } + } else { + text_value(content, markers, pair) + }; + FieldShape { + key: name(&part_name, markers, pair), + value, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::strip::{reduce_authorization, strip_url}; + use base64ct::{Base64, Encoding as B64}; + use kharon_recording::{Encoding, MarkerHit, Role}; + + fn user(s: &str) -> Marker { + Marker::new(Zeroizing::new(s.to_owned()), Role::Username) + } + fn pw(s: &str) -> Marker { + Marker::new(Zeroizing::new(s.to_owned()), Role::Password) + } + fn markers() -> Vec { + vec![user("kh.throwaway7"), pw("Tr0ub4dor-9x")] + } + fn hit(role: Role, encoding: Encoding) -> MarkerHit { + MarkerHit { + role, + encoding, + low_confidence: false, + } + } + fn shape(ct: &str, body: &[u8]) -> BodyShape { + shape_of_body(Some(ct), body, &markers(), None) + } + fn vs(kind: ValueKind, len: u32, markers: Vec) -> ValueShape { + ValueShape { kind, len, markers } + } + fn field(key: &str, value: ValueShape) -> FieldShape { + FieldShape { + key: key.into(), + value, + } + } + fn leaf(value: ValueShape) -> JsonShape { + JsonShape::Value { value } + } + fn jf(key: &str, value: JsonShape) -> JsonField { + JsonField { + key: key.into(), + value, + } + } + const URLENC: &str = "application/x-www-form-urlencoded"; + + #[test] + fn urlencoded_fields_with_roles() { + let s = shape( + URLENC, + b"user=kh.throwaway7&password=Tr0ub4dor-9x&remember=1", + ); + assert_eq!( + s, + BodyShape::Urlencoded { + fields: vec![ + field( + "user", + vs( + ValueKind::String, + 13, + vec![hit(Role::Username, Encoding::AsTyped)] + ) + ), + field( + "password", + vs( + ValueKind::String, + 12, + vec![hit(Role::Password, Encoding::AsTyped)] + ) + ), + field("remember", vs(ValueKind::String, 1, vec![])), + ] + } + ); + } + + #[test] + fn urlencoded_value_is_searched_after_decoding() { + let s = shape(URLENC, b"password=Tr0ub4dor%2D9x&q=a+b%zz"); + let BodyShape::Urlencoded { fields } = s else { + panic!("not urlencoded") + }; + assert_eq!( + fields[0].value.markers, + [hit(Role::Password, Encoding::AsTyped)] + ); + // "+" is a space; a lone "%zz" stays as typed: "a b%zz" is 6 characters. + assert_eq!(fields[1].value.len, 6); + } + + #[test] + fn json_nested() { + let s = shape( + "Application/JSON; charset=utf-8", + br#"{"auth":{"id":"kh.throwaway7","secret":"Tr0ub4dor-9x"},"n":3,"ok":true,"z":null}"#, + ); + let BodyShape::Json { root } = s else { + panic!("not json") + }; + let JsonShape::Object { mut fields } = root else { + panic!("not an object") + }; + fields.sort_by(|a, b| a.key.cmp(&b.key)); + let want = vec![ + jf( + "auth", + JsonShape::Object { + fields: vec![ + jf( + "id", + leaf(vs( + ValueKind::String, + 13, + vec![hit(Role::Username, Encoding::AsTyped)], + )), + ), + jf( + "secret", + leaf(vs( + ValueKind::String, + 12, + vec![hit(Role::Password, Encoding::AsTyped)], + )), + ), + ], + }, + ), + jf("n", leaf(vs(ValueKind::Number, 1, vec![]))), + jf("ok", leaf(vs(ValueKind::Bool, 0, vec![]))), + jf("z", leaf(vs(ValueKind::Null, 0, vec![]))), + ]; + let JsonShape::Object { fields: mut inner } = fields[0].value.clone() else { + panic!() + }; + inner.sort_by(|a, b| a.key.cmp(&b.key)); + fields[0].value = JsonShape::Object { fields: inner }; + assert_eq!(fields, want); + } + + #[test] + fn json_arrays_and_scalars_and_plus_json() { + let s = shape("application/vnd.api+json", br#"["Tr0ub4dor-9x", 12]"#); + assert_eq!( + s, + BodyShape::Json { + root: JsonShape::Array { + items: vec![ + leaf(vs( + ValueKind::String, + 12, + vec![hit(Role::Password, Encoding::AsTyped)] + )), + leaf(vs(ValueKind::Number, 2, vec![])), + ] + } + } + ); + } + + #[test] + fn a_json_number_holding_a_typed_value_is_a_hit() { + let m = [pw("424242")]; + let s = shape_of_body(Some("application/json"), br#"{"pin":424242}"#, &m, None); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = s + else { + panic!() + }; + assert_eq!( + fields[0].value, + leaf(vs( + ValueKind::Number, + 6, + vec![hit(Role::Password, Encoding::AsTyped)] + )) + ); + } + + #[test] + fn graphql_is_json() { + let s = shape( + "application/json", + br#"{"query":"mutation Login($password: String!) { login(password: $password) }","variables":{"password":"Tr0ub4dor-9x"}}"#, + ); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = s + else { + panic!() + }; + let vars = fields.iter().find(|f| f.key == "variables").unwrap(); + let JsonShape::Object { fields: inner } = &vars.value else { + panic!() + }; + assert_eq!(inner[0].key, "password"); + assert_eq!( + inner[0].value, + leaf(vs( + ValueKind::String, + 12, + vec![hit(Role::Password, Encoding::AsTyped)] + )) + ); + } + + #[test] + fn a_json_key_that_holds_a_typed_value_is_not_kept() { + let s = shape( + "application/json", + br#"{"users":{"kh.throwaway7":{"secret":"Tr0ub4dor-9x"}}}"#, + ); + let BodyShape::Json { root } = s else { + panic!() + }; + let text = serde_json::to_string(&root).unwrap(); + assert!(text.contains(r#""key":"{typed}""#), "{text}"); + assert!(text.contains(r#""key":"secret""#), "{text}"); + assert!(!text.contains("throwaway"), "{text}"); + assert!(text.contains("password"), "{text}"); + } + + #[test] + fn multipart_text_and_file() { + let body = b"--XyZ\r\nContent-Disposition: form-data; name=\"password\"\r\n\r\nTr0ub4dor-9x\r\n--XyZ\r\nContent-Disposition: form-data; name=\"avatar\"; filename=\"me.png\"\r\nContent-Type: image/png\r\n\r\nkh.throwaway7\r\n--XyZ--\r\n"; + let s = shape("multipart/form-data; boundary=XyZ", body); + assert_eq!( + s, + BodyShape::Multipart { + parts: vec![ + field( + "password", + vs( + ValueKind::String, + 12, + vec![hit(Role::Password, Encoding::AsTyped)] + ) + ), + field("avatar", vs(ValueKind::File, 13, vec![])), + ] + } + ); + } + + #[test] + fn a_multipart_part_name_that_holds_a_typed_value_is_not_kept() { + let body = + b"--b\r\nContent-Disposition: form-data; name=\"kh.throwaway7\"\r\n\r\nx\r\n--b--"; + let s = shape("multipart/form-data; boundary=\"b\"", body); + let BodyShape::Multipart { parts } = s else { + panic!() + }; + assert_eq!(parts[0].key, "{typed}"); + } + + #[test] + fn urlencoded_key_that_holds_a_typed_value_is_not_kept() { + let s = shape(URLENC, b"kh.throwaway7=1"); + let BodyShape::Urlencoded { fields } = s else { + panic!() + }; + assert_eq!(fields[0].key, "{typed}"); + } + + #[test] + fn unknown_type_is_opaque_with_whole_body_hit() { + let s = shape("application/octet-stream", b"xx Tr0ub4dor-9x yy"); + assert_eq!( + s, + BodyShape::Opaque { + len: 18, + markers: vec![hit(Role::Password, Encoding::AsTyped)] + } + ); + assert!(matches!( + shape_of_body(None, b"a=b", &[], None), + BodyShape::Opaque { len: 3, .. } + )); + } + + #[test] + fn bad_json_falls_back_to_opaque() { + assert!(matches!( + shape("application/json", b"{not json"), + BodyShape::Opaque { len: 9, .. } + )); + assert!(matches!( + shape("application/json", b""), + BodyShape::Opaque { len: 0, .. } + )); + assert_eq!(shape(URLENC, b""), BodyShape::Urlencoded { fields: vec![] }); + } + + #[test] + fn a_combined_credential_lands_on_the_value_that_holds_it() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let body = format!(r#"{{"auth":"{b64}","n":1}}"#); + let (u, p) = (user("kh.throwaway7"), pw("Tr0ub4dor-9x")); + let with = shape_of_body( + Some("application/json"), + body.as_bytes(), + &markers(), + Some((&u, &p)), + ); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = with + else { + panic!() + }; + let auth = fields.iter().find(|f| f.key == "auth").unwrap(); + assert_eq!( + auth.value, + leaf(vs( + ValueKind::String, + u32::try_from(b64.len()).unwrap(), + vec![ + hit(Role::Username, Encoding::Base64), + hit(Role::Password, Encoding::Base64) + ] + )) + ); + let without = shape("application/json", body.as_bytes()); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = without + else { + panic!() + }; + let auth = fields.iter().find(|f| f.key == "auth").unwrap(); + let JsonShape::Value { value } = &auth.value else { + panic!() + }; + assert!(value.markers.is_empty()); + } + + #[test] + fn an_opaque_body_holds_a_combined_credential_at_whole_body_level() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let (u, p) = (user("kh.throwaway7"), pw("Tr0ub4dor-9x")); + let s = shape_of_body(None, b64.as_bytes(), &[], Some((&u, &p))); + let BodyShape::Opaque { markers, .. } = s else { + panic!() + }; + assert_eq!(markers.len(), 2); + } + + #[test] + fn oversized_bodies_are_opaque() { + let big = vec![b'a'; MAX_BODY_BYTES + 1]; + assert!(matches!( + shape("application/json", &big), + BodyShape::Opaque { .. } + )); + } + + #[test] + fn hostile_input_never_panics() { + let mut nested = "[".repeat(200); + nested.push_str(&"]".repeat(200)); + assert!(matches!( + shape("application/json", nested.as_bytes()), + BodyShape::Opaque { .. } + )); + for ct in [ + "multipart/form-data; boundary=XyZ", + "multipart/form-data; boundary=", + "multipart/form-data", + "multipart/form-data; boundary=\"unterminated", + URLENC, + "application/json", + ] { + for body in [ + &b"--XyZ\r\nContent-Disposition: form-data; name=\"a"[..], + b"--XyZ\r\n\r\n", + b"--XyZ\r\nContent-Disposition: form-data; name=\"a\"\r\n\r\nvalue", + b"--XyZ--", + b"%", + b"a=%&b=%4&c=%zz", + b"\xff\xfe=\xc3\x28&\xf0\x9f", + b"{\"a\":\"\xff\"}", + b"", + ] { + let _ = shape(ct, body); + } + } + let _ = strip_url("https://ex.test/%", &markers()); + let _ = strip_url("https://ex.test/%zz?%=%&%ff", &markers()); + let _ = strip_url("data:text/plain,Tr0ub4dor-9x", &markers()); + let _ = reduce_authorization(b"Basic \xff\xfe", &markers(), None); + let _ = reduce_authorization(b"", &markers(), None); + let _ = reduce_authorization(b" ", &markers(), None); + } + + #[test] + fn serialized_shapes_never_contain_a_marker() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let (u, p) = (user("kh.throwaway7"), pw("Tr0ub4dor-9x")); + let m = markers(); + let json_pair = format!(r#"{{"auth":"{b64}","n":1}}"#); + let multipart = b"--XyZ\r\nContent-Disposition: form-data; name=\"password\"\r\n\r\nTr0ub4dor-9x\r\n--XyZ\r\nContent-Disposition: form-data; name=\"kh.throwaway7\"; filename=\"kh.throwaway7\"\r\n\r\nkh.throwaway7\r\n--XyZ--\r\n"; + let bodies: Vec = vec![ + shape( + URLENC, + b"user=kh.throwaway7&password=Tr0ub4dor-9x&remember=1", + ), + shape(URLENC, b"password=Tr0ub4dor%2D9x&kh.throwaway7=1"), + shape( + "application/json", + br#"{"auth":{"id":"kh.throwaway7","secret":"Tr0ub4dor-9x"},"n":3}"#, + ), + shape( + "application/json", + br#"{"query":"x","variables":{"password":"Tr0ub4dor-9x"}}"#, + ), + shape( + "application/json", + br#"{"users":{"kh.throwaway7":{"secret":"Tr0ub4dor-9x"}}}"#, + ), + shape("multipart/form-data; boundary=XyZ", multipart), + shape("application/octet-stream", b"Tr0ub4dor-9x kh.throwaway7"), + shape("application/json", b"{Tr0ub4dor-9x"), + shape_of_body( + Some("application/json"), + json_pair.as_bytes(), + &m, + Some((&u, &p)), + ), + ]; + let mut outputs: Vec = bodies + .iter() + .map(|b| serde_json::to_string(b).unwrap()) + .collect(); + let urls = [ + "https://kh.throwaway7.ex.test/login", + "https://ex.test/u/kh.throwaway7/dash?kh.throwaway7=1&password=Tr0ub4dor-9x", + "https://ex.test/Tr0ub4dor-9x?x=Tr0ub4dor-9x#Tr0ub4dor-9x", + "https://kh.throwaway7:Tr0ub4dor-9x@ex.test/", + ]; + for url in urls { + let s = strip_url(url, &m).unwrap(); + outputs.push(serde_json::to_string(&s.value).unwrap()); + } + for header in [ + format!("Basic {b64}"), + "Bearer Tr0ub4dor-9x".to_owned(), + "Tr0ub4dor-9x".to_owned(), + "Basic Tr0ub4dor-9x".to_owned(), + ] { + let a = reduce_authorization(header.as_bytes(), &m, Some((&u, &p))); + outputs.push(serde_json::to_string(&a).unwrap()); + } + for text in outputs { + assert!(!text.contains("Tr0ub4dor-9x"), "{text}"); + assert!(!text.contains("kh.throwaway7"), "{text}"); + } + } +} diff --git a/crates/kharon-teach/src/lib.rs b/crates/kharon-teach/src/lib.rs index c07839b..7a1eebb 100644 --- a/crates/kharon-teach/src/lib.rs +++ b/crates/kharon-teach/src/lib.rs @@ -1,3 +1,5 @@ //! Teach-mode recorder for kharon (#361). +pub mod body; pub mod marker; +pub mod strip; diff --git a/crates/kharon-teach/src/strip.rs b/crates/kharon-teach/src/strip.rs new file mode 100644 index 0000000..8225d48 --- /dev/null +++ b/crates/kharon-teach/src/strip.rs @@ -0,0 +1,433 @@ +//! Stripping: a URL or an `Authorization` header reduced to structure only. + +use base64ct::{Base64, Base64Unpadded, Base64Url, Base64UrlUnpadded, Encoding as B64}; +use kharon_recording::{AuthShape, Encoding, MarkerHit, StrippedUrl}; +use percent_encoding::percent_decode; +use url::Url; +use zeroize::Zeroizing; + +use crate::marker::{Marker, search, search_pair}; + +/// Written in place of a name (key, part name, host labels) that held a typed value. +pub const TYPED: &str = "{typed}"; +/// Written in place of a path segment that is an identifier or held a typed value. +pub const ID: &str = "{id}"; + +pub struct Stripped { + pub value: T, + pub query_values_removed: u64, + pub path_segments_replaced: u64, +} + +/// Every hit of every marker in `haystack`, plus the pair, without repeats of (role, encoding). +pub(crate) fn hits_in( + haystack: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> Vec { + let mut out = Vec::new(); + for m in markers { + out.extend(search(haystack, m)); + } + if let Some((u, p)) = pair { + out.extend(search_pair(haystack, u, p)); + } + dedup_hits(out) +} + +/// Keeps the first hit for each (role, encoding). +pub(crate) fn dedup_hits(hits: Vec) -> Vec { + let mut out: Vec = Vec::with_capacity(hits.len()); + for h in hits { + if !out + .iter() + .any(|k| k.role == h.role && k.encoding == h.encoding) + { + out.push(h); + } + } + out +} + +/// Whether any marker (or the pair) occurs in `haystack` under any encoding. +pub(crate) fn any_hit( + haystack: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> bool { + !hits_in(haystack, markers, pair).is_empty() +} + +/// Origin, path and query keys of `url`. A path segment becomes "{id}" when it is all ASCII digits, +/// a UUID, 16 or more hex characters, or contains any marker (as typed or percent-encoded). +/// +/// Userinfo and fragment are dropped. A query key or host label holding a typed value is written +/// as "{typed}". Known limit: an internationalised host is compared in its punycode form, so a +/// non-ASCII typed value inside such a label is not recognised. +pub fn strip_url(url: &str, markers: &[Marker]) -> Option> { + let parsed = Url::parse(url).ok()?; + let mut path_segments_replaced = 0u64; + + let origin = match parsed.host_str() { + Some(host) => { + let port = parsed.port().map(|p| format!(":{p}")).unwrap_or_default(); + format!( + "{}://{}{}", + parsed.scheme(), + strip_host(host, markers), + port + ) + } + None => format!("{}:", parsed.scheme()), + }; + + let path = if parsed.cannot_be_a_base() { + // Opaque path (`data:`, `mailto:` ...): it is a payload, not a route. + path_segments_replaced += 1; + ID.to_owned() + } else { + let mut owned: Vec = Vec::new(); + for seg in parsed.path().split('/') { + if segment_is_id(seg, markers) { + path_segments_replaced += 1; + owned.push(ID.to_owned()); + } else { + owned.push(seg.to_owned()); + } + } + owned.join("/") + }; + + let mut query_keys = Vec::new(); + let mut query_values_removed = 0u64; + for (key, _value) in parsed.query_pairs() { + query_values_removed += 1; + if any_hit(key.as_bytes(), markers, None) { + query_keys.push(TYPED.to_owned()); + } else { + query_keys.push(key.into_owned()); + } + } + + Some(Stripped { + value: StrippedUrl { + origin, + path, + query_keys, + }, + query_values_removed, + path_segments_replaced, + }) +} + +/// Replaces host labels that hold a typed value. Every occurrence of every marker is located +/// (ASCII case-insensitively: the parser lowercases hosts) and every label the occurrence touches is +/// replaced. Adjacent replaced labels become ONE `{typed}` label, so a value that spans labels +/// (`kh.throwaway7`) leaves no part of itself behind. +fn strip_host(host: &str, markers: &[Marker]) -> String { + let mut masked = vec![false; host.len()]; + for m in markers { + let mut from = 0; + while let Some(range) = host + .get(from..) + .and_then(|rest| m.find_ignore_ascii_case(rest)) + { + let (start, end) = (from + range.start, from + range.end); + masked[start..end].iter_mut().for_each(|b| *b = true); + // Resume one character after the start so overlapping occurrences are found. + from = start + host[start..].chars().next().map_or(1, char::len_utf8); + } + } + let mut labels: Vec<&str> = Vec::new(); + let mut offset = 0; + let mut previous_typed = false; + for label in host.split('.') { + let typed = masked[offset..offset + label.len()].iter().any(|b| *b); + offset += label.len() + 1; + if typed { + if !previous_typed { + labels.push(TYPED); + } + } else { + labels.push(label); + } + previous_typed = typed; + } + labels.join(".") +} + +fn segment_is_id(seg: &str, markers: &[Marker]) -> bool { + if seg.is_empty() { + return false; + } + let all_digits = seg.bytes().all(|b| b.is_ascii_digit()); + let long_hex = seg.len() >= 16 && seg.bytes().all(|b| b.is_ascii_hexdigit()); + if all_digits || long_hex || is_uuid(seg) { + return true; + } + let decoded: Zeroizing> = Zeroizing::new(percent_decode(seg.as_bytes()).collect()); + any_hit(seg.as_bytes(), markers, None) || any_hit(&decoded, markers, None) +} + +fn is_uuid(seg: &str) -> bool { + let b = seg.as_bytes(); + b.len() == 36 + && b.iter().enumerate().all(|(i, c)| match i { + 8 | 13 | 18 | 23 => *c == b'-', + _ => c.is_ascii_hexdigit(), + }) +} + +/// scheme = first token, lowercased. For `basic`, the credentials are base64-decoded and the decoded bytes +/// are searched; a marker found there is reported with `Encoding::Base64`. Other schemes are searched as sent. +/// When `pair` is given the value is also searched with `search_pair`. No (role, encoding) is listed twice. +/// +/// The value as sent is always searched too. The scheme is kept only when it is a plain token +/// (at most 32 ASCII letters, digits, `-` or `_`) holding no typed value; otherwise it is +/// `{typed}` or `{other}`, so a bare credential sent with no scheme is not copied into the file. +pub fn reduce_authorization( + header_value: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> AuthShape { + let (scheme_bytes, credentials) = match header_value.iter().position(|b| *b == b' ') { + Some(i) => (&header_value[..i], &header_value[i + 1..]), + None => (header_value, &header_value[..0]), + }; + let mut hits = hits_in(header_value, markers, pair); + + let scheme_lower = scheme_bytes.to_ascii_lowercase(); + if scheme_lower == b"basic" + && let Some(decoded) = decode_basic(credentials) + { + let decoded_hits = hits_in(&decoded, markers, None); + hits.extend(decoded_hits.into_iter().map(|h| MarkerHit { + encoding: Encoding::Base64, + ..h + })); + } + + let scheme = if any_hit(scheme_bytes, markers, pair) { + TYPED.to_owned() + } else if !scheme_lower.is_empty() + && scheme_lower.len() <= 32 + && scheme_lower + .iter() + .all(|b| b.is_ascii_alphanumeric() || *b == b'-' || *b == b'_') + { + String::from_utf8_lossy(&scheme_lower).into_owned() + } else { + "{other}".to_owned() + }; + + AuthShape { + scheme, + markers: dedup_hits(hits), + } +} + +/// The decoded `user:password` bytes, or `None` when the credentials are not base64. +fn decode_basic(credentials: &[u8]) -> Option>> { + let text = std::str::from_utf8(credentials).ok()?.trim(); + type Decoder = fn(&str) -> Option>; + let decoders: [Decoder; 4] = [ + |t| Base64::decode_vec(t).ok(), + |t| Base64Unpadded::decode_vec(t).ok(), + |t| Base64Url::decode_vec(t).ok(), + |t| Base64UrlUnpadded::decode_vec(t).ok(), + ]; + decoders + .iter() + .find_map(|decode| decode(text)) + .map(Zeroizing::new) +} + +#[cfg(test)] +mod tests { + use super::*; + use kharon_recording::Role; + + pub(crate) fn user(s: &str) -> Marker { + Marker::new(Zeroizing::new(s.to_owned()), Role::Username) + } + pub(crate) fn pw(s: &str) -> Marker { + Marker::new(Zeroizing::new(s.to_owned()), Role::Password) + } + fn hit(role: Role, encoding: Encoding) -> MarkerHit { + MarkerHit { + role, + encoding, + low_confidence: false, + } + } + + #[test] + fn keeps_origin_path_and_query_keys() { + let s = strip_url("https://ex.test/login/step?next=%2Fhome&lang=en#frag", &[]).unwrap(); + assert_eq!( + s.value, + StrippedUrl { + origin: "https://ex.test".into(), + path: "/login/step".into(), + query_keys: vec!["next".into(), "lang".into()] + } + ); + assert_eq!(s.query_values_removed, 2); + assert_eq!(s.path_segments_replaced, 0); + } + + #[test] + fn origin_has_no_userinfo_keeps_a_port_and_duplicate_keys() { + let s = strip_url("http://bob:s3cret@ex.test:8080/a?k=1&k=2", &[]).unwrap(); + assert_eq!(s.value.origin, "http://ex.test:8080"); + assert_eq!(s.value.query_keys, ["k", "k"]); + } + + #[test] + fn replaces_id_like_segments() { + for (path, want, n) in [ + ("/users/12345/home", "/users/{id}/home", 1), + ("/s/3f2b8c1e-9a4d-4c1b-8e2f-0a1b2c3d4e5f", "/s/{id}", 1), + ("/s/3F2B8C1E-9A4D-4C1B-8E2F-0A1B2C3D4E5F", "/s/{id}", 1), + ("/t/0123456789abcdef0123", "/t/{id}", 1), + ("/v2/login", "/v2/login", 0), + ("/a/b/", "/a/b/", 0), + ] { + let s = strip_url(&format!("https://ex.test{path}"), &[]).unwrap(); + assert_eq!(s.value.path, want); + assert_eq!(s.path_segments_replaced, n); + } + } + + #[test] + fn replaces_a_segment_that_contains_a_marker() { + let m = [user("kh.throwaway7")]; + let s = strip_url("https://ex.test/u/kh.throwaway7/dash", &m).unwrap(); + assert_eq!(s.value.path, "/u/{id}/dash"); + let s = strip_url("https://ex.test/u/kh%2Ethrow%61way7/dash", &m).unwrap(); + assert_eq!(s.value.path, "/u/{id}/dash"); + assert_eq!(s.path_segments_replaced, 1); + } + + #[test] + fn unparseable_url_is_none() { + assert!(strip_url("not a url", &[]).is_none()); + } + + #[test] + fn a_query_key_that_holds_a_typed_value_is_not_kept() { + let m = [user("kh.throwaway7")]; + let s = strip_url("https://ex.test/x?kh.throwaway7=1&ok=2", &m).unwrap(); + assert_eq!(s.value.query_keys, ["{typed}", "ok"]); + let s = strip_url("https://ex.test/x?kh%2Ethrowaway7=1", &m).unwrap(); + assert_eq!(s.value.query_keys, ["{typed}"]); + } + + #[test] + fn a_host_label_that_holds_a_typed_value_is_replaced() { + let m = [user("kh.throwaway7")]; + // The value spans two labels: the whole run of labels the match covers is one `{typed}`. + let s = strip_url("https://kh.throwaway7.ex.test/login", &m).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + // Case-insensitive, and a partial label is replaced whole. + let m = [user("Kh.Throwaway7")]; + let s = strip_url("https://x-kh.throwaway7-y.ex.test:81/", &m).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test:81"); + // Every occurrence. + let m = [user("bob12")]; + let s = strip_url("https://bob12.ex.bob12.test/", &m).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.{typed}.test"); + } + + #[test] + fn basic_auth_is_reduced_to_scheme_and_hits() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let a = reduce_authorization( + format!("Basic {b64}").as_bytes(), + &[user("kh.throwaway7"), pw("Tr0ub4dor-9x")], + None, + ); + assert_eq!(a.scheme, "basic"); + assert_eq!( + a.markers, + [ + hit(Role::Username, Encoding::Base64), + hit(Role::Password, Encoding::Base64) + ] + ); + } + + #[test] + fn basic_auth_with_the_pair_lists_each_hit_once() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let (u, p) = (user("kh.throwaway7"), pw("Tr0ub4dor-9x")); + let a = reduce_authorization( + format!("Basic {b64}").as_bytes(), + &[user("kh.throwaway7"), pw("Tr0ub4dor-9x")], + Some((&u, &p)), + ); + assert_eq!( + a.markers, + [ + hit(Role::Username, Encoding::Base64), + hit(Role::Password, Encoding::Base64) + ] + ); + } + + #[test] + fn bearer_keeps_no_token() { + let a = reduce_authorization( + b"Bearer eyJhbGciOiJIUzI1NiJ9.abc.def", + &[pw("Tr0ub4dor-9x")], + None, + ); + assert_eq!( + a, + AuthShape { + scheme: "bearer".into(), + markers: vec![] + } + ); + assert!(!serde_json::to_string(&a).unwrap().contains("eyJ")); + } + + #[test] + fn a_typed_password_sent_as_a_bearer_token_is_a_hit() { + let a = reduce_authorization(b"Bearer Tr0ub4dor-9x", &[pw("Tr0ub4dor-9x")], None); + assert_eq!(a.markers, [hit(Role::Password, Encoding::AsTyped)]); + } + + #[test] + fn no_space_is_a_scheme_with_no_credentials() { + let a = reduce_authorization(b"Negotiate", &[], None); + assert_eq!( + a, + AuthShape { + scheme: "negotiate".into(), + markers: vec![] + } + ); + // A bare typed value never becomes the scheme. + let a = reduce_authorization(b"Tr0ub4dor-9x", &[pw("Tr0ub4dor-9x")], None); + assert_eq!(a.scheme, "{typed}"); + assert_eq!(a.markers, [hit(Role::Password, Encoding::AsTyped)]); + // Nor does anything that is not a plain token. + let a = reduce_authorization(b"s3cr3t/token+value=", &[], None); + assert_eq!(a.scheme, "{other}"); + } + + #[test] + fn malformed_basic_has_no_markers_but_is_searched_as_sent() { + let a = reduce_authorization(b"Basic !!!not-base64!!!", &[pw("Tr0ub4dor-9x")], None); + assert_eq!( + a, + AuthShape { + scheme: "basic".into(), + markers: vec![] + } + ); + let a = reduce_authorization(b"Basic Tr0ub4dor-9x", &[pw("Tr0ub4dor-9x")], None); + assert_eq!(a.markers, [hit(Role::Password, Encoding::AsTyped)]); + } +} From e78daadc493a1cf0ea7357b012f5fcefa2984a85 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:11:49 -0500 Subject: [PATCH 10/36] fix(teach): find percent-encoded values by decoding the haystack (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- Cargo.lock | 1 + crates/kharon-teach/Cargo.toml | 1 + crates/kharon-teach/src/marker.rs | 249 +++++++++++++++++++++++++----- 3 files changed, 212 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9fbe748..e83dee8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1087,6 +1087,7 @@ dependencies = [ "hex", "kharon-recording", "md-5", + "memchr", "percent-encoding", "serde_json", "sha1", diff --git a/crates/kharon-teach/Cargo.toml b/crates/kharon-teach/Cargo.toml index c8b1694..d0d00e2 100644 --- a/crates/kharon-teach/Cargo.toml +++ b/crates/kharon-teach/Cargo.toml @@ -24,5 +24,6 @@ md-5 = "0.10" base64ct = { version = "1", features = ["alloc"] } hex = "0.4" percent-encoding = "2" +memchr = "2" url = "2" serde_json = "1" diff --git a/crates/kharon-teach/src/marker.rs b/crates/kharon-teach/src/marker.rs index d35e126..26971b5 100644 --- a/crates/kharon-teach/src/marker.rs +++ b/crates/kharon-teach/src/marker.rs @@ -4,7 +4,7 @@ use base64ct::{Base64, Base64Unpadded, Base64Url, Base64UrlUnpadded, Encoding as B64}; use kharon_recording::{Encoding, MarkerHit, Role}; use md5::Md5; -use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode}; +use memchr::memmem; use sha1::Sha1; use sha2::{Digest, Sha256, Sha512}; use zeroize::Zeroizing; @@ -18,6 +18,16 @@ pub const LOW_CONFIDENCE_BELOW: u32 = 6; /// let m = kharon_teach::marker::Marker::new(zeroize::Zeroizing::new(String::from("x")), kharon_recording::Role::Password); /// println!("{:?}", m); /// ``` +/// +/// ```compile_fail +/// let m = kharon_teach::marker::Marker::new(zeroize::Zeroizing::new(String::from("x")), kharon_recording::Role::Password); +/// println!("{}", m); +/// ``` +/// +/// ```compile_fail +/// let m = kharon_teach::marker::Marker::new(zeroize::Zeroizing::new(String::from("x")), kharon_recording::Role::Password); +/// let _ = m.clone(); +/// ``` pub struct Marker { value: Zeroizing, role: Role, @@ -71,37 +81,104 @@ impl Marker { type Needle = Zeroizing>; -/// RFC 3986 unreserved characters stay as they are; every other byte is percent-encoded. -const UNRESERVED: &AsciiSet = &NON_ALPHANUMERIC - .remove(b'-') - .remove(b'.') - .remove(b'_') - .remove(b'~'); +/// The request bytes as sent, plus up to two percent-decoded copies: `%XX` only, and `+` as a +/// space then `%XX`. A copy identical to the raw bytes is left out. The copies are zeroized. +struct Haystacks<'a> { + raw: &'a [u8], + decoded: Vec, +} + +impl<'a> Haystacks<'a> { + fn new(raw: &'a [u8]) -> Self { + let mut decoded: Vec = Vec::new(); + for plus_is_space in [false, true] { + let copy = percent_decode(raw, plus_is_space); + if copy.as_slice() != raw && !decoded.iter().any(|d| d.as_slice() == copy.as_slice()) { + decoded.push(copy); + } + } + Self { raw, decoded } + } + + fn in_raw(&self, forms: &[Needle]) -> bool { + forms.iter().any(|n| contains(self.raw, n)) + } + + fn in_decoded(&self, forms: &[Needle]) -> bool { + forms + .iter() + .any(|n| self.decoded.iter().any(|d| contains(d, n))) + } +} + +fn hex_value(b: u8) -> Option { + char::from(b) + .to_digit(16) + .and_then(|d| u8::try_from(d).ok()) +} + +/// Decodes `%XX` (either hex case); a `%` not followed by two hex digits is copied unchanged. +fn percent_decode(raw: &[u8], plus_is_space: bool) -> Needle { + let mut out = Zeroizing::new(Vec::with_capacity(raw.len())); + let mut i = 0; + while i < raw.len() { + let b = raw[i]; + if b == b'%' + && let (Some(hi), Some(lo)) = ( + raw.get(i + 1).copied().and_then(hex_value), + raw.get(i + 2).copied().and_then(hex_value), + ) + { + out.push(hi * 16 + lo); + i += 3; + continue; + } + out.push(if plus_is_space && b == b'+' { b' ' } else { b }); + i += 1; + } + out +} /// Every way `marker` occurs in `haystack`. Digests are searched only for Role::Password. /// /// At most one hit per `Encoding`, in the order of the enum's variants. An encoded form that is /// byte-identical to the as-typed form is not searched, so a plain value reports only `AsTyped`. +/// +/// Percent encoding is found by decoding the haystack (see `Haystacks`) and searching the raw +/// bytes and both decoded copies. `Percent` is reported when the typed value is found only in a +/// decoded copy; every other encoding is reported when its form is found in any of the three. pub fn search(haystack: &[u8], marker: &Marker) -> Vec { if marker.is_empty() { return Vec::new(); } let low_confidence = marker.len() < LOW_CONFIDENCE_BELOW; - needles(marker) - .into_iter() - .filter(|(_, forms)| forms.iter().any(|n| contains(haystack, n))) - .map(|(encoding, _)| MarkerHit { + let haystacks = Haystacks::new(haystack); + let mut hits = Vec::new(); + let mut hit = |encoding| { + hits.push(MarkerHit { role: marker.role, encoding, low_confidence, - }) - .collect() + }); + }; + for (encoding, forms) in needles(marker) { + if encoding == Encoding::AsTyped { + if haystacks.in_raw(&forms) { + hit(Encoding::AsTyped); + } else if haystacks.in_decoded(&forms) { + hit(Encoding::Percent); + } + } else if haystacks.in_raw(&forms) || haystacks.in_decoded(&forms) { + hit(encoding); + } + } + hits } /// The base64 and base64url (padded or not) of `username:password`. A match returns two hits, /// one per role, both `Encoding::Base64` (or `Base64Url`). /// -/// Nothing is returned when either marker is empty. `low_confidence` is set when the combined +/// Searches the same three haystacks as `search`. Nothing is returned when either marker is empty. `low_confidence` is set when the combined /// length of both values is under `LOW_CONFIDENCE_BELOW` characters. pub fn search_pair(haystack: &[u8], username: &Marker, password: &Marker) -> Vec { if username.is_empty() || password.is_empty() { @@ -119,12 +196,13 @@ pub fn search_pair(haystack: &[u8], username: &Marker, password: &Marker) -> Vec let seen: Vec<&[u8]> = standard.iter().map(|n| n.as_slice()).collect(); dedup(url_safe, &seen) }; + let haystacks = Haystacks::new(haystack); let mut hits = Vec::new(); for (encoding, forms) in [ (Encoding::Base64, standard), (Encoding::Base64Url, url_safe), ] { - if forms.iter().any(|n| contains(haystack, n)) { + if haystacks.in_raw(&forms) || haystacks.in_decoded(&forms) { for role in [username.role, password.role] { hits.push(MarkerHit { role, @@ -138,7 +216,7 @@ pub fn search_pair(haystack: &[u8], username: &Marker, password: &Marker) -> Vec } fn contains(haystack: &[u8], needle: &[u8]) -> bool { - !needle.is_empty() && haystack.windows(needle.len()).any(|w| w == needle) + !needle.is_empty() && memmem::find(haystack, needle).is_some() } /// The search forms per encoding, in `Encoding` order. @@ -153,7 +231,6 @@ fn needles(marker: &Marker) -> Vec<(Encoding, Vec)> { }; let mut out = vec![ (Encoding::AsTyped, vec![Zeroizing::new(value.to_vec())]), - (Encoding::Percent, dedup(percent_forms(value), &[value])), ( Encoding::JsonEscaped, dedup(json_forms(&marker.value), &[value]), @@ -238,28 +315,7 @@ fn hex_forms(bytes: &[u8]) -> Vec { } let mut upper = lower.clone(); upper.make_ascii_uppercase(); - vec![lower, upper] -} - -/// Uppercase and lowercase percent hex, each with space as `%20` and as `+`. -fn percent_forms(bytes: &[u8]) -> Vec { - let mut forms = Vec::new(); - for plus in [false, true] { - for lower in [false, true] { - let mut out = Zeroizing::new(Vec::with_capacity(bytes.len() * 3)); - for chunk in percent_encode(bytes, UNRESERVED) { - if plus && chunk == "%20" { - out.push(b'+'); - } else if lower && chunk.starts_with('%') { - out.extend(chunk.bytes().map(|b| b.to_ascii_lowercase())); - } else { - out.extend_from_slice(chunk.as_bytes()); - } - } - forms.push(out); - } - } - forms + dedup(vec![lower, upper], &[]) } #[derive(Clone, Copy)] @@ -529,6 +585,121 @@ mod tests { ); } + #[test] + fn component_encoding_leaves_punctuation_literal() { + let m = pw("my pass!x*9"); + assert_eq!(encodings(b"p=my%20pass!x*9", &m), [Encoding::Percent]); + assert_eq!(encodings(b"p=my+pass%21x*9", &m), [Encoding::Percent]); + } + + #[test] + fn percent_hex_digits_in_mixed_case() { + assert_eq!( + encodings(b"p=p%c3%A4ssw%C3%b6rd12", &pw("p\u{e4}ssw\u{f6}rd12")), + [Encoding::Percent] + ); + } + + #[test] + fn stray_percent_does_not_panic_or_match() { + let m = pw("abcdef12"); + assert!(search(b"p=abc%", &m).is_empty()); + assert!(search(b"p=abc%zzdef12", &m).is_empty()); + assert!(search(b"%", &m).is_empty()); + assert!(search(b"%4", &m).is_empty()); + assert_eq!(encodings(b"p=%zz%61bcdef12", &m), [Encoding::Percent]); + } + + #[test] + fn base64_with_percent_encoded_symbols() { + assert_eq!( + encodings(b"x=Pj4%2BPz8%2FMTI%3D", &pw(">>>???12")), + [Encoding::Base64] + ); + } + + #[test] + fn raw_match_reports_only_as_typed() { + assert_eq!( + encodings(b"a=my%20pass!x*9&b=my pass!x*9", &pw("my pass!x*9")), + [Encoding::AsTyped] + ); + } + + #[test] + fn every_digest_in_hex_and_base64() { + let value = "Tr0ub4dor-9x"; + let digests: [(Encoding, Encoding, Vec); 4] = [ + ( + Encoding::Md5Hex, + Encoding::Md5Base64, + Md5::digest(value).to_vec(), + ), + ( + Encoding::Sha1Hex, + Encoding::Sha1Base64, + Sha1::digest(value).to_vec(), + ), + ( + Encoding::Sha256Hex, + Encoding::Sha256Base64, + Sha256::digest(value).to_vec(), + ), + ( + Encoding::Sha512Hex, + Encoding::Sha512Base64, + Sha512::digest(value).to_vec(), + ), + ]; + for (hex_enc, b64_enc, digest) in digests { + for (expected, text) in [ + (hex_enc, hex::encode(&digest)), + (hex_enc, hex::encode(&digest).to_uppercase()), + (b64_enc, Base64::encode_string(&digest)), + (b64_enc, Base64Unpadded::encode_string(&digest)), + ] { + let haystack = format!("h={text}&z=1"); + assert_eq!(encodings(haystack.as_bytes(), &pw(value)), [expected]); + assert!(search(haystack.as_bytes(), &user(value)).is_empty()); + } + } + } + + #[test] + fn pair_in_url_safe_base64() { + let joined = b"kh>>>user:pa??w0rd-9"; + let std = Base64::encode_string(joined); + assert!(std.contains('+')); + let url = Base64UrlUnpadded::encode_string(joined); + let haystack = format!("auth={url}"); + let hits = search_pair(haystack.as_bytes(), &user("kh>>>user"), &pw("pa??w0rd-9")); + assert_eq!( + hits.iter() + .map(|h| (h.role, h.encoding)) + .collect::>(), + [ + (Role::Username, Encoding::Base64Url), + (Role::Password, Encoding::Base64Url) + ] + ); + } + + #[test] + fn pair_with_percent_encoded_base64() { + let joined = b"kh>>>user:pa??w0rd-9"; + let haystack = format!( + "auth={}", + Base64::encode_string(joined) + .replace('+', "%2B") + .replace('/', "%2F") + .replace('=', "%3D") + ); + assert_eq!( + search_pair(haystack.as_bytes(), &user("kh>>>user"), &pw("pa??w0rd-9")).len(), + 2 + ); + } + #[test] fn same_value_and_with_role() { let a = pw("abc"); From 4931542a554a14606ebc98e0f8ada02769918fac Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:13:50 -0500 Subject: [PATCH 11/36] docs(plan): strip_url takes the pair, and one shared no-typed-value check The task 4 review found that a typed value split by a separator, or in a different letter case, could survive in a path or a key. The plan's signatures and the shared test helper now say what the fix builds. Refs #361 Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- .../plans/2026-11-18-361-teach-recorder.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/superpowers/plans/2026-11-18-361-teach-recorder.md b/docs/superpowers/plans/2026-11-18-361-teach-recorder.md index cd300b1..3ab1dcb 100644 --- a/docs/superpowers/plans/2026-11-18-361-teach-recorder.md +++ b/docs/superpowers/plans/2026-11-18-361-teach-recorder.md @@ -358,8 +358,10 @@ Also add this doc test on `Marker`, which is the compile-time guard: // strip.rs pub struct Stripped { pub value: T, pub query_values_removed: u64, pub path_segments_replaced: u64 } /// Origin, path and query keys of `url`. A path segment becomes "{id}" when it is all ASCII digits, -/// a UUID, 16 or more hex characters, or contains any marker (as typed or percent-encoded). -pub fn strip_url(url: &str, markers: &[Marker]) -> Option>; // None when `url` does not parse +/// a UUID, 16 or more hex characters, or is touched by a marker match anywhere in the path (any encoding `search` +/// knows, any letter case, with or without `pair`). A query key or host label a marker touches is written `{typed}`. +/// A URL that cannot be a base (`data:`, `blob:`, `about:`) keeps its scheme only. +pub fn strip_url(url: &str, markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> Option>; // None when `url` does not parse /// scheme = first token, lowercased. For `basic`, the credentials are base64-decoded and the decoded bytes /// are searched; a marker found there is reported with `Encoding::Base64`. Other schemes are searched as sent. /// When `pair` is given the value is also searched with `search_pair`. No (role, encoding) is listed twice. @@ -407,7 +409,7 @@ A multipart part with a `filename` is `ValueKind::File` with its length and no m ``` - [ ] **Step 2: Run** `cargo test -p kharon-teach -- strip body` (cargo takes one filter before `--`; several go after it). Expected: FAIL. -- [ ] **Step 3: Implement.** Add `url = "2"`, `serde_json`. The JSON body is the one place a site payload is read as `serde_json::Value`; say so in a comment. Urlencoded keys and JSON keys are kept as they are; values never are. +- [ ] **Step 3: Implement.** Add `url = "2"`, `serde_json`. The JSON body is the one place a site payload is read as `serde_json::Value`; say so in a comment. Urlencoded keys, JSON keys and multipart part names are kept unless a typed value is found in one, in which case it is written `{typed}`; values never are. An `Authorization` scheme is kept only when it is a known scheme name. `kharon_teach::testing` (doc-hidden) holds the needle list and the "holds no typed value" assertion that this task's tests, Task 5's and Task 8's all use. - [ ] **Step 4: Run** tests, clippy, fmt. Expected: pass. - [ ] **Step 5: Commit** `feat(teach): strip URLs and reduce request bodies to shapes (#361)` @@ -492,7 +494,7 @@ Decisions `finish` implements, beyond the spec's text: #[test] fn events_before_started_are_ignored_for_steps() #[test] fn the_serialized_recording_holds_no_typed_value() { // for every scenario above: recording.to_json() contains neither marker in any encoding - // (as typed, percent, base64, hex, sha256 hex). Share the needle list with Task 8's test via a pub(crate) helper. + // (as typed, percent, base64, hex, sha256 hex): `kharon_teach::testing::assert_holds_no_typed_value`, from Task 4. } // site.rs #[test] fn registrable_domains() { /* login.ex.co.uk -> ex.co.uk; 127.0.0.1 and localhost unchanged */ } @@ -686,7 +688,7 @@ Wiring, in order: `Profile::sweep_stale`; `Profile::create`; `Browser::launch` w // req_tch_002_no_typed_value.rs #[tokio::test] async fn no_typed_value_in_the_recording_for_any_fixture() { // username "kh.throwaway7", password "Tr0ub4dor-9x": recording.to_json() contains neither, in none of the - // encodings marker::search knows (reuse the needle helper from Task 5). + // encodings marker::search knows (`kharon_teach::testing::assert_holds_no_typed_value`). } // req_tch_006_profile_and_visible.rs From 2ad42b8857d8eaaf980d306672b3a8d61cbd9933 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:18:30 -0500 Subject: [PATCH 12/36] fix(teach): match typed values across separators and letter case in kept text (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-teach/src/body.rs | 225 +++++++----- crates/kharon-teach/src/lib.rs | 2 + crates/kharon-teach/src/strip.rs | 535 +++++++++++++++++++++++------ crates/kharon-teach/src/testing.rs | 104 ++++++ 4 files changed, 683 insertions(+), 183 deletions(-) create mode 100644 crates/kharon-teach/src/testing.rs diff --git a/crates/kharon-teach/src/body.rs b/crates/kharon-teach/src/body.rs index f62f44f..35d6d74 100644 --- a/crates/kharon-teach/src/body.rs +++ b/crates/kharon-teach/src/body.rs @@ -5,11 +5,11 @@ //! ordinary `String`s that serde_json does not zeroize. use kharon_recording::{BodyShape, FieldShape, JsonField, JsonShape, ValueKind, ValueShape}; -use percent_encoding::percent_decode; +use memchr::memmem; use zeroize::Zeroizing; use crate::marker::Marker; -use crate::strip::{TYPED, any_hit, hits_in}; +use crate::strip::{TYPED, decode_percent, hits_in, holds_typed}; /// Bodies larger than this are not parsed: they are `Opaque`. pub const MAX_BODY_BYTES: usize = 1 << 20; @@ -145,25 +145,16 @@ fn text_value(bytes: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)> value_shape(ValueKind::String, char_len(bytes), bytes, markers, pair) } -/// A name that is kept only when it holds no typed value. +/// A name that is kept only when it holds no typed value (any encoding, or the text in any +/// letter case, raw or percent-decoded; see `strip::holds_typed`). fn name(bytes: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> String { - if any_hit(bytes, markers, pair) { + if holds_typed(bytes, markers, pair) { TYPED.to_owned() } else { String::from_utf8_lossy(bytes).into_owned() } } -/// Form decoding: `+` is a space, then percent-decoding. A lone `%` stays as it is. -fn form_decode(raw: &[u8]) -> Zeroizing> { - let spaced: Zeroizing> = Zeroizing::new( - raw.iter() - .map(|b| if *b == b'+' { b' ' } else { *b }) - .collect(), - ); - Zeroizing::new(percent_decode(&spaced).collect()) -} - fn urlencoded_fields( body: &[u8], markers: &[Marker], @@ -176,8 +167,8 @@ fn urlencoded_fields( Some(i) => (&piece[..i], &piece[i + 1..]), None => (piece, &piece[..0]), }; - let key = form_decode(k); - let value = form_decode(v); + let key = decode_percent(k, true); + let value = decode_percent(v, true); FieldShape { key: name(&key, markers, pair), value: text_value(&value, markers, pair), @@ -240,14 +231,14 @@ fn find(haystack: &[u8], needle: &[u8], from: usize) -> Option { if needle.is_empty() || from > haystack.len() { return None; } - haystack[from..] - .windows(needle.len()) - .position(|w| w == needle) - .map(|i| from + i) + memmem::find(&haystack[from..], needle).map(|i| from + i) } /// Parts of a `multipart/form-data` body, or `None` when it is not well formed (no opening /// delimiter, a part with no header end, or a part not closed by a delimiter). +/// +/// The line ending (CRLF or LF) is read once, from the first delimiter line, and used for every +/// part, so each search stops at its own terminator and the whole parse is linear in the body. fn multipart_parts( body: &[u8], boundary: &str, @@ -256,13 +247,18 @@ fn multipart_parts( ) -> Option> { let mut delim = b"--".to_vec(); delim.extend_from_slice(boundary.as_bytes()); - // The delimiter that closes a part is preceded by a line break. - let mut crlf_delim = b"\r\n".to_vec(); - crlf_delim.extend_from_slice(&delim); - let mut lf_delim = b"\n".to_vec(); - lf_delim.extend_from_slice(&delim); let mut pos = find(body, &delim, 0)? + delim.len(); + let first_newline = pos + body[pos..].iter().position(|b| *b == b'\n')?; + let eol: &[u8] = if first_newline > pos && body[first_newline - 1] == b'\r' { + b"\r\n" + } else { + b"\n" + }; + let blank_line = [eol, eol].concat(); + // The delimiter that closes a part is preceded by a line break. + let closing = [eol, delim.as_slice()].concat(); + let mut parts = Vec::new(); loop { if body[pos..].starts_with(b"--") { @@ -270,28 +266,14 @@ fn multipart_parts( } // Rest of the delimiter line (transport padding), then the headers. let headers_start = pos + body[pos..].iter().position(|b| *b == b'\n')? + 1; - let (headers_end, content_start) = match ( - find(body, b"\r\n\r\n", headers_start), - find(body, b"\n\n", headers_start), - ) { - (Some(a), Some(b)) if b < a => (b, b + 2), - (Some(a), _) => (a, a + 4), - (None, Some(b)) => (b, b + 2), - (None, None) => return None, - }; + let headers_end = find(body, &blank_line, headers_start)?; + let content_start = headers_end + blank_line.len(); let headers = &body[headers_start..headers_end]; - let (content_end, next) = match ( - find(body, &crlf_delim, content_start.saturating_sub(2)), - find(body, &lf_delim, content_start.saturating_sub(1)), - ) { - (Some(a), Some(b)) if b < a => (b, b + 1), - (Some(a), _) => (a, a + 2), - (None, Some(b)) => (b, b + 1), - (None, None) => return None, - }; + // The line break that ends an empty content is the blank line's own. + let content_end = find(body, &closing, content_start - eol.len())?; let content = body.get(content_start..content_end).unwrap_or(&[]); parts.push(part_shape(headers, content, markers, pair)); - pos = next + delim.len(); + pos = content_end + closing.len(); } } @@ -338,15 +320,10 @@ fn part_shape( mod tests { use super::*; use crate::strip::{reduce_authorization, strip_url}; + use crate::testing::{assert_holds_no_typed_value, pw, user}; use base64ct::{Base64, Encoding as B64}; use kharon_recording::{Encoding, MarkerHit, Role}; - fn user(s: &str) -> Marker { - Marker::new(Zeroizing::new(s.to_owned()), Role::Username) - } - fn pw(s: &str) -> Marker { - Marker::new(Zeroizing::new(s.to_owned()), Role::Password) - } fn markers() -> Vec { vec![user("kh.throwaway7"), pw("Tr0ub4dor-9x")] } @@ -722,48 +699,127 @@ mod tests { let _ = shape(ct, body); } } - let _ = strip_url("https://ex.test/%", &markers()); - let _ = strip_url("https://ex.test/%zz?%=%&%ff", &markers()); - let _ = strip_url("data:text/plain,Tr0ub4dor-9x", &markers()); + let _ = strip_url("https://ex.test/%", &markers(), None); + let _ = strip_url("https://ex.test/%zz?%=%&%ff", &markers(), None); + let _ = strip_url("data:text/plain,Tr0ub4dor-9x", &markers(), None); let _ = reduce_authorization(b"Basic \xff\xfe", &markers(), None); let _ = reduce_authorization(b"", &markers(), None); let _ = reduce_authorization(b" ", &markers(), None); } #[test] - fn serialized_shapes_never_contain_a_marker() { + fn a_site_cased_key_and_name_is_not_kept() { + let m = [user("Bob.Smith@Ex.Test")]; + let s = shape_of_body( + Some(URLENC), + b"bob.smith%40ex.test=1&BOB.SMITH@EX.TEST=2", + &m, + None, + ); + let BodyShape::Urlencoded { fields } = s else { + panic!() + }; + assert!(fields.iter().all(|f| f.key == "{typed}")); + let s = shape_of_body( + Some("application/json"), + br#"{"bob.smith@ex.test":1}"#, + &m, + None, + ); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = s + else { + panic!() + }; + assert_eq!(fields[0].key, "{typed}"); + let body = + b"--b\r\nContent-Disposition: form-data; name=\"BOB.smith@ex.test\"\r\n\r\nx\r\n--b--"; + let s = shape_of_body(Some("multipart/form-data; boundary=b"), body, &m, None); + let BodyShape::Multipart { parts } = s else { + panic!() + }; + assert_eq!(parts[0].key, "{typed}"); + } + + #[test] + fn a_name_holding_the_combined_credential_is_not_kept() { let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); let (u, p) = (user("kh.throwaway7"), pw("Tr0ub4dor-9x")); - let m = markers(); + let body = format!("{b64}=1"); + let s = shape_of_body(Some(URLENC), body.as_bytes(), &[], Some((&u, &p))); + let BodyShape::Urlencoded { fields } = s else { + panic!() + }; + assert_eq!(fields[0].key, "{typed}"); + } + + #[test] + fn many_crlf_parts_are_read_in_linear_time() { + let mut body = Vec::new(); + for i in 0..5000 { + body.extend_from_slice( + format!("--b\r\nContent-Disposition: form-data; name=\"f{i}\"\r\n\r\nv\r\n") + .as_bytes(), + ); + } + body.extend_from_slice(b"--b--\r\n"); + let s = shape("multipart/form-data; boundary=b", &body); + let BodyShape::Multipart { parts } = s else { + panic!("not multipart") + }; + assert_eq!(parts.len(), 5000); + // LF-only bodies too. + let lf = String::from_utf8_lossy(&body).replace("\r\n", "\n"); + let s = shape("multipart/form-data; boundary=b", lf.as_bytes()); + let BodyShape::Multipart { parts } = s else { + panic!("not multipart") + }; + assert_eq!(parts.len(), 5000); + } + + #[test] + fn serialized_shapes_never_contain_a_marker() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let values = [ + "kh.throwaway7", + "Tr0ub4dor-9x", + "Tr0ub4/dor", + "Bob.Smith@Ex.Test", + ]; + let m: Vec = vec![ + user(values[0]), + pw(values[1]), + pw(values[2]), + user(values[3]), + ]; + let (u, p) = (user(values[0]), pw(values[1])); + let pair = Some((&u, &p)); let json_pair = format!(r#"{{"auth":"{b64}","n":1}}"#); - let multipart = b"--XyZ\r\nContent-Disposition: form-data; name=\"password\"\r\n\r\nTr0ub4dor-9x\r\n--XyZ\r\nContent-Disposition: form-data; name=\"kh.throwaway7\"; filename=\"kh.throwaway7\"\r\n\r\nkh.throwaway7\r\n--XyZ--\r\n"; - let bodies: Vec = vec![ - shape( - URLENC, - b"user=kh.throwaway7&password=Tr0ub4dor-9x&remember=1", - ), - shape(URLENC, b"password=Tr0ub4dor%2D9x&kh.throwaway7=1"), - shape( + let multipart = b"--XyZ\r\nContent-Disposition: form-data; name=\"password\"\r\n\r\nTr0ub4dor-9x\r\n--XyZ\r\nContent-Disposition: form-data; name=\"kh.throwaway7\"; filename=\"kh.throwaway7\"\r\n\r\nkh.throwaway7\r\n--XyZ\r\nContent-Disposition: form-data; name=\"bob.smith@ex.test\"\r\n\r\nTr0ub4/dor\r\n--XyZ--\r\n"; + let body = |ct: &str, bytes: &[u8]| shape_of_body(Some(ct), bytes, &m, pair); + let bodies: Vec = + vec![ + body(URLENC, b"user=kh.throwaway7&password=Tr0ub4dor-9x&remember=1"), + body(URLENC, b"password=Tr0ub4dor%2D9x&kh.throwaway7=1&p=Tr0ub4%2Fdor"), + body(URLENC, b"bob.smith%40ex.test=1&BOB.SMITH@EX.TEST=2"), + body(URLENC, format!("{b64}=1").as_bytes()), + body( "application/json", br#"{"auth":{"id":"kh.throwaway7","secret":"Tr0ub4dor-9x"},"n":3}"#, ), - shape( + body( "application/json", br#"{"query":"x","variables":{"password":"Tr0ub4dor-9x"}}"#, ), - shape( + body( "application/json", - br#"{"users":{"kh.throwaway7":{"secret":"Tr0ub4dor-9x"}}}"#, - ), - shape("multipart/form-data; boundary=XyZ", multipart), - shape("application/octet-stream", b"Tr0ub4dor-9x kh.throwaway7"), - shape("application/json", b"{Tr0ub4dor-9x"), - shape_of_body( - Some("application/json"), - json_pair.as_bytes(), - &m, - Some((&u, &p)), + br#"{"users":{"kh.throwaway7":{"secret":"Tr0ub4dor-9x"},"bob.smith@ex.test":1}}"#, ), + body("application/json", json_pair.as_bytes()), + body("application/json", b"{Tr0ub4dor-9x"), + body("multipart/form-data; boundary=XyZ", multipart), + body("application/octet-stream", b"Tr0ub4dor-9x kh.throwaway7"), ]; let mut outputs: Vec = bodies .iter() @@ -771,26 +827,37 @@ mod tests { .collect(); let urls = [ "https://kh.throwaway7.ex.test/login", + "https://Bob.Smith@Ex.Test.ex.test/login", + "https://a.kh.throwaway7.ex.kh.throwaway7.test/", "https://ex.test/u/kh.throwaway7/dash?kh.throwaway7=1&password=Tr0ub4dor-9x", + "https://ex.test/u/kh%2Ethrow%61way7/x?kh%2Ethrowaway7=1", "https://ex.test/Tr0ub4dor-9x?x=Tr0ub4dor-9x#Tr0ub4dor-9x", "https://kh.throwaway7:Tr0ub4dor-9x@ex.test/", + "https://ex.test/a/Tr0ub4/dor/z", + "https://ex.test/a/Tr0ub4%2Fdor/z?Tr0ub4%2Fdor=1", + "https://ex.test/u/bob.smith@ex.test/x?BOB.SMITH%40EX.TEST=1", + "data:text/plain,Tr0ub4dor-9x,kh.throwaway7", + "kh.throwaway7://ex.test/", ]; for url in urls { - let s = strip_url(url, &m).unwrap(); + let s = strip_url(url, &m, pair).unwrap(); outputs.push(serde_json::to_string(&s.value).unwrap()); } + let in_path = format!("https://ex.test/{b64}/x?{b64}=1"); + outputs + .push(serde_json::to_string(&strip_url(&in_path, &[], pair).unwrap().value).unwrap()); for header in [ format!("Basic {b64}"), "Bearer Tr0ub4dor-9x".to_owned(), "Tr0ub4dor-9x".to_owned(), + "kh.throwaway7".to_owned(), "Basic Tr0ub4dor-9x".to_owned(), ] { - let a = reduce_authorization(header.as_bytes(), &m, Some((&u, &p))); + let a = reduce_authorization(header.as_bytes(), &m, pair); outputs.push(serde_json::to_string(&a).unwrap()); } for text in outputs { - assert!(!text.contains("Tr0ub4dor-9x"), "{text}"); - assert!(!text.contains("kh.throwaway7"), "{text}"); + assert_holds_no_typed_value(&text, &values); } } } diff --git a/crates/kharon-teach/src/lib.rs b/crates/kharon-teach/src/lib.rs index 7a1eebb..b572f0d 100644 --- a/crates/kharon-teach/src/lib.rs +++ b/crates/kharon-teach/src/lib.rs @@ -3,3 +3,5 @@ pub mod body; pub mod marker; pub mod strip; +#[doc(hidden)] +pub mod testing; diff --git a/crates/kharon-teach/src/strip.rs b/crates/kharon-teach/src/strip.rs index 8225d48..2f35966 100644 --- a/crates/kharon-teach/src/strip.rs +++ b/crates/kharon-teach/src/strip.rs @@ -1,4 +1,10 @@ //! Stripping: a URL or an `Authorization` header reduced to structure only. +//! +//! Everything this module KEEPS (origin, path, query keys, scheme) is tested for typed values as a +//! whole string, never piece by piece, and with ASCII letter case ignored, because a site may +//! lowercase what was typed and a typed value may itself hold a separator. + +use std::ops::Range; use base64ct::{Base64, Base64Unpadded, Base64Url, Base64UrlUnpadded, Encoding as B64}; use kharon_recording::{AuthShape, Encoding, MarkerHit, StrippedUrl}; @@ -12,6 +18,38 @@ use crate::marker::{Marker, search, search_pair}; pub const TYPED: &str = "{typed}"; /// Written in place of a path segment that is an identifier or held a typed value. pub const ID: &str = "{id}"; +const OTHER: &str = "{other}"; + +/// URL schemes that are kept; any other is written `{other}`. +const URL_SCHEMES: [&str; 11] = [ + "http", + "https", + "ws", + "wss", + "data", + "blob", + "about", + "file", + "chrome", + "chrome-extension", + "devtools", +]; + +/// Authorization schemes that are kept (compared lowercased); any other is written `{other}`. +const AUTH_SCHEMES: [&str; 12] = [ + "basic", + "bearer", + "digest", + "negotiate", + "ntlm", + "hoba", + "mutual", + "token", + "apikey", + "oauth", + "hawk", + "aws4-hmac-sha256", +]; pub struct Stripped { pub value: T, @@ -35,15 +73,17 @@ pub(crate) fn hits_in( dedup_hits(out) } -/// Keeps the first hit for each (role, encoding). +/// One hit per (role, encoding), in first-seen order. When two share a key the result is +/// `low_confidence: false` if either is. pub(crate) fn dedup_hits(hits: Vec) -> Vec { let mut out: Vec = Vec::with_capacity(hits.len()); for h in hits { - if !out - .iter() - .any(|k| k.role == h.role && k.encoding == h.encoding) + match out + .iter_mut() + .find(|k| k.role == h.role && k.encoding == h.encoding) { - out.push(h); + Some(known) => known.low_confidence &= h.low_confidence, + None => out.push(h), } } out @@ -58,57 +98,118 @@ pub(crate) fn any_hit( !hits_in(haystack, markers, pair).is_empty() } +/// Percent-decodes `raw`, with `+` as a space first when `plus_is_space`. A lone `%` stays. +pub(crate) fn decode_percent(raw: &[u8], plus_is_space: bool) -> Zeroizing> { + if plus_is_space { + let spaced: Zeroizing> = Zeroizing::new( + raw.iter() + .map(|b| if *b == b'+' { b' ' } else { *b }) + .collect(), + ); + return Zeroizing::new(percent_decode(&spaced).collect()); + } + Zeroizing::new(percent_decode(raw).collect()) +} + +/// `bytes` as text of the SAME byte length: every invalid UTF-8 byte becomes `?`, so byte +/// offsets in the text are offsets in `bytes`. +fn text_same_len(bytes: &[u8]) -> Zeroizing { + let mut out = Zeroizing::new(String::with_capacity(bytes.len())); + for chunk in bytes.utf8_chunks() { + out.push_str(chunk.valid()); + out.extend(std::iter::repeat_n('?', chunk.invalid().len())); + } + out +} + +/// Byte ranges of every occurrence (overlapping ones too) of every marker in `text`, ignoring +/// ASCII letter case. +fn occurrences(text: &str, markers: &[Marker]) -> Vec> { + let mut out = Vec::new(); + for m in markers { + let mut from = 0; + while let Some(r) = text + .get(from..) + .and_then(|rest| m.find_ignore_ascii_case(rest)) + { + let (start, end) = (from + r.start, from + r.end); + out.push(start..end); + // Resume one character after the start so overlapping occurrences are found. + from = start + text[start..].chars().next().map_or(1, char::len_utf8); + } + } + out +} + +/// For each span, whether any range overlaps it. +fn touched(ranges: &[Range], spans: &[Range]) -> Vec { + spans + .iter() + .map(|s| ranges.iter().any(|r| r.start < s.end && r.end > s.start)) + .collect() +} + +/// Whether kept text `bytes` holds a typed value: `search` finds any hit (any encoding, the pair +/// too), or the value occurs ignoring ASCII case in the text or in its percent-decoded forms. +pub(crate) fn holds_typed( + bytes: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> bool { + if any_hit(bytes, markers, pair) || !occurrences(&text_same_len(bytes), markers).is_empty() { + return true; + } + [false, true].into_iter().any(|plus| { + let decoded = decode_percent(bytes, plus); + decoded.as_slice() != bytes && !occurrences(&text_same_len(&decoded), markers).is_empty() + }) +} + /// Origin, path and query keys of `url`. A path segment becomes "{id}" when it is all ASCII digits, -/// a UUID, 16 or more hex characters, or contains any marker (as typed or percent-encoded). +/// a UUID, 16 or more hex characters, or any match of a marker (see below) touches it. /// /// Userinfo and fragment are dropped. A query key or host label holding a typed value is written -/// as "{typed}". Known limit: an internationalised host is compared in its punycode form, so a -/// non-ASCII typed value inside such a label is not recognised. -pub fn strip_url(url: &str, markers: &[Marker]) -> Option> { +/// as "{typed}". Matching is over the WHOLE path, query and host (raw and percent-decoded, ASCII +/// case ignored; the path also with `/` read as `\`, which the parser rewrites), and every +/// segment, key or label a match touches is replaced, so a typed value holding a separator +/// leaves no piece behind. A combined credential (`pair`) found in a segment or key replaces it. +/// +/// A URL that cannot be a base (`data:`, `blob:`, `about:`, `mailto:`) keeps only its scheme. +/// The scheme is kept only when it is in a short known list, else it is `{other}`. +/// Known limits: an internationalised host is compared in its punycode form, so a non-ASCII +/// typed value inside such a label is not recognised; the port is not tested. +pub fn strip_url( + url: &str, + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> Option> { let parsed = Url::parse(url).ok()?; - let mut path_segments_replaced = 0u64; - + let scheme = if URL_SCHEMES.contains(&parsed.scheme()) { + parsed.scheme() + } else { + OTHER + }; + if parsed.cannot_be_a_base() { + return Some(Stripped { + value: StrippedUrl { + origin: format!("{scheme}:"), + path: String::new(), + query_keys: Vec::new(), + }, + query_values_removed: 0, + path_segments_replaced: 0, + }); + } let origin = match parsed.host_str() { Some(host) => { let port = parsed.port().map(|p| format!(":{p}")).unwrap_or_default(); - format!( - "{}://{}{}", - parsed.scheme(), - strip_host(host, markers), - port - ) - } - None => format!("{}:", parsed.scheme()), - }; - - let path = if parsed.cannot_be_a_base() { - // Opaque path (`data:`, `mailto:` ...): it is a payload, not a route. - path_segments_replaced += 1; - ID.to_owned() - } else { - let mut owned: Vec = Vec::new(); - for seg in parsed.path().split('/') { - if segment_is_id(seg, markers) { - path_segments_replaced += 1; - owned.push(ID.to_owned()); - } else { - owned.push(seg.to_owned()); - } + format!("{scheme}://{}{port}", strip_host(host, markers)) } - owned.join("/") + None => format!("{scheme}:"), }; - - let mut query_keys = Vec::new(); - let mut query_values_removed = 0u64; - for (key, _value) in parsed.query_pairs() { - query_values_removed += 1; - if any_hit(key.as_bytes(), markers, None) { - query_keys.push(TYPED.to_owned()); - } else { - query_keys.push(key.into_owned()); - } - } - + let (path, path_segments_replaced) = strip_path(parsed.path(), markers, pair); + let (query_keys, query_values_removed) = + strip_query(parsed.query().unwrap_or(""), markers, pair); Some(Stripped { value: StrippedUrl { origin, @@ -125,48 +226,101 @@ pub fn strip_url(url: &str, markers: &[Marker]) -> Option> /// replaced. Adjacent replaced labels become ONE `{typed}` label, so a value that spans labels /// (`kh.throwaway7`) leaves no part of itself behind. fn strip_host(host: &str, markers: &[Marker]) -> String { - let mut masked = vec![false; host.len()]; - for m in markers { - let mut from = 0; - while let Some(range) = host - .get(from..) - .and_then(|rest| m.find_ignore_ascii_case(rest)) - { - let (start, end) = (from + range.start, from + range.end); - masked[start..end].iter_mut().for_each(|b| *b = true); - // Resume one character after the start so overlapping occurrences are found. - from = start + host[start..].chars().next().map_or(1, char::len_utf8); - } - } - let mut labels: Vec<&str> = Vec::new(); + let ranges = occurrences(host, markers); + let mut spans = Vec::new(); let mut offset = 0; - let mut previous_typed = false; for label in host.split('.') { - let typed = masked[offset..offset + label.len()].iter().any(|b| *b); + spans.push(offset..offset + label.len()); offset += label.len() + 1; - if typed { + } + let typed = touched(&ranges, &spans); + let mut labels: Vec<&str> = Vec::new(); + let mut previous_typed = false; + for (label, is_typed) in host.split('.').zip(typed) { + if is_typed { if !previous_typed { labels.push(TYPED); } } else { labels.push(label); } - previous_typed = typed; + previous_typed = is_typed; } labels.join(".") } -fn segment_is_id(seg: &str, markers: &[Marker]) -> bool { - if seg.is_empty() { - return false; +/// Segments of `path` are replaced when they look like an identifier or a match touches them. +fn strip_path(path: &str, markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> (String, u64) { + let segments: Vec<&str> = path.split('/').collect(); + let decoded: Vec>> = segments + .iter() + .map(|s| decode_percent(s.as_bytes(), false)) + .collect(); + let decoded_text: Vec> = decoded.iter().map(|d| text_same_len(d)).collect(); + + let spans_of = |lens: Vec| -> Vec> { + let mut offset = 0; + lens.into_iter() + .map(|len| { + let span = offset..offset + len; + offset += len + 1; + span + }) + .collect() + }; + let raw_spans = spans_of(segments.iter().map(|s| s.len()).collect()); + let decoded_spans = spans_of(decoded_text.iter().map(|t| t.len()).collect()); + let mut decoded_path = Zeroizing::new(String::with_capacity(path.len())); + for (i, t) in decoded_text.iter().enumerate() { + if i > 0 { + decoded_path.push('/'); + } + decoded_path.push_str(t); + } + + // The raw and decoded whole path, each also with `/` read as `\` (same byte length, so the + // same spans apply). + let mut typed = vec![false; segments.len()]; + for (text, spans) in [(path, &raw_spans), (decoded_path.as_str(), &decoded_spans)] { + let backslashed = Zeroizing::new(text.replace('/', "\\")); + for variant in [text, backslashed.as_str()] { + let hit = touched(&occurrences(variant, markers), spans); + typed.iter_mut().zip(hit).for_each(|(t, h)| *t |= h); + } } - let all_digits = seg.bytes().all(|b| b.is_ascii_digit()); - let long_hex = seg.len() >= 16 && seg.bytes().all(|b| b.is_ascii_hexdigit()); - if all_digits || long_hex || is_uuid(seg) { - return true; + for (i, seg) in segments.iter().enumerate() { + typed[i] |= any_hit(seg.as_bytes(), markers, pair) || any_hit(&decoded[i], markers, pair); } - let decoded: Zeroizing> = Zeroizing::new(percent_decode(seg.as_bytes()).collect()); - any_hit(seg.as_bytes(), markers, None) || any_hit(&decoded, markers, None) + // A hit over the whole path that no segment explains (an encoded value that spans a `/`): + // every non-empty segment goes. + if !typed.iter().any(|t| *t) + && (any_hit(path.as_bytes(), markers, pair) + || any_hit(decoded_path.as_bytes(), markers, pair)) + { + typed = segments.iter().map(|s| !s.is_empty()).collect(); + } + + let mut replaced = 0u64; + let kept: Vec<&str> = segments + .iter() + .zip(typed) + .map(|(seg, is_typed)| { + if is_typed || segment_is_id(seg) { + replaced += 1; + ID + } else { + seg + } + }) + .collect(); + (kept.join("/"), replaced) +} + +fn segment_is_id(seg: &str) -> bool { + !seg.is_empty() + && (seg.bytes().all(|b| b.is_ascii_digit()) + || (seg.len() >= 16 && seg.bytes().all(|b| b.is_ascii_hexdigit())) + || is_uuid(seg)) } fn is_uuid(seg: &str) -> bool { @@ -178,13 +332,81 @@ fn is_uuid(seg: &str) -> bool { }) } +/// Query keys, in order, with a key a match touches written `{typed}`. The raw query string and a +/// decoded copy (`+` as space, `%XX`) are each searched whole; a match touching a key's bytes +/// (separators included) marks the key. A match inside a value alone costs nothing. +fn strip_query( + query: &str, + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> (Vec, u64) { + struct Piece<'a> { + raw_key: &'a str, + raw_span: Range, + key: Zeroizing>, + decoded_span: Range, + } + let mut pieces: Vec = Vec::new(); + let mut decoded_query = Zeroizing::new(String::with_capacity(query.len())); + let mut offset = 0; + for piece in query.split('&') { + let start = offset; + offset += piece.len() + 1; + if piece.is_empty() { + continue; + } + let (raw_key, value) = match piece.split_once('=') { + Some((k, v)) => (k, Some(v)), + None => (piece, None), + }; + let key = decode_percent(raw_key.as_bytes(), true); + let key_text = text_same_len(&key); + if !decoded_query.is_empty() { + decoded_query.push('&'); + } + let decoded_start = decoded_query.len(); + decoded_query.push_str(&key_text); + let decoded_span = decoded_start..decoded_query.len(); + if let Some(v) = value { + decoded_query.push('='); + decoded_query.push_str(&text_same_len(&decode_percent(v.as_bytes(), true))); + } + pieces.push(Piece { + raw_key, + raw_span: start..start + raw_key.len(), + key, + decoded_span, + }); + } + let raw_spans: Vec> = pieces.iter().map(|p| p.raw_span.clone()).collect(); + let decoded_spans: Vec> = pieces.iter().map(|p| p.decoded_span.clone()).collect(); + let by_raw = touched(&occurrences(query, markers), &raw_spans); + let by_decoded = touched(&occurrences(&decoded_query, markers), &decoded_spans); + let keys = pieces + .iter() + .enumerate() + .map(|(i, p)| { + if by_raw[i] + || by_decoded[i] + || any_hit(p.raw_key.as_bytes(), markers, pair) + || any_hit(&p.key, markers, pair) + { + TYPED.to_owned() + } else { + String::from_utf8_lossy(&p.key).into_owned() + } + }) + .collect(); + (keys, u64::try_from(pieces.len()).unwrap_or(u64::MAX)) +} + /// scheme = first token, lowercased. For `basic`, the credentials are base64-decoded and the decoded bytes /// are searched; a marker found there is reported with `Encoding::Base64`. Other schemes are searched as sent. /// When `pair` is given the value is also searched with `search_pair`. No (role, encoding) is listed twice. /// -/// The value as sent is always searched too. The scheme is kept only when it is a plain token -/// (at most 32 ASCII letters, digits, `-` or `_`) holding no typed value; otherwise it is -/// `{typed}` or `{other}`, so a bare credential sent with no scheme is not copied into the file. +/// The value as sent is always searched too. The scheme is kept only when it is one of a short +/// known list (`AUTH_SCHEMES`); a scheme holding a typed value is `{typed}`; anything else, +/// including a header with no space (a bare credential), is `{other}`. pub fn reduce_authorization( header_value: &[u8], markers: &[Marker], @@ -194,6 +416,7 @@ pub fn reduce_authorization( Some(i) => (&header_value[..i], &header_value[i + 1..]), None => (header_value, &header_value[..0]), }; + let has_credentials = scheme_bytes.len() < header_value.len(); let mut hits = hits_in(header_value, markers, pair); let scheme_lower = scheme_bytes.to_ascii_lowercase(); @@ -207,17 +430,13 @@ pub fn reduce_authorization( })); } - let scheme = if any_hit(scheme_bytes, markers, pair) { + let known = std::str::from_utf8(&scheme_lower) + .ok() + .filter(|s| has_credentials && AUTH_SCHEMES.contains(s)); + let scheme = if holds_typed(scheme_bytes, markers, pair) { TYPED.to_owned() - } else if !scheme_lower.is_empty() - && scheme_lower.len() <= 32 - && scheme_lower - .iter() - .all(|b| b.is_ascii_alphanumeric() || *b == b'-' || *b == b'_') - { - String::from_utf8_lossy(&scheme_lower).into_owned() } else { - "{other}".to_owned() + known.map_or_else(|| OTHER.to_owned(), str::to_owned) }; AuthShape { @@ -245,14 +464,9 @@ fn decode_basic(credentials: &[u8]) -> Option>> { #[cfg(test)] mod tests { use super::*; + use crate::testing::{assert_holds_no_typed_value, pw, user}; use kharon_recording::Role; - pub(crate) fn user(s: &str) -> Marker { - Marker::new(Zeroizing::new(s.to_owned()), Role::Username) - } - pub(crate) fn pw(s: &str) -> Marker { - Marker::new(Zeroizing::new(s.to_owned()), Role::Password) - } fn hit(role: Role, encoding: Encoding) -> MarkerHit { MarkerHit { role, @@ -263,7 +477,12 @@ mod tests { #[test] fn keeps_origin_path_and_query_keys() { - let s = strip_url("https://ex.test/login/step?next=%2Fhome&lang=en#frag", &[]).unwrap(); + let s = strip_url( + "https://ex.test/login/step?next=%2Fhome&lang=en#frag", + &[], + None, + ) + .unwrap(); assert_eq!( s.value, StrippedUrl { @@ -278,7 +497,7 @@ mod tests { #[test] fn origin_has_no_userinfo_keeps_a_port_and_duplicate_keys() { - let s = strip_url("http://bob:s3cret@ex.test:8080/a?k=1&k=2", &[]).unwrap(); + let s = strip_url("http://bob:s3cret@ex.test:8080/a?k=1&k=2", &[], None).unwrap(); assert_eq!(s.value.origin, "http://ex.test:8080"); assert_eq!(s.value.query_keys, ["k", "k"]); } @@ -293,7 +512,7 @@ mod tests { ("/v2/login", "/v2/login", 0), ("/a/b/", "/a/b/", 0), ] { - let s = strip_url(&format!("https://ex.test{path}"), &[]).unwrap(); + let s = strip_url(&format!("https://ex.test{path}"), &[], None).unwrap(); assert_eq!(s.value.path, want); assert_eq!(s.path_segments_replaced, n); } @@ -302,24 +521,24 @@ mod tests { #[test] fn replaces_a_segment_that_contains_a_marker() { let m = [user("kh.throwaway7")]; - let s = strip_url("https://ex.test/u/kh.throwaway7/dash", &m).unwrap(); + let s = strip_url("https://ex.test/u/kh.throwaway7/dash", &m, None).unwrap(); assert_eq!(s.value.path, "/u/{id}/dash"); - let s = strip_url("https://ex.test/u/kh%2Ethrow%61way7/dash", &m).unwrap(); + let s = strip_url("https://ex.test/u/kh%2Ethrow%61way7/dash", &m, None).unwrap(); assert_eq!(s.value.path, "/u/{id}/dash"); assert_eq!(s.path_segments_replaced, 1); } #[test] fn unparseable_url_is_none() { - assert!(strip_url("not a url", &[]).is_none()); + assert!(strip_url("not a url", &[], None).is_none()); } #[test] fn a_query_key_that_holds_a_typed_value_is_not_kept() { let m = [user("kh.throwaway7")]; - let s = strip_url("https://ex.test/x?kh.throwaway7=1&ok=2", &m).unwrap(); + let s = strip_url("https://ex.test/x?kh.throwaway7=1&ok=2", &m, None).unwrap(); assert_eq!(s.value.query_keys, ["{typed}", "ok"]); - let s = strip_url("https://ex.test/x?kh%2Ethrowaway7=1", &m).unwrap(); + let s = strip_url("https://ex.test/x?kh%2Ethrowaway7=1", &m, None).unwrap(); assert_eq!(s.value.query_keys, ["{typed}"]); } @@ -327,15 +546,15 @@ mod tests { fn a_host_label_that_holds_a_typed_value_is_replaced() { let m = [user("kh.throwaway7")]; // The value spans two labels: the whole run of labels the match covers is one `{typed}`. - let s = strip_url("https://kh.throwaway7.ex.test/login", &m).unwrap(); + let s = strip_url("https://kh.throwaway7.ex.test/login", &m, None).unwrap(); assert_eq!(s.value.origin, "https://{typed}.ex.test"); // Case-insensitive, and a partial label is replaced whole. let m = [user("Kh.Throwaway7")]; - let s = strip_url("https://x-kh.throwaway7-y.ex.test:81/", &m).unwrap(); + let s = strip_url("https://x-kh.throwaway7-y.ex.test:81/", &m, None).unwrap(); assert_eq!(s.value.origin, "https://{typed}.ex.test:81"); // Every occurrence. let m = [user("bob12")]; - let s = strip_url("https://bob12.ex.bob12.test/", &m).unwrap(); + let s = strip_url("https://bob12.ex.bob12.test/", &m, None).unwrap(); assert_eq!(s.value.origin, "https://{typed}.ex.{typed}.test"); } @@ -404,7 +623,7 @@ mod tests { assert_eq!( a, AuthShape { - scheme: "negotiate".into(), + scheme: "{other}".into(), markers: vec![] } ); @@ -430,4 +649,112 @@ mod tests { let a = reduce_authorization(b"Basic Tr0ub4dor-9x", &[pw("Tr0ub4dor-9x")], None); assert_eq!(a.markers, [hit(Role::Password, Encoding::AsTyped)]); } + + #[test] + fn a_typed_value_holding_a_slash_is_not_kept_in_the_path() { + let m = [pw("Tr0ub4/dor")]; + let s = strip_url("https://ex.test/a/Tr0ub4/dor", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}/{id}"); + assert_eq!(s.path_segments_replaced, 2); + let s = strip_url("https://ex.test/a/Tr0ub4%2Fdor/z", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}/z"); + // The parser rewrites a backslash in the path to a slash. + let m = [pw("Tr0ub4\\dor")]; + let s = strip_url("https://ex.test/a/Tr0ub4\\dor/z", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}/{id}/z"); + } + + #[test] + fn a_site_cased_value_is_not_kept_in_the_path_or_query() { + let m = [user("Bob.Smith@Ex.Test")]; + let s = strip_url( + "https://ex.test/u/bob.smith@ex.test/x?BOB.SMITH%40EX.TEST=1&ok=2", + &m, + None, + ) + .unwrap(); + assert_eq!(s.value.path, "/u/{id}/x"); + assert_eq!(s.value.query_keys, ["{typed}", "ok"]); + } + + #[test] + fn a_separator_inside_a_typed_value_does_not_leave_a_key() { + let m = [pw("pa&ss")]; + let s = strip_url("https://ex.test/x?a=pa&ss=1&b=2", &m, None).unwrap(); + assert_eq!(s.value.query_keys, ["a", "{typed}", "b"]); + let m = [pw("pa=ss")]; + let s = strip_url("https://ex.test/x?pa=ss&b=2", &m, None).unwrap(); + assert_eq!(s.value.query_keys, ["{typed}", "b"]); + // A typed value sitting in a query VALUE does not cost its key. + let m = [pw("Tr0ub4dor-9x")]; + let s = strip_url("https://ex.test/x?p=Tr0ub4dor-9x", &m, None).unwrap(); + assert_eq!(s.value.query_keys, ["p"]); + } + + #[test] + fn the_combined_credential_in_a_path_or_key_is_not_kept() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let (u, p) = (user("kh.throwaway7"), pw("Tr0ub4dor-9x")); + let s = strip_url(&format!("https://ex.test/a/{b64}/z"), &[], Some((&u, &p))).unwrap(); + assert!(s.path_segments_replaced >= 1); + assert_holds_no_typed_value(&s.value.path, &["kh.throwaway7:Tr0ub4dor-9x"]); + let url = format!("https://ex.test/a?{}=1", b64.trim_end_matches('=')); + let s = strip_url(&url, &[], Some((&u, &p))).unwrap(); + assert_eq!(s.value.query_keys, ["{typed}"]); + } + + #[test] + fn a_url_that_cannot_be_a_base_keeps_only_its_scheme() { + for (url, origin) in [ + ("data:text/plain,Tr0ub4dor-9x", "data:"), + ("blob:https://ex.test/3f2b", "blob:"), + ("about:blank", "about:"), + ("mailto:a@b.test?subject=x", "{other}:"), + ] { + let s = strip_url(url, &[], None).unwrap(); + assert_eq!( + s.value, + StrippedUrl { + origin: origin.into(), + path: String::new(), + query_keys: vec![] + } + ); + } + } + + #[test] + fn an_unknown_url_scheme_is_other() { + let s = strip_url("kh.throwaway7://ex.test/x", &[user("kh.throwaway7")], None).unwrap(); + assert_eq!(s.value.origin, "{other}://ex.test"); + let s = strip_url("chrome-extension://abc/x", &[], None).unwrap(); + assert_eq!(s.value.origin, "chrome-extension://abc"); + } + + #[test] + fn only_a_known_authorization_scheme_is_kept() { + for (header, want) in [ + (&b"Digest username=x"[..], "digest"), + (b"AWS4-HMAC-SHA256 Credential=x", "aws4-hmac-sha256"), + (b"Negotiate abc", "negotiate"), + (b"s3cr3t-session-token-0123456789 x", "{other}"), + (b"s3cr3t-session-token-0123456789", "{other}"), + (b"Tr0ub4dor-9x", "{typed}"), + (b"Tr0ub4dor-9x rest", "{typed}"), + ] { + let a = reduce_authorization(header, &[pw("Tr0ub4dor-9x")], None); + assert_eq!(a.scheme, want); + } + } + + #[test] + fn a_confident_duplicate_wins_over_a_low_confidence_one() { + let h = |low| MarkerHit { + role: Role::Password, + encoding: Encoding::AsTyped, + low_confidence: low, + }; + assert_eq!(dedup_hits(vec![h(true), h(false)]), [h(false)]); + assert_eq!(dedup_hits(vec![h(false), h(true)]), [h(false)]); + } } diff --git a/crates/kharon-teach/src/testing.rs b/crates/kharon-teach/src/testing.rs new file mode 100644 index 0000000..33e9396 --- /dev/null +++ b/crates/kharon-teach/src/testing.rs @@ -0,0 +1,104 @@ +//! Support for this crate's tests. It takes only test constants, never a live typed value. +//! +//! It is public (and hidden from the docs) so unit tests and the integration tests under +//! `tests/` share one definition of "the encodings a site might apply to a value". + +use base64ct::{Base64, Base64Unpadded, Base64Url, Base64UrlUnpadded, Encoding as B64}; +use kharon_recording::Role; +use md5::Md5; +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode, percent_encode}; +use sha1::Sha1; +use sha2::{Digest, Sha256, Sha512}; +use zeroize::Zeroizing; + +use crate::marker::Marker; + +/// A username marker holding `value`. +pub fn user(value: &str) -> Marker { + Marker::new(Zeroizing::new(value.to_owned()), Role::Username) +} + +/// A password marker holding `value`. +pub fn pw(value: &str) -> Marker { + Marker::new(Zeroizing::new(value.to_owned()), Role::Password) +} + +const UNRESERVED: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~'); + +/// Label and bytes of `value` as typed, ASCII-lowercased, ASCII-uppercased, percent-encoded +/// (RFC 3986 strict, upper and lower hex), form-encoded (space as `+`), JSON-escaped, base64 and +/// base64url (padded and not), hex (lower and upper), and the hex of MD5, SHA-1, SHA-256 and +/// SHA-512. A form identical to an earlier one is skipped. +pub fn needles_for(value: &str) -> Vec<(&'static str, Vec)> { + let bytes = value.as_bytes(); + let strict: String = percent_encode(bytes, UNRESERVED).to_string(); + let lower_hex = { + let mut out = String::new(); + for chunk in percent_encode(bytes, UNRESERVED) { + if chunk.starts_with('%') { + out.push_str(&chunk.to_ascii_lowercase()); + } else { + out.push_str(chunk); + } + } + out + }; + let json = serde_json::to_string(value).unwrap_or_default(); + let json = json + .strip_prefix('"') + .and_then(|j| j.strip_suffix('"')) + .unwrap_or_default() + .to_owned(); + let hex_of = |b: &[u8]| hex::encode(b); + let candidates: Vec<(&'static str, Vec)> = vec![ + ("as typed", bytes.to_vec()), + ("lowercased", value.to_ascii_lowercase().into_bytes()), + ("uppercased", value.to_ascii_uppercase().into_bytes()), + ("percent upper", strict.clone().into_bytes()), + ("percent lower", lower_hex.into_bytes()), + ("form encoded", strict.replace("%20", "+").into_bytes()), + ("json escaped", json.into_bytes()), + ("base64", Base64::encode_string(bytes).into_bytes()), + ( + "base64 unpadded", + Base64Unpadded::encode_string(bytes).into_bytes(), + ), + ("base64url", Base64Url::encode_string(bytes).into_bytes()), + ( + "base64url unpadded", + Base64UrlUnpadded::encode_string(bytes).into_bytes(), + ), + ("hex lower", hex_of(bytes).into_bytes()), + ("hex upper", hex_of(bytes).to_uppercase().into_bytes()), + ("md5 hex", hex_of(&Md5::digest(bytes)).into_bytes()), + ("sha1 hex", hex_of(&Sha1::digest(bytes)).into_bytes()), + ("sha256 hex", hex_of(&Sha256::digest(bytes)).into_bytes()), + ("sha512 hex", hex_of(&Sha512::digest(bytes)).into_bytes()), + ]; + let mut out: Vec<(&'static str, Vec)> = Vec::new(); + for (label, form) in candidates { + if !form.is_empty() && !out.iter().any(|(_, f)| *f == form) { + out.push((label, form)); + } + } + out +} + +/// Panics when any needle of any of `values` occurs in `text` or in its percent-decoded form. +/// The message names the value's position and the encoding, never `text` or a value. +pub fn assert_holds_no_typed_value(text: &str, values: &[&str]) { + let decoded: Vec = percent_decode(text.as_bytes()).collect(); + for (i, value) in values.iter().enumerate() { + for (label, needle) in needles_for(value) { + let found = |h: &[u8]| h.windows(needle.len()).any(|w| w == needle.as_slice()); + assert!( + !found(text.as_bytes()) && !found(&decoded), + "typed value #{i} found in the output, as: {label}" + ); + } + } +} From b5e8dff3b8609a8053f4f3439e11b18d8bc19282 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:25:18 -0500 Subject: [PATCH 13/36] feat(teach): capture state machine from events to a stripped recording (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- Cargo.lock | 1 + crates/kharon-teach/Cargo.toml | 1 + crates/kharon-teach/src/capture/build.rs | 209 +++ crates/kharon-teach/src/capture/mod.rs | 1430 ++++++++++++++++++++ crates/kharon-teach/src/capture/resolve.rs | 156 +++ crates/kharon-teach/src/capture/types.rs | 98 ++ crates/kharon-teach/src/lib.rs | 2 + crates/kharon-teach/src/site.rs | 76 ++ 8 files changed, 1973 insertions(+) create mode 100644 crates/kharon-teach/src/capture/build.rs create mode 100644 crates/kharon-teach/src/capture/mod.rs create mode 100644 crates/kharon-teach/src/capture/resolve.rs create mode 100644 crates/kharon-teach/src/capture/types.rs create mode 100644 crates/kharon-teach/src/site.rs diff --git a/Cargo.lock b/Cargo.lock index e83dee8..caa5f4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1089,6 +1089,7 @@ dependencies = [ "md-5", "memchr", "percent-encoding", + "publicsuffix", "serde_json", "sha1", "sha2", diff --git a/crates/kharon-teach/Cargo.toml b/crates/kharon-teach/Cargo.toml index d0d00e2..69fcf09 100644 --- a/crates/kharon-teach/Cargo.toml +++ b/crates/kharon-teach/Cargo.toml @@ -27,3 +27,4 @@ percent-encoding = "2" memchr = "2" url = "2" serde_json = "1" +publicsuffix = "2.3" diff --git a/crates/kharon-teach/src/capture/build.rs b/crates/kharon-teach/src/capture/build.rs new file mode 100644 index 0000000..c0583ae --- /dev/null +++ b/crates/kharon-teach/src/capture/build.rs @@ -0,0 +1,209 @@ +//! Turning buffered events into the value-free parts of a `Recording`. + +use kharon_recording::{ + BodyShape, ElementDescriptor, FormRef, FrameRef, ResponseShape, Role, StoredRequest, + StrippedUrl, +}; + +use super::{Buffered, FrameInfo, RawBody, RawElement, RawForm}; +use crate::body::shape_of_body; +use crate::marker::Marker; +use crate::strip::{TYPED, hits_in, holds_typed, reduce_authorization, strip_url}; + +/// What is known while stripping: every typed value, and the credential pair once chosen. +#[derive(Clone, Copy)] +pub(super) struct Ctx<'a> { + pub markers: &'a [Marker], + pub pair: Option<(&'a Marker, &'a Marker)>, +} + +impl Ctx<'_> { + fn holds_typed(&self, text: &str) -> bool { + holds_typed(text.as_bytes(), self.markers, self.pair) + } +} + +/// Counts of what stripping removed, for URLs that are written to the file. +#[derive(Default, Clone, Copy)] +pub(super) struct Tally { + pub query_values_removed: u64, + pub path_segments_replaced: u64, +} + +impl Tally { + pub fn add(&mut self, other: Tally) { + self.query_values_removed += other.query_values_removed; + self.path_segments_replaced += other.path_segments_replaced; + } +} + +const OTHER_ORIGIN: &str = "{other}:"; + +/// `Some` when the URL parses. The tally is the URL's own counts. +fn strip(url: &str, ctx: Ctx<'_>) -> Option<(StrippedUrl, Tally)> { + strip_url(url, ctx.markers, ctx.pair).map(|s| { + ( + s.value, + Tally { + query_values_removed: s.query_values_removed, + path_segments_replaced: s.path_segments_replaced, + }, + ) + }) +} + +/// A URL for the file; one that does not parse becomes an empty `{other}:` origin. +pub(super) fn stripped_url(url: &str, ctx: Ctx<'_>, tally: &mut Tally) -> StrippedUrl { + match strip(url, ctx) { + Some((u, t)) => { + tally.add(t); + u + } + None => StrippedUrl { + origin: OTHER_ORIGIN.to_owned(), + path: String::new(), + query_keys: Vec::new(), + }, + } +} + +pub(super) fn frame_ref(info: &FrameInfo, ctx: Ctx<'_>) -> FrameRef { + FrameRef { + main: info.main, + origin: strip(&info.url, ctx).map_or_else(|| OTHER_ORIGIN.to_owned(), |(u, _)| u.origin), + cross_origin: info.cross_origin, + } +} + +/// An attribute is kept only when it holds no typed value. +fn clean(attr: Option<&String>, ctx: Ctx<'_>) -> Option { + attr.filter(|a| !ctx.holds_typed(a)).cloned() +} + +/// A selector holding a typed value, also once CSS escapes (`\.`) are undone, is removed. +fn selector_is_clean(selector: &str, ctx: Ctx<'_>) -> bool { + let unescaped: String = selector.chars().filter(|c| *c != '\\').collect(); + !ctx.holds_typed(selector) && !ctx.holds_typed(&unescaped) +} + +fn known_method(method: &str, lowercase: bool) -> String { + let upper = method.to_ascii_uppercase(); + let known = ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]; + if !known.contains(&upper.as_str()) { + return "{other}".to_owned(); + } + if lowercase { + upper.to_lowercase() + } else { + upper + } +} + +pub(super) fn form_ref(form: &RawForm, ctx: Ctx<'_>, tally: &mut Tally) -> FormRef { + FormRef { + index: form.index, + action: form + .action + .as_deref() + .and_then(|a| strip(a, ctx)) + .map(|(u, t)| { + tally.add(t); + u + }), + method: known_method(&form.method, true), + } +} + +pub(super) fn describe( + raw: &RawElement, + was_password: bool, + ctx: Ctx<'_>, + tally: &mut Tally, +) -> ElementDescriptor { + ElementDescriptor { + element_id: raw.element_id, + tag: raw.tag.to_lowercase(), + input_type: raw.input_type.as_deref().map(str::to_lowercase), + id: clean(raw.id.as_ref(), ctx), + name: clean(raw.name.as_ref(), ctx), + autocomplete: clean(raw.autocomplete.as_ref(), ctx), + placeholder: clean(raw.placeholder.as_ref(), ctx), + aria_label: clean(raw.aria_label.as_ref(), ctx), + was_password, + form: raw.form.as_ref().map(|f| form_ref(f, ctx, tally)), + selectors: raw + .selectors + .iter() + .filter(|s| selector_is_clean(s, ctx)) + .cloned() + .collect(), + in_shadow_root: raw.in_shadow_root, + } +} + +/// The media type alone, lowercased: parameters (a multipart boundary) are not kept. One that +/// holds a typed value is `{typed}`. +fn media_type(content_type: Option<&str>, ctx: Ctx<'_>) -> Option { + let essence = content_type? + .split(';') + .next() + .unwrap_or_default() + .trim() + .to_ascii_lowercase(); + Some(if ctx.holds_typed(&essence) { + TYPED.to_owned() + } else { + essence + }) +} + +pub(super) fn response_shape(r: &super::BufferedResponse, ctx: Ctx<'_>) -> ResponseShape { + ResponseShape { + status: r.status, + content_type: media_type(r.content_type.as_deref(), ctx), + set_cookie: r.set_cookie, + } +} + +/// A hit that makes a request worth keeping: a credential, or another typed value that is not a +/// weak (very short) match. +fn is_telling(hit: &kharon_recording::MarkerHit) -> bool { + match hit.role { + Role::Username | Role::Password => true, + Role::None => !hit.low_confidence, + } +} + +/// The reduced request, with the counts of what stripping its URL removed. `None` when the URL +/// does not parse. Whether to keep it is the caller's call. +pub(super) fn reduce_request( + b: &Buffered, + ctx: Ctx<'_>, +) -> Option<(StoredRequest, Tally, bool, bool)> { + let (url, tally) = strip(&b.url, ctx)?; + let reduced_body = matches!(b.body, RawBody::Bytes(_)); + let body = match &b.body { + RawBody::None => BodyShape::None, + RawBody::Unavailable => BodyShape::Unavailable, + RawBody::Bytes(bytes) => { + shape_of_body(b.content_type.as_deref(), bytes, ctx.markers, ctx.pair) + } + }; + let stored = StoredRequest { + t_ms: b.t_ms, + method: known_method(&b.method, false), + url, + resource: b.resource, + main_frame_navigation: b.main_frame_navigation, + content_type: media_type(b.content_type.as_deref(), ctx), + authorization: b + .authorization + .as_ref() + .map(|a| reduce_authorization(a, ctx.markers, ctx.pair)), + url_markers: hits_in(b.url.as_bytes(), ctx.markers, ctx.pair), + body, + response: b.response.as_ref().map(|r| response_shape(r, ctx)), + }; + let telling = stored.hits().into_iter().any(is_telling); + Some((stored, tally, reduced_body, telling)) +} diff --git a/crates/kharon-teach/src/capture/mod.rs b/crates/kharon-teach/src/capture/mod.rs new file mode 100644 index 0000000..08d099c --- /dev/null +++ b/crates/kharon-teach/src/capture/mod.rs @@ -0,0 +1,1430 @@ +//! The capture state machine: events from a recorded login go in, and `finish` returns a +//! `Recording` that says how the login works and holds nothing the person typed. +//! +//! Pure: no browser, no I/O, no logging. Requests are buffered with their bodies in `Zeroizing` +//! buffers and analysed in `finish`, after the typed values are known. + +mod build; +mod resolve; +mod types; + +use kharon_recording::{ + BrowserInfo, Flags, ListenerWorld, Recording, Role, Step, StoredRequest, StrippedCounts, + Viewport, classify, is_otp_descriptor, +}; +pub use types::{CaptureEvent, EndInfo, PageEventKind, RawBody, RawElement, RawForm, RawFrame}; +use url::Url; +use zeroize::Zeroizing; + +use crate::body::MAX_BODY_BYTES; +use crate::site::{is_captcha_host, registrable_domain}; +use build::{Ctx, Tally, describe, form_ref, frame_ref, reduce_request, stripped_url}; +use resolve::{Elem, resolve}; + +/// A frame as it was when an event arrived. +struct FrameInfo { + main: bool, + url: String, + cross_origin: bool, +} + +enum What { + Navigation(String), + Input(usize), + Click(RawElement), + Submit(RawForm), + EnterKey(RawElement), +} + +struct Pending { + t_ms: u64, + frame: FrameInfo, + what: What, +} + +struct BufferedResponse { + status: u16, + content_type: Option, + set_cookie: bool, +} + +/// A request waiting for `finish`, with its bytes in zeroizing buffers. +struct Buffered { + request_id: String, + t_ms: u64, + method: String, + url: String, + resource: kharon_recording::ResourceKind, + main_frame_navigation: bool, + content_type: Option, + authorization: Option>>, + body: RawBody, + /// A non-GET request to the login site made after the first `Input`. + login_site_write: bool, + response: Option, +} + +pub struct Capture { + start_url: String, + tool_version: String, + login_site: String, + started: bool, + browser: Option, + listener_world: ListenerWorld, + main_origin: Option, + final_url: Option, + steps: Vec, + elems: Vec, + reqs: Vec, + seq: u64, + inputs_seen: bool, + requests_seen: u64, + popup: bool, + captcha: bool, + cross_site: bool, + framed_login: bool, + body_unavailable: bool, +} + +fn host_of(url: &str) -> Option { + Url::parse(url) + .ok() + .and_then(|u| u.host_str().map(str::to_owned)) +} + +fn raw_origin(url: &str) -> Option { + Url::parse(url) + .ok() + .map(|u| u.origin().ascii_serialization()) +} + +/// A CAPTCHA host, or `www.google.com/recaptcha/…`. +fn is_captcha_url(url: &str) -> bool { + let Ok(parsed) = Url::parse(url) else { + return false; + }; + let Some(host) = parsed.host_str() else { + return false; + }; + is_captcha_host(host) + || (matches!(host, "www.google.com" | "google.com") + && parsed.path().starts_with("/recaptcha/")) +} + +fn same_element(e: &Elem, raw: &RawElement) -> bool { + e.raw.document_id == raw.document_id && e.raw.element_id == raw.element_id +} + +impl Capture { + pub fn new(start_url: &str, tool_version: &str) -> Self { + let login_site = host_of(start_url).map(|h| registrable_domain(&h)); + Self { + start_url: start_url.to_owned(), + tool_version: tool_version.to_owned(), + login_site: login_site.unwrap_or_default(), + started: false, + browser: None, + listener_world: ListenerWorld::Isolated, + main_origin: raw_origin(start_url), + final_url: None, + steps: Vec::new(), + elems: Vec::new(), + reqs: Vec::new(), + seq: 0, + inputs_seen: false, + requests_seen: 0, + popup: false, + captcha: false, + cross_site: false, + framed_login: false, + body_unavailable: false, + } + } + + pub fn started(&self) -> bool { + self.started + } + + /// Takes one event. Everything but `Popup` is ignored until `Started` arrives. + pub fn push(&mut self, ev: CaptureEvent) { + match ev { + CaptureEvent::Popup => self.popup = true, + CaptureEvent::Started { + browser, + listener_world, + } => { + if !self.started { + self.started = true; + self.browser = Some(browser); + self.listener_world = listener_world; + } + } + _ if !self.started => {} + CaptureEvent::Navigation { t_ms, frame } => self.navigation(t_ms, &frame), + CaptureEvent::Page { + t_ms, + kind, + frame, + element, + value, + } => self.page(t_ms, kind, &frame, element, value), + CaptureEvent::Request { + request_id, + t_ms, + method, + url, + resource, + main_frame_navigation, + content_type, + authorization, + body, + frame_url: _, + } => self.request(Buffered { + request_id, + t_ms, + method, + url, + resource, + main_frame_navigation, + content_type, + authorization, + body, + login_site_write: false, + response: None, + }), + CaptureEvent::Response { + request_id, + status, + content_type, + set_cookie, + } => { + if let Some(r) = self + .reqs + .iter_mut() + .rev() + .find(|r| r.request_id == request_id) + { + r.response = Some(BufferedResponse { + status, + content_type, + set_cookie, + }); + } + } + } + } + + fn frame_info(&self, frame: &RawFrame) -> FrameInfo { + let cross_origin = !frame.main && raw_origin(&frame.url) != self.main_origin; + FrameInfo { + main: frame.main, + url: frame.url.clone(), + cross_origin, + } + } + + fn navigation(&mut self, t_ms: u64, frame: &RawFrame) { + if is_captcha_url(&frame.url) { + self.captcha = true; + } + if frame.main { + self.main_origin = raw_origin(&frame.url); + self.final_url = Some(frame.url.clone()); + if host_of(&frame.url).is_some_and(|h| registrable_domain(&h) != self.login_site) { + self.cross_site = true; + } + } + let info = self.frame_info(frame); + self.steps.push(Pending { + t_ms, + frame: info, + what: What::Navigation(frame.url.clone()), + }); + } + + fn page( + &mut self, + t_ms: u64, + kind: PageEventKind, + frame: &RawFrame, + element: RawElement, + value: Option>, + ) { + let info = self.frame_info(frame); + let known = self.elems.iter().position(|e| same_element(e, &element)); + if let Some(i) = known { + self.note_report(i, &element, &info); + } + let what = match kind { + PageEventKind::Input | PageEventKind::Change => { + let i = match known { + Some(i) => i, + None => { + self.elems.push(Elem { + raw: element.clone(), + value: Zeroizing::new(String::new()), + was_password: false, + shadow: false, + has_step: false, + first_seq: self.seq + 1, + last_seq: 0, + }); + let i = self.elems.len() - 1; + self.note_report(i, &element, &info); + i + } + }; + self.seq += 1; + self.elems[i].last_seq = self.seq; + if let Some(v) = value { + self.elems[i].value = v; + } + if kind == PageEventKind::Change || self.elems[i].has_step { + if kind == PageEventKind::Input { + self.inputs_seen = true; + } + return; + } + self.elems[i].has_step = true; + self.inputs_seen = true; + What::Input(i) + } + PageEventKind::Click => What::Click(element), + PageEventKind::EnterKey => What::EnterKey(element), + PageEventKind::Submit => match element.form { + Some(form) => What::Submit(form), + None => return, + }, + }; + self.steps.push(Pending { + t_ms, + frame: info, + what, + }); + } + + fn note_report(&mut self, i: usize, element: &RawElement, info: &FrameInfo) { + let e = &mut self.elems[i]; + e.was_password |= element.is_password_now; + e.shadow |= element.in_shadow_root; + if e.was_password && !info.main && info.cross_origin { + self.framed_login = true; + } + } + + fn request(&mut self, mut b: Buffered) { + self.requests_seen += 1; + if is_captcha_url(&b.url) { + self.captcha = true; + } + if matches!(&b.body, RawBody::Bytes(bytes) if bytes.len() > MAX_BODY_BYTES) { + b.body = RawBody::Unavailable; + } + let to_login_site = + host_of(&b.url).is_some_and(|h| registrable_domain(&h) == self.login_site); + let write = !b.method.eq_ignore_ascii_case("GET") && to_login_site; + if write && matches!(b.body, RawBody::Unavailable) { + self.body_unavailable = true; + } + b.login_site_write = write && self.inputs_seen; + self.reqs.push(b); + } + + /// Resolves the typed values, searches the buffered requests, strips, sets the flags and + /// classifies. Consumes the capture, so every buffered value is dropped (and zeroed) here. + pub fn finish(self, end: EndInfo) -> Recording { + let resolved = resolve(&self.elems, &self.reqs); + let ctx = Ctx { + markers: &resolved.markers, + pair: resolved.pair(), + }; + let mut tally = Tally::default(); + + let start_url = stripped_url(&self.start_url, ctx, &mut tally); + let final_url = self + .final_url + .as_deref() + .map(|u| stripped_url(u, ctx, &mut tally)); + let steps = self.build_steps(&resolved, ctx, &mut tally); + + let mut requests: Vec = Vec::new(); + let mut bodies_reduced = 0; + for b in &self.reqs { + let Some((stored, t, reduced, telling)) = reduce_request(b, ctx) else { + continue; + }; + if b.main_frame_navigation || telling || b.login_site_write { + tally.add(t); + bodies_reduced += u64::from(reduced); + requests.push(stored); + } + } + + let flags = + Flags { + captcha: self.captcha, + otp_step: self.otp_step(), + cross_site: self.cross_site, + popup: self.popup, + framed_login: self.framed_login, + shadow_dom: self.elems.iter().enumerate().any(|(i, e)| { + e.shadow && (e.was_password || Some(i) == resolved.username_elem) + }), + multiple_passwords: resolved.multiple_passwords, + body_unavailable: self.body_unavailable, + }; + let classification = classify(&steps, &requests, &flags); + Recording { + format_version: kharon_recording::FORMAT_VERSION, + tool_version: self.tool_version.clone(), + browser: self.browser.clone().unwrap_or(BrowserInfo { + product: String::new(), + user_agent: String::new(), + viewport: Viewport { + width: 0, + height: 0, + }, + }), + listener_world: self.listener_world, + started_at: end.started_at, + ended_at: end.ended_at, + ended_by: end.ended_by, + success_confirmed: end.success_confirmed, + start_url, + final_url, + steps, + flags, + classification, + stripped: StrippedCounts { + requests_seen: self.requests_seen, + requests_stored: requests.len() as u64, + request_bodies_reduced: bodies_reduced, + query_values_removed: tally.query_values_removed, + path_segments_replaced: tally.path_segments_replaced, + typed_values_removed: resolved.markers.len() as u64, + }, + requests, + } + } + + /// An `Input` step after the first password input whose element looks like a one-time code + /// field. Judged on the element as the page described it, before any attribute is dropped. + fn otp_step(&self) -> bool { + let none = Ctx { + markers: &[], + pair: None, + }; + let mut seen_password = false; + for p in &self.steps { + let What::Input(i) = p.what else { continue }; + let e = &self.elems[i]; + if seen_password + && is_otp_descriptor(&describe( + &e.raw, + e.was_password, + none, + &mut Tally::default(), + )) + { + return true; + } + seen_password |= e.was_password; + } + false + } + + fn build_steps( + &self, + resolved: &resolve::Resolved, + ctx: Ctx<'_>, + tally: &mut Tally, + ) -> Vec { + self.steps + .iter() + .map(|p| { + let frame = frame_ref(&p.frame, ctx); + let t_ms = p.t_ms; + match &p.what { + What::Navigation(url) => Step::Navigation { + t_ms, + frame, + url: stripped_url(url, ctx, tally), + }, + What::Input(i) => { + let e = &self.elems[*i]; + let role = if e.was_password { + Role::Password + } else if Some(*i) == resolved.username_elem { + Role::Username + } else { + Role::None + }; + Step::Input { + t_ms, + frame, + element: describe(&e.raw, e.was_password, ctx, tally), + value_len: u32::try_from(e.value.chars().count()).unwrap_or(u32::MAX), + role, + } + } + What::Click(raw) => Step::Click { + t_ms, + frame, + element: self.describe_known(raw, ctx, tally), + }, + What::EnterKey(raw) => Step::EnterKey { + t_ms, + frame, + element: self.describe_known(raw, ctx, tally), + }, + What::Submit(form) => Step::Submit { + t_ms, + frame, + form: form_ref(form, ctx, tally), + }, + } + }) + .collect() + } + + fn describe_known( + &self, + raw: &RawElement, + ctx: Ctx<'_>, + tally: &mut Tally, + ) -> kharon_recording::ElementDescriptor { + let was_password = raw.is_password_now + || self + .elems + .iter() + .any(|e| same_element(e, raw) && e.was_password); + describe(raw, was_password, ctx, tally) + } +} + +#[cfg(test)] +mod tests { + use base64ct::{Base64, Encoding}; + use kharon_recording::{ + BodyShape, BrowserInfo, EndedBy, ListenerWorld, MarkerHit, PasswordLeg, Recording, + ResourceKind, Role, Shape, Step, StoredRequest, Viewport, + }; + use zeroize::Zeroizing; + + use super::*; + use crate::body::MAX_BODY_BYTES; + use crate::testing::assert_holds_no_typed_value; + + const LOGIN: &str = "https://ex.test/login"; + const USER: &str = "kh.throwaway7"; + const PASS: &str = "Tr0ub4dor-9x"; + const PASS2: &str = "Qx7!other-pass"; + const OTP: &str = "482913"; + const TENANT: &str = "acme-corp-77"; + /// Every value typed in any scenario below. + const ALL_VALUES: [&str; 5] = [USER, PASS, PASS2, OTP, TENANT]; + const FORM: &str = "application/x-www-form-urlencoded"; + + fn el(id: u32, name: &str, ty: &str) -> RawElement { + RawElement { + element_id: id, + document_id: "doc1".into(), + tag: "INPUT".into(), + input_type: Some(ty.to_uppercase()), + id: Some(name.into()), + name: Some(name.into()), + autocomplete: None, + placeholder: None, + aria_label: None, + is_password_now: ty == "password", + form: Some(RawForm { + index: 0, + action: Some("https://ex.test/login".into()), + method: "post".into(), + }), + selectors: vec![format!("#{name}")], + in_shadow_root: false, + } + } + + fn user_el() -> RawElement { + el(1, "username", "text") + } + + fn pw_el() -> RawElement { + el(2, "password", "password") + } + + fn no_form(mut e: RawElement) -> RawElement { + e.form = None; + e + } + + fn button() -> RawElement { + let mut b = el(9, "go", "submit"); + b.tag = "button".into(); + b + } + + struct Session { + cap: Capture, + t: u64, + rid: u32, + url: String, + } + + impl Session { + fn unstarted(url: &str) -> Self { + Session { + cap: Capture::new(url, "0.0.0"), + t: 0, + rid: 0, + url: url.into(), + } + } + + fn new(url: &str) -> Self { + let mut s = Self::unstarted(url); + s.start(); + s + } + + fn start(&mut self) { + self.cap.push(CaptureEvent::Started { + browser: BrowserInfo { + product: "Chrome/1".into(), + user_agent: "UA".into(), + viewport: Viewport { + width: 1280, + height: 800, + }, + }, + listener_world: ListenerWorld::Isolated, + }); + let url = self.url.clone(); + self.nav_to(&url); + } + + fn tick(&mut self) -> u64 { + self.t += 10; + self.t + } + + fn frame(&self) -> RawFrame { + RawFrame { + main: true, + url: self.url.clone(), + } + } + + fn nav_to(&mut self, url: &str) { + self.url = url.into(); + let (t_ms, frame) = (self.tick(), self.frame()); + self.cap.push(CaptureEvent::Navigation { t_ms, frame }); + } + + fn nav(mut self, url: &str) -> Self { + self.nav_to(url); + self + } + + fn child_nav(mut self, url: &str) -> Self { + let t_ms = self.tick(); + self.cap.push(CaptureEvent::Navigation { + t_ms, + frame: RawFrame { + main: false, + url: url.into(), + }, + }); + self + } + + fn page(&mut self, kind: PageEventKind, e: &RawElement, value: Option<&str>) { + let (t_ms, frame) = (self.tick(), self.frame()); + self.cap.push(CaptureEvent::Page { + t_ms, + kind, + frame, + element: e.clone(), + value: value.map(|v| Zeroizing::new(v.to_owned())), + }); + } + + fn type_into(mut self, e: &RawElement, text: &str) -> Self { + self.page(PageEventKind::Input, e, Some(text)); + self + } + + /// One Input event per character, each carrying the value so far. + fn typing(mut self, e: &RawElement, text: &str) -> Self { + let ends: Vec = text.char_indices().map(|(i, c)| i + c.len_utf8()).collect(); + for end in ends { + self.page(PageEventKind::Input, e, Some(&text[..end])); + } + self + } + + fn change(mut self, e: &RawElement, text: &str) -> Self { + self.page(PageEventKind::Change, e, Some(text)); + self + } + + fn click(mut self, e: &RawElement) -> Self { + self.page(PageEventKind::Click, e, None); + self + } + + fn submit(mut self, e: &RawElement) -> Self { + self.page(PageEventKind::Submit, e, None); + self + } + + fn enter(mut self, e: &RawElement) -> Self { + self.page(PageEventKind::EnterKey, e, None); + self + } + + fn popup(mut self) -> Self { + self.cap.push(CaptureEvent::Popup); + self + } + + fn request( + mut self, + method: &str, + url: &str, + resource: ResourceKind, + main: bool, + content_type: Option<&str>, + body: RawBody, + ) -> Self { + self.rid += 1; + let (t_ms, request_id) = (self.tick(), format!("r{}", self.rid)); + self.cap.push(CaptureEvent::Request { + request_id: request_id.clone(), + t_ms, + method: method.into(), + url: url.into(), + resource, + main_frame_navigation: main, + content_type: content_type.map(str::to_owned), + authorization: None, + body, + frame_url: self.url.clone(), + }); + self.cap.push(CaptureEvent::Response { + request_id, + status: 200, + content_type: Some("text/html; charset=utf-8".into()), + set_cookie: false, + }); + self + } + + fn post_form(self, url: &str, body: &str) -> Self { + self.request( + "POST", + url, + ResourceKind::Document, + true, + Some(FORM), + RawBody::Bytes(Zeroizing::new(body.as_bytes().to_vec())), + ) + } + + fn post_json(self, url: &str, body: &str) -> Self { + self.request( + "POST", + url, + ResourceKind::Fetch, + false, + Some("application/json"), + RawBody::Bytes(Zeroizing::new(body.as_bytes().to_vec())), + ) + } + + fn get(self, url: &str, resource: ResourceKind) -> Self { + self.request("GET", url, resource, false, None, RawBody::None) + } + + fn finish(self) -> Recording { + self.cap.finish(EndInfo { + ended_by: EndedBy::Confirmed, + success_confirmed: true, + started_at: "2026-11-18T00:00:00Z".into(), + ended_at: "2026-11-18T00:01:00Z".into(), + }) + } + } + + fn inputs(rec: &Recording) -> Vec<(&kharon_recording::ElementDescriptor, u32, Role)> { + rec.steps + .iter() + .filter_map(|s| match s { + Step::Input { + element, + value_len, + role, + .. + } => Some((element, *value_len, *role)), + _ => None, + }) + .collect() + } + + fn role_of(rec: &Recording, element_id: u32) -> Option { + inputs(rec) + .into_iter() + .find(|(e, _, _)| e.element_id == element_id) + .map(|(_, _, r)| r) + } + + // ---- scenarios, each a complete recording; the leak test walks the whole list ---- + + fn sc_form_post() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .click(&button()) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}&csrf=abc"), + ) + .get("https://ex.test/logo.png", ResourceKind::Other) + .nav("https://ex.test/home") + .finish() + } + + fn sc_keystrokes() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .typing(&pw_el(), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish() + } + + fn sc_show_toggle() -> Recording { + let mut shown = pw_el(); + shown.is_password_now = false; + shown.input_type = Some("text".into()); + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), "Tr0ub4") + .type_into(&shown, PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish() + } + + fn sc_confirm_field() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .type_into(&el(3, "confirm", "password"), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}&confirm={PASS}"), + ) + .finish() + } + + fn sc_two_passwords() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .type_into(&el(3, "confirm", "password"), PASS2) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}&confirm={PASS2}"), + ) + .finish() + } + + fn sc_two_step() -> Recording { + let mut pw = pw_el(); + pw.document_id = "doc2".into(); + Session::new(LOGIN) + .type_into(&user_el(), USER) + .post_form("https://ex.test/identify", &format!("username={USER}")) + .nav("https://ex.test/password") + .type_into(&pw, PASS) + .post_form("https://ex.test/login", &format!("password={PASS}")) + .finish() + } + + fn sc_username_fallback() -> Recording { + let tenant = no_form(el(4, "tenant", "text")); + Session::new(LOGIN) + .type_into(&tenant, TENANT) + .type_into(&no_form(user_el()), USER) + .type_into(&no_form(pw_el()), PASS) + .enter(&no_form(pw_el())) + .post_json( + "https://ex.test/api/login", + &format!(r#"{{"tenant":"{TENANT}","user":"{USER}","pass":"{PASS}"}}"#), + ) + .finish() + } + + fn sc_cross_site() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .nav("https://idp.other.test/auth") + .type_into(&pw_el(), PASS) + .post_form("https://idp.other.test/auth", &format!("u={USER}&p={PASS}")) + .finish() + } + + fn sc_subdomain() -> Recording { + Session::new(LOGIN) + .nav("https://login.ex.test/sso") + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .post_form("https://login.ex.test/sso", &format!("u={USER}&p={PASS}")) + .finish() + } + + fn sc_captcha() -> Recording { + Session::new(LOGIN) + .child_nav("https://newassets.hcaptcha.com/captcha/v1/frame.html") + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish() + } + + fn sc_recaptcha_request() -> Recording { + Session::new(LOGIN) + .get( + "https://www.google.com/recaptcha/api2/anchor?k=site", + ResourceKind::Other, + ) + .finish() + } + + fn sc_google_search() -> Recording { + Session::new(LOGIN) + .get("https://www.google.com/search?q=x", ResourceKind::Other) + .child_nav("https://www.google.com/maps") + .finish() + } + + fn sc_otp_after() -> Recording { + let mut otp = el(5, "code", "text"); + otp.autocomplete = Some("one-time-code".into()); + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .nav("https://ex.test/verify") + .type_into(&otp, OTP) + .post_form("https://ex.test/verify", &format!("code={OTP}")) + .finish() + } + + fn sc_otp_before() -> Recording { + let mut otp = el(5, "code", "text"); + otp.autocomplete = Some("one-time-code".into()); + Session::new(LOGIN) + .type_into(&otp, OTP) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}&code={OTP}"), + ) + .finish() + } + + fn sc_oversized() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .request( + "POST", + "https://ex.test/login", + ResourceKind::Fetch, + false, + Some("application/octet-stream"), + RawBody::Bytes(Zeroizing::new(vec![b'a'; MAX_BODY_BYTES + 1])), + ) + .finish() + } + + fn sc_combined_base64() -> Recording { + let combined = Base64::encode_string(format!("{USER}:{PASS}").as_bytes()); + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .post_json( + "https://ex.test/api/login", + &format!(r#"{{"auth":"{combined}","remember":true}}"#), + ) + .finish() + } + + fn sc_which_requests() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .get("https://ex.test/logo.png", ResourceKind::Other) + .post_json("https://ads.other.test/collect", r#"{"event":"view"}"#) + .post_json("https://ex.test/api/beacon", r#"{"event":"typed"}"#) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish() + } + + fn sc_values_everywhere() -> Recording { + let tenant = no_form(el(4, "tenant", "text")); + let basic = Base64::encode_string(format!("{USER}:{PASS}").as_bytes()); + let mut s = Session::new(&format!("https://ex.test/login?tenant={TENANT}")) + .type_into(&tenant, TENANT) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .get( + &format!("https://{TENANT}.ex.test/u/{USER}/profile?name={USER}&{PASS}=1"), + ResourceKind::Xhr, + ); + s.rid += 1; + s.cap.push(CaptureEvent::Request { + request_id: "auth".into(), + t_ms: 500, + method: "POST".into(), + url: format!("https://ex.test/api/session/{PASS}"), + resource: ResourceKind::Fetch, + main_frame_navigation: false, + content_type: Some(format!("multipart/form-data; boundary={TENANT}")), + authorization: Some(Zeroizing::new(format!("Basic {basic}").into_bytes())), + body: RawBody::Bytes(Zeroizing::new( + format!("--{TENANT}\r\nContent-Disposition: form-data; name=\"{USER}\"\r\n\r\n{PASS}\r\n--{TENANT}--\r\n") + .into_bytes(), + )), + frame_url: LOGIN.into(), + }); + s.nav(&format!("https://ex.test/home?welcome={USER}")) + .finish() + } + + type Build = fn() -> Recording; + + const SCENARIOS: &[(&str, Build)] = &[ + ("form_post", sc_form_post), + ("keystrokes", sc_keystrokes), + ("show_toggle", sc_show_toggle), + ("confirm_field", sc_confirm_field), + ("two_passwords", sc_two_passwords), + ("two_step", sc_two_step), + ("username_fallback", sc_username_fallback), + ("cross_site", sc_cross_site), + ("subdomain", sc_subdomain), + ("captcha", sc_captcha), + ("recaptcha_request", sc_recaptcha_request), + ("google_search", sc_google_search), + ("otp_after", sc_otp_after), + ("otp_before", sc_otp_before), + ("oversized", sc_oversized), + ("combined_base64", sc_combined_base64), + ("which_requests", sc_which_requests), + ("values_everywhere", sc_values_everywhere), + ]; + + fn scenario(name: &str) -> Recording { + SCENARIOS + .iter() + .find(|(n, _)| *n == name) + .map(|(_, build)| build()) + .unwrap_or_else(|| panic!("no scenario named {name}")) + } + + #[test] + fn the_serialized_recording_holds_no_typed_value() { + for (name, build) in SCENARIOS { + let rec = build(); + let json = rec.to_json(); + std::panic::catch_unwind(|| assert_holds_no_typed_value(&json, &ALL_VALUES)) + .unwrap_or_else(|_| panic!("scenario {name} leaked a typed value")); + } + } + + #[test] + fn form_post_end_to_end() { + let rec = scenario("form_post"); + assert_eq!(rec.classification.shape, Shape::FormPost); + let ins = inputs(&rec); + assert_eq!(ins.len(), 2); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 2), Some(Role::Password)); + assert_eq!(rec.stripped.typed_values_removed, 2); + assert_eq!( + rec.final_url.as_ref().map(|u| u.path.as_str()), + Some("/home") + ); + // Types are lowercased, the descriptor keeps the shape of the element. + assert_eq!(ins[0].0.tag, "input"); + assert_eq!(ins[1].0.input_type.as_deref(), Some("password")); + assert!(ins[1].0.was_password); + assert_eq!(rec.browser.product, "Chrome/1"); + } + + #[test] + fn keystrokes_collapse_to_one_input_step() { + let rec = scenario("keystrokes"); + let pw: Vec<_> = inputs(&rec) + .into_iter() + .filter(|(e, _, _)| e.element_id == 2) + .collect(); + assert_eq!(pw.len(), 1); + assert_eq!(pw[0].1, 12); + } + + #[test] + fn show_password_toggle_keeps_the_password() { + let rec = scenario("show_toggle"); + assert_eq!(role_of(&rec, 2), Some(Role::Password)); + assert_eq!(rec.classification.shape, Shape::FormPost); + assert!(!rec.flags.multiple_passwords); + } + + #[test] + fn confirm_field_with_the_same_value_is_one_password() { + let rec = scenario("confirm_field"); + assert!(!rec.flags.multiple_passwords); + assert_eq!(role_of(&rec, 3), Some(Role::Password)); + assert_eq!(rec.classification.shape, Shape::FormPost); + } + + #[test] + fn two_different_passwords_set_the_flag_and_block() { + let rec = scenario("two_passwords"); + assert!(rec.flags.multiple_passwords); + assert_eq!(rec.classification.shape, Shape::Unrecognized); + } + + #[test] + fn two_step_by_navigation() { + let rec = scenario("two_step"); + assert_eq!( + rec.classification.shape, + Shape::TwoStep { + password_leg: PasswordLeg::FormPost + } + ); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert!(!rec.flags.cross_site); + } + + #[test] + fn username_falls_back_to_the_value_sent_with_the_password() { + let rec = scenario("username_fallback"); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 4), Some(Role::None)); + assert_eq!(rec.classification.shape, Shape::ScriptPost); + // The tenant's hit stays in the file, with no role. + let sent_tenant = rec + .requests + .iter() + .flat_map(StoredRequest::hits) + .any(|h: &MarkerHit| h.role == Role::None); + assert!(sent_tenant); + } + + #[test] + fn cross_site_navigation_sets_the_flag() { + assert!(scenario("cross_site").flags.cross_site); + } + + #[test] + fn same_site_subdomain_is_not_cross_site() { + assert!(!scenario("subdomain").flags.cross_site); + } + + #[test] + fn captcha_frame_sets_the_flag() { + assert!(scenario("captcha").flags.captcha); + assert!(scenario("recaptcha_request").flags.captcha); + assert!(!scenario("google_search").flags.captcha); + assert!(!scenario("form_post").flags.captcha); + } + + #[test] + fn otp_input_after_the_password_sets_the_flag() { + assert!(scenario("otp_after").flags.otp_step); + assert!(!scenario("otp_before").flags.otp_step); + assert!(!scenario("form_post").flags.otp_step); + } + + #[test] + fn oversized_body_is_unavailable_and_flagged() { + let rec = scenario("oversized"); + assert!(rec.flags.body_unavailable); + assert_eq!(rec.requests.len(), 1); + assert_eq!(rec.requests[0].body, BodyShape::Unavailable); + assert_eq!(rec.classification.shape, Shape::Unrecognized); + } + + #[test] + fn a_combined_base64_credential_in_a_json_body_is_a_plain_hit() { + let rec = scenario("combined_base64"); + assert_eq!(rec.classification.shape, Shape::ScriptPost); + } + + #[test] + fn only_the_three_kinds_of_request_are_stored() { + let rec = scenario("which_requests"); + // The image GET and the third-party POST are counted, not stored. + assert_eq!(rec.stripped.requests_seen, 4); + let urls: Vec<&str> = rec.requests.iter().map(|r| r.url.path.as_str()).collect(); + assert!( + urls.contains(&"/login"), + "the navigation, kept as a document: {urls:?}" + ); + assert!( + urls.contains(&"/api/beacon"), + "a POST to the login site after the first input" + ); + assert!(!urls.contains(&"/logo.png")); + assert!(!urls.contains(&"/collect")); + assert_eq!(rec.stripped.requests_stored, rec.requests.len() as u64); + assert_eq!(rec.stripped.request_bodies_reduced, 2); + } + + #[test] + fn requests_with_a_hit_are_stored_even_when_third_party() { + let rec = Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .post_json( + "https://sso.other.test/token", + &format!(r#"{{"p":"{PASS}"}}"#), + ) + .finish(); + assert_eq!(rec.requests.len(), 1); + assert_eq!(rec.requests[0].url.origin, "https://sso.other.test"); + } + + #[test] + fn events_before_started_are_ignored_for_steps() { + let mut s = Session::unstarted(LOGIN); + s = s + .type_into(&user_el(), USER) + .click(&button()) + .post_form("https://ex.test/login", "a=b") + .popup(); + assert!(!s.cap.started()); + s.start(); + assert!(s.cap.started()); + let rec = s + .type_into(&pw_el(), PASS) + .post_form("https://ex.test/login", &format!("password={PASS}")) + .finish(); + assert_eq!(rec.stripped.requests_seen, 1); + assert_eq!(inputs(&rec).len(), 1); + assert!(!rec.steps.iter().any(|s| matches!(s, Step::Click { .. }))); + // A pop-up is remembered whenever it arrives, and the earlier typing never counted. + assert!(rec.flags.popup); + assert_eq!(rec.stripped.typed_values_removed, 1); + } + + #[test] + fn a_second_started_is_ignored_and_unknown_responses_too() { + let mut s = Session::new(LOGIN); + s.start(); + s.cap.push(CaptureEvent::Response { + request_id: "nope".into(), + status: 500, + content_type: None, + set_cookie: true, + }); + let rec = s.finish(); + assert_eq!(rec.browser.product, "Chrome/1"); + assert!(rec.requests.is_empty()); + } + + #[test] + fn finish_on_a_capture_that_never_started_still_returns_a_recording() { + let rec = Session::unstarted(LOGIN).finish(); + assert!(rec.steps.is_empty() && rec.requests.is_empty()); + assert_eq!(rec.browser.product, ""); + assert_eq!(rec.browser.viewport.width, 0); + assert_eq!(rec.listener_world, ListenerWorld::Isolated); + assert_eq!(rec.stripped.requests_seen, 0); + } + + #[test] + fn change_adds_no_step_and_click_adds_one_every_time() { + let rec = Session::new(LOGIN) + .change(&user_el(), USER) + .click(&button()) + .click(&button()) + .submit(&no_form(button())) + .submit(&pw_el()) + .finish(); + assert!(inputs(&rec).is_empty()); + let clicks = rec + .steps + .iter() + .filter(|s| matches!(s, Step::Click { .. })) + .count(); + let submits = rec + .steps + .iter() + .filter(|s| matches!(s, Step::Submit { .. })) + .count(); + assert_eq!((clicks, submits), (2, 1)); + // The changed value still counts as typed. + assert_eq!(rec.stripped.typed_values_removed, 1); + assert_holds_no_typed_value(&rec.to_json(), &[USER]); + } + + #[test] + fn an_element_attribute_holding_a_typed_value_is_dropped() { + let mut e = user_el(); + e.id = Some(format!("user-{USER}")); + e.placeholder = Some(format!("e.g. {USER}")); + e.selectors = vec![ + format!(r"#user-{}", USER.replace('.', r"\.")), + "input[name=username]".into(), + ]; + let rec = Session::new(LOGIN).type_into(&e, USER).finish(); + let (descriptor, _, _) = inputs(&rec)[0]; + assert_eq!(descriptor.id, None); + assert_eq!(descriptor.placeholder, None); + assert_eq!(descriptor.name.as_deref(), Some("username")); + assert_eq!( + descriptor.selectors, + vec!["input[name=username]".to_owned()] + ); + assert_holds_no_typed_value(&rec.to_json(), &[USER]); + } + + #[test] + fn child_frames_are_cross_origin_and_a_framed_login_is_flagged() { + let mut s = Session::new(LOGIN); + let frame = RawFrame { + main: false, + url: "https://pay.other.test/frame".into(), + }; + let (t_ms, element) = (50, pw_el()); + s.cap.push(CaptureEvent::Page { + t_ms, + kind: PageEventKind::Input, + frame, + element, + value: Some(Zeroizing::new(PASS.to_owned())), + }); + let rec = s.finish(); + assert!(rec.flags.framed_login); + match &rec.steps[1] { + Step::Input { frame, .. } => { + assert!(!frame.main && frame.cross_origin); + assert_eq!(frame.origin, "https://pay.other.test"); + } + other => panic!("expected an input step, got {other:?}"), + } + } + + #[test] + fn shadow_dom_and_popup_flags() { + let mut shadowed = pw_el(); + shadowed.in_shadow_root = true; + let rec = Session::new(LOGIN) + .type_into(&shadowed, PASS) + .popup() + .finish(); + assert!(rec.flags.shadow_dom && rec.flags.popup); + } + + #[test] + fn hostile_event_order_does_not_panic() { + let mut cap = Capture::new("not a url", "0.0.0"); + let response = |id: &str| CaptureEvent::Response { + request_id: id.into(), + status: 200, + content_type: None, + set_cookie: false, + }; + cap.push(response("ghost")); + cap.push(CaptureEvent::Popup); + cap.push(CaptureEvent::Started { + browser: BrowserInfo { + product: String::new(), + user_agent: String::new(), + viewport: Viewport { + width: 0, + height: 0, + }, + }, + listener_world: ListenerWorld::Main, + }); + cap.push(response("ghost")); + cap.push(CaptureEvent::Navigation { + t_ms: 0, + frame: RawFrame { + main: false, + url: "::::".into(), + }, + }); + let mut empty_doc = pw_el(); + empty_doc.document_id = String::new(); + empty_doc.form = Some(RawForm { + index: u32::MAX, + action: Some("%%%".into()), + method: "\u{0}".into(), + }); + for kind in [ + PageEventKind::Input, + PageEventKind::Submit, + PageEventKind::Click, + ] { + cap.push(CaptureEvent::Page { + t_ms: u64::MAX, + kind, + frame: RawFrame { + main: false, + url: String::new(), + }, + element: empty_doc.clone(), + value: Some(Zeroizing::new(String::new())), + }); + } + for url in ["", "http://", "data:text/html,x", "http://[::1", "\u{0}"] { + cap.push(CaptureEvent::Request { + request_id: "r".into(), + t_ms: 0, + method: "".into(), + url: url.into(), + resource: ResourceKind::Other, + main_frame_navigation: true, + content_type: Some("multipart/form-data".into()), + authorization: Some(Zeroizing::new(vec![0xff, 0x00])), + body: RawBody::Bytes(Zeroizing::new(vec![0xff, 0xfe])), + frame_url: "????".into(), + }); + } + cap.push(response("r")); + let rec = cap.finish(EndInfo { + ended_by: EndedBy::WindowClosed, + success_confirmed: false, + started_at: String::new(), + ended_at: String::new(), + }); + assert!(rec.flags.popup); + assert_eq!(rec.stripped.requests_seen, 5); + let _ = rec.to_json(); + } +} diff --git a/crates/kharon-teach/src/capture/resolve.rs b/crates/kharon-teach/src/capture/resolve.rs new file mode 100644 index 0000000..8e28fb7 --- /dev/null +++ b/crates/kharon-teach/src/capture/resolve.rs @@ -0,0 +1,156 @@ +//! Choosing which typed value is the username and which are passwords. + +use kharon_recording::Role; +use zeroize::Zeroizing; + +use super::{Buffered, RawBody, RawElement}; +use crate::marker::Marker; +use crate::strip::any_hit; + +/// Everything known about one element: its first report, and the latest value typed into it. +pub(super) struct Elem { + pub raw: RawElement, + pub value: Zeroizing, + pub was_password: bool, + pub shadow: bool, + pub has_step: bool, + /// Sequence numbers of the first and the latest value-bearing event. + pub first_seq: u64, + pub last_seq: u64, +} + +impl Elem { + fn form_key(&self) -> Option<(&str, u32)> { + self.raw + .form + .as_ref() + .map(|f| (self.raw.document_id.as_str(), f.index)) + } +} + +/// One marker per element that holds a typed value, with the roles chosen. +pub(super) struct Resolved { + pub markers: Vec, + pub username_elem: Option, + pub username_marker: Option, + pub password_marker: Option, + pub multiple_passwords: bool, +} + +impl Resolved { + pub fn pair(&self) -> Option<(&Marker, &Marker)> { + match (self.username_marker, self.password_marker) { + (Some(u), Some(p)) => Some((self.markers.get(u)?, self.markers.get(p)?)), + _ => None, + } + } +} + +fn plain_marker(value: &str, role: Role) -> Marker { + Marker::new(Zeroizing::new(value.to_owned()), role) +} + +/// Whether `marker` occurs in the request's URL, authorization header or body, under any encoding. +fn carries(req: &Buffered, marker: &Marker) -> bool { + let one = std::slice::from_ref(marker); + any_hit(req.url.as_bytes(), one, None) + || req + .authorization + .as_ref() + .is_some_and(|a| any_hit(a, one, None)) + || matches!(&req.body, RawBody::Bytes(b) if any_hit(b, one, None)) +} + +/// The candidate typed in most recently before the first password input; failing that, the latest. +fn pick(cands: &[usize], elems: &[Elem], first_password_seq: Option) -> Option { + cands + .iter() + .copied() + .filter(|&i| first_password_seq.is_none_or(|s| elems[i].first_seq < s)) + .max_by_key(|&i| elems[i].last_seq) + .or_else(|| cands.iter().copied().max_by_key(|&i| elems[i].last_seq)) +} + +pub(super) fn resolve(elems: &[Elem], reqs: &[Buffered]) -> Resolved { + let password_values: Vec<&str> = { + let mut seen: Vec<&str> = Vec::new(); + for e in elems + .iter() + .filter(|e| e.was_password && !e.value.is_empty()) + { + if !seen.contains(&e.value.as_str()) { + seen.push(e.value.as_str()); + } + } + seen + }; + let multiple_passwords = password_values.len() > 1; + let password_markers: Vec = password_values + .iter() + .map(|v| plain_marker(v, Role::Password)) + .collect(); + let first_password_seq = elems + .iter() + .filter(|e| e.was_password) + .map(|e| e.first_seq) + .min(); + let cands: Vec = (0..elems.len()) + .filter(|&i| !elems[i].was_password && !elems[i].value.is_empty()) + .collect(); + let probes: Vec<(usize, Marker)> = cands + .iter() + .map(|&i| (i, plain_marker(&elems[i].value, Role::None))) + .collect(); + let narrowed = |keep: &dyn Fn(usize, &Marker) -> bool| -> Vec { + probes + .iter() + .filter(|(i, m)| keep(*i, m)) + .map(|(i, _)| *i) + .collect() + }; + + let same_form = narrowed(&|i, _| { + elems[i].form_key().is_some_and(|k| { + elems + .iter() + .any(|p| p.was_password && p.form_key() == Some(k)) + }) + }); + let with_password = narrowed(&|_, m| { + reqs.iter() + .any(|r| carries(r, m) && password_markers.iter().any(|p| carries(r, p))) + }); + let in_any_request = narrowed(&|_, m| reqs.iter().any(|r| carries(r, m))); + + let username_elem = [same_form, with_password, in_any_request, cands] + .iter() + .find_map(|level| pick(level, elems, first_password_seq)); + + let mut markers = Vec::new(); + let (mut username_marker, mut password_marker) = (None, None); + for (i, e) in elems.iter().enumerate() { + if e.value.is_empty() { + continue; + } + let role = if e.was_password { + Role::Password + } else if Some(i) == username_elem { + Role::Username + } else { + Role::None + }; + match role { + Role::Username => username_marker = Some(markers.len()), + Role::Password if password_marker.is_none() => password_marker = Some(markers.len()), + Role::Password | Role::None => {} + } + markers.push(plain_marker(&e.value, role)); + } + Resolved { + markers, + username_elem, + username_marker, + password_marker, + multiple_passwords, + } +} diff --git a/crates/kharon-teach/src/capture/types.rs b/crates/kharon-teach/src/capture/types.rs new file mode 100644 index 0000000..6955cc1 --- /dev/null +++ b/crates/kharon-teach/src/capture/types.rs @@ -0,0 +1,98 @@ +//! The events a capture takes and the description it ends with. None of these types derives +//! `Debug`: some hold a typed value. + +use kharon_recording::{BrowserInfo, EndedBy, ListenerWorld, ResourceKind}; +use zeroize::Zeroizing; + +/// An element as the page script described it. No value: that travels beside it. +#[derive(Clone)] +pub struct RawElement { + pub element_id: u32, + pub document_id: String, + pub tag: String, + pub input_type: Option, + pub id: Option, + pub name: Option, + pub autocomplete: Option, + pub placeholder: Option, + pub aria_label: Option, + pub is_password_now: bool, + pub form: Option, + pub selectors: Vec, + pub in_shadow_root: bool, +} + +#[derive(Clone)] +pub struct RawForm { + pub index: u32, + pub action: Option, + pub method: String, +} + +#[derive(Clone)] +pub struct RawFrame { + pub main: bool, + pub url: String, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum PageEventKind { + Input, + Change, + Click, + Submit, + EnterKey, +} + +pub enum RawBody { + None, + Unavailable, + Bytes(Zeroizing>), +} + +pub enum CaptureEvent { + /// The main frame left the notice page. + Started { + browser: BrowserInfo, + listener_world: ListenerWorld, + }, + /// Main and child frames alike. + Navigation { + t_ms: u64, + frame: RawFrame, + }, + /// `value` is set only for `Input` and `Change`. + Page { + t_ms: u64, + kind: PageEventKind, + frame: RawFrame, + element: RawElement, + value: Option>, + }, + Request { + request_id: String, + t_ms: u64, + method: String, + url: String, + resource: ResourceKind, + main_frame_navigation: bool, + content_type: Option, + authorization: Option>>, + body: RawBody, + frame_url: String, + }, + Response { + request_id: String, + status: u16, + content_type: Option, + set_cookie: bool, + }, + Popup, +} + +pub struct EndInfo { + pub ended_by: EndedBy, + pub success_confirmed: bool, + pub started_at: String, + pub ended_at: String, +} diff --git a/crates/kharon-teach/src/lib.rs b/crates/kharon-teach/src/lib.rs index b572f0d..2bc7e71 100644 --- a/crates/kharon-teach/src/lib.rs +++ b/crates/kharon-teach/src/lib.rs @@ -1,7 +1,9 @@ //! Teach-mode recorder for kharon (#361). pub mod body; +pub mod capture; pub mod marker; +pub mod site; pub mod strip; #[doc(hidden)] pub mod testing; diff --git a/crates/kharon-teach/src/site.rs b/crates/kharon-teach/src/site.rs new file mode 100644 index 0000000..de6fbee --- /dev/null +++ b/crates/kharon-teach/src/site.rs @@ -0,0 +1,76 @@ +//! Which site a host belongs to, by the Public Suffix List, and which hosts serve CAPTCHAs. + +use std::net::IpAddr; +use std::sync::OnceLock; + +use publicsuffix::{List, Psl}; + +/// The one vendored snapshot, shared with `kharond` as a data file (not as a crate). +const PSL_SNAPSHOT: &[u8] = include_bytes!("../../kharond/data/public_suffix_list.dat"); + +/// Hosts that serve a CAPTCHA. `google.com/recaptcha` is a path, so `capture` checks that case. +const CAPTCHA_HOSTS: [&str; 5] = [ + "recaptcha.net", + "hcaptcha.com", + "challenges.cloudflare.com", + "arkoselabs.com", + "funcaptcha.com", +]; + +fn psl() -> Option<&'static List> { + static PSL: OnceLock> = OnceLock::new(); + PSL.get_or_init(|| List::from_bytes(PSL_SNAPSHOT).ok()) + .as_ref() +} + +/// The registrable domain of `host` (its eTLD+1), lowercased. An IP address, a single-label host +/// such as `localhost`, or a host the list has no answer for is returned unchanged. +pub fn registrable_domain(host: &str) -> String { + let bare = host + .strip_prefix('[') + .and_then(|h| h.strip_suffix(']')) + .unwrap_or(host); + if bare.parse::().is_ok() { + return host.to_owned(); + } + psl() + .and_then(|list| list.domain(host.as_bytes())) + .map_or_else( + || host.to_owned(), + |d| String::from_utf8_lossy(d.as_bytes()).into_owned(), + ) +} + +/// Whether `host`, or any parent domain of it, is a known CAPTCHA host. +pub fn is_captcha_host(host: &str) -> bool { + let host = host.trim_end_matches('.').to_ascii_lowercase(); + CAPTCHA_HOSTS + .iter() + .any(|h| host == *h || host.ends_with(&format!(".{h}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn registrable_domains() { + assert_eq!(registrable_domain("login.ex.co.uk"), "ex.co.uk"); + assert_eq!(registrable_domain("login.ex.test"), "ex.test"); + assert_eq!(registrable_domain("ex.test"), "ex.test"); + assert_eq!(registrable_domain("127.0.0.1"), "127.0.0.1"); + assert_eq!(registrable_domain("localhost"), "localhost"); + assert_eq!(registrable_domain("[::1]"), "[::1]"); + } + + #[test] + fn captcha_hosts_match_the_host_and_its_subdomains() { + assert!(is_captcha_host("hcaptcha.com")); + assert!(is_captcha_host("newassets.hcaptcha.com")); + assert!(is_captcha_host("challenges.cloudflare.com")); + assert!(is_captcha_host("www.recaptcha.net")); + assert!(!is_captcha_host("cloudflare.com")); + assert!(!is_captcha_host("nothcaptcha.com")); + assert!(!is_captcha_host("www.google.com")); + } +} From ef6b532ab1b5093bbb00df042db76f7f0af93bd6 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:29:34 -0500 Subject: [PATCH 14/36] fix(teach): match typed values across URL parts, and short ones only as whole tokens (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-teach/src/body.rs | 43 ++- crates/kharon-teach/src/strip.rs | 461 ++++++++++++++++++++++++------- 2 files changed, 394 insertions(+), 110 deletions(-) diff --git a/crates/kharon-teach/src/body.rs b/crates/kharon-teach/src/body.rs index 35d6d74..bfc4df1 100644 --- a/crates/kharon-teach/src/body.rs +++ b/crates/kharon-teach/src/body.rs @@ -756,26 +756,40 @@ mod tests { #[test] fn many_crlf_parts_are_read_in_linear_time() { + // The earlier quadratic parse (CRLF bodies never matched the LF-only terminator, so every + // part scanned the rest of the body) took minutes at this size; this one takes milliseconds. let mut body = Vec::new(); - for i in 0..5000 { - body.extend_from_slice( - format!("--b\r\nContent-Disposition: form-data; name=\"f{i}\"\r\n\r\nv\r\n") - .as_bytes(), - ); + for _ in 0..20_000 { + body.extend_from_slice(b"--b\r\nContent-Disposition: form-data; name=f\r\n\r\nv\r\n"); } body.extend_from_slice(b"--b--\r\n"); - let s = shape("multipart/form-data; boundary=b", &body); + assert!( + body.len() <= MAX_BODY_BYTES, + "the body must be parsed, not Opaque" + ); + // No markers: this measures the parse, not the per-value search. + let started = std::time::Instant::now(); + let s = shape_of_body(Some("multipart/form-data; boundary=b"), &body, &[], None); let BodyShape::Multipart { parts } = s else { panic!("not multipart") }; - assert_eq!(parts.len(), 5000); + assert_eq!(parts.len(), 20_000); // LF-only bodies too. let lf = String::from_utf8_lossy(&body).replace("\r\n", "\n"); - let s = shape("multipart/form-data; boundary=b", lf.as_bytes()); + let s = shape_of_body( + Some("multipart/form-data; boundary=b"), + lf.as_bytes(), + &[], + None, + ); let BodyShape::Multipart { parts } = s else { panic!("not multipart") }; - assert_eq!(parts.len(), 5000); + assert_eq!(parts.len(), 20_000); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "multipart parse is not linear" + ); } #[test] @@ -786,12 +800,16 @@ mod tests { "Tr0ub4dor-9x", "Tr0ub4/dor", "Bob.Smith@Ex.Test", + "Zed.Quincy9", + "ab?cd#ef@gh", ]; let m: Vec = vec![ user(values[0]), pw(values[1]), pw(values[2]), user(values[3]), + user(values[4]), + pw(values[5]), ]; let (u, p) = (user(values[0]), pw(values[1])); let pair = Some((&u, &p)); @@ -827,7 +845,12 @@ mod tests { .collect(); let urls = [ "https://kh.throwaway7.ex.test/login", - "https://Bob.Smith@Ex.Test.ex.test/login", + // The HOST holds the value in a different case from the marker. + "https://zed.quincy9.ex.test/login", + "https://x-zed.quincy9-y.ex.test:81/login", + "https://ab?cd#ef@gh/x", + "https://ex.test/a/ab?cd#ef@gh", + "https://ab@gh.ex.test/ab?cd=1#ef", "https://a.kh.throwaway7.ex.kh.throwaway7.test/", "https://ex.test/u/kh.throwaway7/dash?kh.throwaway7=1&password=Tr0ub4dor-9x", "https://ex.test/u/kh%2Ethrow%61way7/x?kh%2Ethrowaway7=1", diff --git a/crates/kharon-teach/src/strip.rs b/crates/kharon-teach/src/strip.rs index 2f35966..1dffec9 100644 --- a/crates/kharon-teach/src/strip.rs +++ b/crates/kharon-teach/src/strip.rs @@ -9,10 +9,10 @@ use std::ops::Range; use base64ct::{Base64, Base64Unpadded, Base64Url, Base64UrlUnpadded, Encoding as B64}; use kharon_recording::{AuthShape, Encoding, MarkerHit, StrippedUrl}; use percent_encoding::percent_decode; -use url::Url; +use url::{Position, Url}; use zeroize::Zeroizing; -use crate::marker::{Marker, search, search_pair}; +use crate::marker::{LOW_CONFIDENCE_BELOW, Marker, search, search_pair}; /// Written in place of a name (key, part name, host labels) that held a typed value. pub const TYPED: &str = "{typed}"; @@ -122,8 +122,25 @@ fn text_same_len(bytes: &[u8]) -> Zeroizing { out } +/// A marker shorter than this many characters matches kept text only as a whole token. +fn is_short(m: &Marker) -> bool { + m.len() < LOW_CONFIDENCE_BELOW +} + +/// Whether `range` in `text` is bounded on each side by the text's edge or by a character that +/// is not ASCII alphanumeric. +fn is_whole_token(text: &str, range: &Range) -> bool { + let not_alnum = |c: char| !c.is_ascii_alphanumeric(); + text[..range.start] + .chars() + .next_back() + .is_none_or(not_alnum) + && text[range.end..].chars().next().is_none_or(not_alnum) +} + /// Byte ranges of every occurrence (overlapping ones too) of every marker in `text`, ignoring -/// ASCII letter case. +/// ASCII letter case. A marker shorter than `LOW_CONFIDENCE_BELOW` characters counts only where +/// it is a whole token (see `is_whole_token`): `on` is not found inside `session`. fn occurrences(text: &str, markers: &[Marker]) -> Vec> { let mut out = Vec::new(); for m in markers { @@ -133,7 +150,9 @@ fn occurrences(text: &str, markers: &[Marker]) -> Vec> { .and_then(|rest| m.find_ignore_ascii_case(rest)) { let (start, end) = (from + r.start, from + r.end); - out.push(start..end); + if !is_short(m) || is_whole_token(text, &(start..end)) { + out.push(start..end); + } // Resume one character after the start so overlapping occurrences are found. from = start + text[start..].chars().next().map_or(1, char::len_utf8); } @@ -149,14 +168,41 @@ fn touched(ranges: &[Range], spans: &[Range]) -> Vec { .collect() } +/// The part of each range inside `start..end`, in coordinates relative to `start`. +fn rebase(ranges: &[Range], start: usize, end: usize) -> Vec> { + ranges + .iter() + .filter_map(|r| { + let (s, e) = (r.start.max(start), r.end.min(end)); + (s < e).then(|| s - start..e - start) + }) + .collect() +} + +/// `search` over kept text, for the rule that applies to KEPT text only: a marker shorter than +/// `LOW_CONFIDENCE_BELOW` characters (and a pair of that combined length) is left out, because +/// its encoded forms would match inside ordinary words. Short markers are matched instead by +/// `occurrences`, as the typed text with token boundaries. Rule chosen: the simple one. +fn kept_hit(haystack: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> bool { + markers + .iter() + .filter(|m| !is_short(m)) + .any(|m| !search(haystack, m).is_empty()) + || pair.is_some_and(|(u, p)| { + u.len().saturating_add(p.len()) >= LOW_CONFIDENCE_BELOW + && !search_pair(haystack, u, p).is_empty() + }) +} + /// Whether kept text `bytes` holds a typed value: `search` finds any hit (any encoding, the pair -/// too), or the value occurs ignoring ASCII case in the text or in its percent-decoded forms. +/// too) for a marker of `LOW_CONFIDENCE_BELOW` characters or more, or the value occurs ignoring +/// ASCII case in the text or in its percent-decoded forms (a shorter marker only as a whole token). pub(crate) fn holds_typed( bytes: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>, ) -> bool { - if any_hit(bytes, markers, pair) || !occurrences(&text_same_len(bytes), markers).is_empty() { + if kept_hit(bytes, markers, pair) || !occurrences(&text_same_len(bytes), markers).is_empty() { return true; } [false, true].into_iter().any(|plus| { @@ -169,10 +215,14 @@ pub(crate) fn holds_typed( /// a UUID, 16 or more hex characters, or any match of a marker (see below) touches it. /// /// Userinfo and fragment are dropped. A query key or host label holding a typed value is written -/// as "{typed}". Matching is over the WHOLE path, query and host (raw and percent-decoded, ASCII -/// case ignored; the path also with `/` read as `\`, which the parser rewrites), and every -/// segment, key or label a match touches is replaced, so a typed value holding a separator -/// leaves no piece behind. A combined credential (`pair`) found in a segment or key replaces it. +/// as "{typed}". Matching is over WHOLE strings: the path, the query and the host each, and also +/// the whole URL as the parser serialises it (`Url::as_str`, with component offsets from +/// `url::Position`) and a percent-decoded copy built piece by piece, so a typed value holding +/// `?`, `#`, `@` or `/` that the parser split into components is still found. All of them ignore +/// ASCII case, and the path also reads `/` as `\` (the parser rewrites it). Every segment, key or +/// label a match touches is replaced. A combined credential (`pair`) found in a segment or key +/// replaces it. A marker shorter than `LOW_CONFIDENCE_BELOW` characters matches only as a whole +/// token (see `occurrences`). /// /// A URL that cannot be a base (`data:`, `blob:`, `about:`, `mailto:`) keeps only its scheme. /// The scheme is kept only when it is in a short known list, else it is `{other}`. @@ -200,16 +250,84 @@ pub fn strip_url( path_segments_replaced: 0, }); } + + // Component offsets in the serialisation. + let ser = parsed.as_str(); + let at = |p: Position| ser.len() - parsed[p..].len(); + let (host_start, host_end) = (at(Position::BeforeHost), at(Position::AfterHost)); + let (path_start, path_end) = (at(Position::BeforePath), at(Position::AfterPath)); + let query = parsed.query(); + let query_start = path_end + 1; + + let path_parts = PathParts::new(parsed.path()); + let query_parts = QueryParts::new(query.unwrap_or("")); + + // The decoded whole URL: the pieces decoded one by one, in order, with the delimiters between + // them, so offsets of the host, path and query inside it are known. Its capacity covers the + // text, so it is never reallocated (the copy is zeroized on drop). + let host = &ser[host_start..host_end]; + let decode = |s: &str| text_same_len(&decode_percent(s.as_bytes(), false)); + let mut whole = Zeroizing::new(String::with_capacity(ser.len() + 2)); + whole.push_str(&decode(&ser[..host_start])); + let d_host = whole.len()..whole.len() + host.len(); + whole.push_str(host); + whole.push_str(&decode(&ser[host_end..path_start])); + let d_path = whole.len()..whole.len() + path_parts.decoded_path.len(); + whole.push_str(&path_parts.decoded_path); + let mut d_query = whole.len()..whole.len(); + if query.is_some() { + whole.push('?'); + d_query = whole.len()..whole.len() + query_parts.decoded_query.len(); + whole.push_str(&query_parts.decoded_query); + } + if let Some(fragment) = parsed.fragment() { + whole.push('#'); + whole.push_str(&decode(fragment)); + } + + let ranges_in = |text: &str| { + let mut ranges = occurrences(text, markers); + ranges.extend(occurrences( + &Zeroizing::new(text.replace('/', "\\")), + markers, + )); + ranges + }; + let raw_all = ranges_in(ser); + let dec_all = ranges_in(&whole); + + let host_ranges: Vec> = rebase(&raw_all, host_start, host_end) + .into_iter() + .chain(rebase(&dec_all, d_host.start, d_host.end)) + .collect(); let origin = match parsed.host_str() { Some(host) => { let port = parsed.port().map(|p| format!(":{p}")).unwrap_or_default(); - format!("{scheme}://{}{port}", strip_host(host, markers)) + format!( + "{scheme}://{}{port}", + strip_host(host, markers, &host_ranges) + ) } None => format!("{scheme}:"), }; - let (path, path_segments_replaced) = strip_path(parsed.path(), markers, pair); - let (query_keys, query_values_removed) = - strip_query(parsed.query().unwrap_or(""), markers, pair); + let (path, path_segments_replaced) = strip_path( + &path_parts, + markers, + pair, + &rebase(&raw_all, path_start, path_end), + &rebase(&dec_all, d_path.start, d_path.end), + ); + let (query_keys, query_values_removed) = strip_query( + &query_parts, + markers, + pair, + &rebase( + &raw_all, + query_start, + query_start + query.map_or(0, str::len), + ), + &rebase(&dec_all, d_query.start, d_query.end), + ); Some(Stripped { value: StrippedUrl { origin, @@ -222,11 +340,13 @@ pub fn strip_url( } /// Replaces host labels that hold a typed value. Every occurrence of every marker is located -/// (ASCII case-insensitively: the parser lowercases hosts) and every label the occurrence touches is -/// replaced. Adjacent replaced labels become ONE `{typed}` label, so a value that spans labels -/// (`kh.throwaway7`) leaves no part of itself behind. -fn strip_host(host: &str, markers: &[Marker]) -> String { - let ranges = occurrences(host, markers); +/// (ASCII case-insensitively: the parser lowercases hosts) and every label an occurrence touches +/// is replaced; `extra` holds matches found over the whole URL, in host coordinates. Adjacent +/// replaced labels become ONE `{typed}` label, so a value that spans labels (`kh.throwaway7`) +/// leaves no part of itself behind. +fn strip_host(host: &str, markers: &[Marker], extra: &[Range]) -> String { + let mut ranges = occurrences(host, markers); + ranges.extend_from_slice(extra); let mut spans = Vec::new(); let mut offset = 0; for label in host.split('.') { @@ -249,53 +369,88 @@ fn strip_host(host: &str, markers: &[Marker]) -> String { labels.join(".") } -/// Segments of `path` are replaced when they look like an identifier or a match touches them. -fn strip_path(path: &str, markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> (String, u64) { - let segments: Vec<&str> = path.split('/').collect(); - let decoded: Vec>> = segments - .iter() - .map(|s| decode_percent(s.as_bytes(), false)) - .collect(); - let decoded_text: Vec> = decoded.iter().map(|d| text_same_len(d)).collect(); +/// The pieces of a URL path that matching needs: raw and decoded segments with their spans. +struct PathParts<'a> { + path: &'a str, + segments: Vec<&'a str>, + decoded: Vec>>, + raw_spans: Vec>, + decoded_spans: Vec>, + /// The decoded segments joined with `/`. + decoded_path: Zeroizing, +} - let spans_of = |lens: Vec| -> Vec> { - let mut offset = 0; - lens.into_iter() - .map(|len| { - let span = offset..offset + len; - offset += len + 1; - span - }) - .collect() - }; - let raw_spans = spans_of(segments.iter().map(|s| s.len()).collect()); - let decoded_spans = spans_of(decoded_text.iter().map(|t| t.len()).collect()); - let mut decoded_path = Zeroizing::new(String::with_capacity(path.len())); - for (i, t) in decoded_text.iter().enumerate() { - if i > 0 { - decoded_path.push('/'); +fn spans_of(lens: impl Iterator) -> Vec> { + let mut offset = 0; + lens.map(|len| { + let span = offset..offset + len; + offset += len + 1; + span + }) + .collect() +} + +impl<'a> PathParts<'a> { + fn new(path: &'a str) -> Self { + let segments: Vec<&str> = path.split('/').collect(); + let decoded: Vec>> = segments + .iter() + .map(|s| decode_percent(s.as_bytes(), false)) + .collect(); + let texts: Vec> = decoded.iter().map(|d| text_same_len(d)).collect(); + let mut decoded_path = Zeroizing::new(String::with_capacity(path.len())); + for (i, t) in texts.iter().enumerate() { + if i > 0 { + decoded_path.push('/'); + } + decoded_path.push_str(t); + } + Self { + path, + raw_spans: spans_of(segments.iter().map(|s| s.len())), + decoded_spans: spans_of(texts.iter().map(|t| t.len())), + segments, + decoded, + decoded_path, } - decoded_path.push_str(t); } +} +/// Segments are replaced when they look like an identifier or a match touches them. `raw_extra` +/// and `dec_extra` are matches found over the whole URL, in raw and decoded path coordinates. +fn strip_path( + parts: &PathParts, + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, + raw_extra: &[Range], + dec_extra: &[Range], +) -> (String, u64) { + let segments = &parts.segments; + let mut typed = vec![false; segments.len()]; + let mut mark = |hit: Vec| typed.iter_mut().zip(hit).for_each(|(t, h)| *t |= h); + + mark(touched(raw_extra, &parts.raw_spans)); + mark(touched(dec_extra, &parts.decoded_spans)); // The raw and decoded whole path, each also with `/` read as `\` (same byte length, so the // same spans apply). - let mut typed = vec![false; segments.len()]; - for (text, spans) in [(path, &raw_spans), (decoded_path.as_str(), &decoded_spans)] { + for (text, spans) in [ + (parts.path, &parts.raw_spans), + (parts.decoded_path.as_str(), &parts.decoded_spans), + ] { let backslashed = Zeroizing::new(text.replace('/', "\\")); for variant in [text, backslashed.as_str()] { - let hit = touched(&occurrences(variant, markers), spans); - typed.iter_mut().zip(hit).for_each(|(t, h)| *t |= h); + mark(touched(&occurrences(variant, markers), spans)); } } for (i, seg) in segments.iter().enumerate() { - typed[i] |= any_hit(seg.as_bytes(), markers, pair) || any_hit(&decoded[i], markers, pair); + typed[i] |= + kept_hit(seg.as_bytes(), markers, pair) || kept_hit(&parts.decoded[i], markers, pair); } // A hit over the whole path that no segment explains (an encoded value that spans a `/`): // every non-empty segment goes. if !typed.iter().any(|t| *t) - && (any_hit(path.as_bytes(), markers, pair) - || any_hit(decoded_path.as_bytes(), markers, pair)) + && (kept_hit(parts.path.as_bytes(), markers, pair) + || kept_hit(parts.decoded_path.as_bytes(), markers, pair)) { typed = segments.iter().map(|s| !s.is_empty()).collect(); } @@ -332,64 +487,94 @@ fn is_uuid(seg: &str) -> bool { }) } +struct QueryPiece<'a> { + raw_key: &'a str, + raw_span: Range, + key: Zeroizing>, + decoded_span: Range, +} + +/// The pieces of a query string that matching needs. +struct QueryParts<'a> { + query: &'a str, + pieces: Vec>, + /// `key=value` pieces decoded (`+` as space, `%XX`) and joined with `&`. + decoded_query: Zeroizing, +} + +impl<'a> QueryParts<'a> { + fn new(query: &'a str) -> Self { + let mut pieces = Vec::new(); + let mut decoded_query = Zeroizing::new(String::with_capacity(query.len())); + let mut offset = 0; + for piece in query.split('&') { + let start = offset; + offset += piece.len() + 1; + if piece.is_empty() { + continue; + } + let (raw_key, value) = match piece.split_once('=') { + Some((k, v)) => (k, Some(v)), + None => (piece, None), + }; + let key = decode_percent(raw_key.as_bytes(), true); + if !decoded_query.is_empty() { + decoded_query.push('&'); + } + let decoded_start = decoded_query.len(); + decoded_query.push_str(&text_same_len(&key)); + let decoded_span = decoded_start..decoded_query.len(); + if let Some(v) = value { + decoded_query.push('='); + decoded_query.push_str(&text_same_len(&decode_percent(v.as_bytes(), true))); + } + pieces.push(QueryPiece { + raw_key, + raw_span: start..start + raw_key.len(), + key, + decoded_span, + }); + } + Self { + query, + pieces, + decoded_query, + } + } +} + /// Query keys, in order, with a key a match touches written `{typed}`. The raw query string and a -/// decoded copy (`+` as space, `%XX`) are each searched whole; a match touching a key's bytes -/// (separators included) marks the key. A match inside a value alone costs nothing. +/// decoded copy (`+` as space, `%XX`) are each searched whole, as are `raw_extra` and `dec_extra` +/// (matches found over the whole URL, in raw and decoded query coordinates); a match touching a +/// key's bytes (separators included) marks the key. A match inside a value alone costs nothing. fn strip_query( - query: &str, + parts: &QueryParts, markers: &[Marker], pair: Option<(&Marker, &Marker)>, + raw_extra: &[Range], + dec_extra: &[Range], ) -> (Vec, u64) { - struct Piece<'a> { - raw_key: &'a str, - raw_span: Range, - key: Zeroizing>, - decoded_span: Range, - } - let mut pieces: Vec = Vec::new(); - let mut decoded_query = Zeroizing::new(String::with_capacity(query.len())); - let mut offset = 0; - for piece in query.split('&') { - let start = offset; - offset += piece.len() + 1; - if piece.is_empty() { - continue; - } - let (raw_key, value) = match piece.split_once('=') { - Some((k, v)) => (k, Some(v)), - None => (piece, None), - }; - let key = decode_percent(raw_key.as_bytes(), true); - let key_text = text_same_len(&key); - if !decoded_query.is_empty() { - decoded_query.push('&'); - } - let decoded_start = decoded_query.len(); - decoded_query.push_str(&key_text); - let decoded_span = decoded_start..decoded_query.len(); - if let Some(v) = value { - decoded_query.push('='); - decoded_query.push_str(&text_same_len(&decode_percent(v.as_bytes(), true))); - } - pieces.push(Piece { - raw_key, - raw_span: start..start + raw_key.len(), - key, - decoded_span, - }); - } - let raw_spans: Vec> = pieces.iter().map(|p| p.raw_span.clone()).collect(); - let decoded_spans: Vec> = pieces.iter().map(|p| p.decoded_span.clone()).collect(); - let by_raw = touched(&occurrences(query, markers), &raw_spans); - let by_decoded = touched(&occurrences(&decoded_query, markers), &decoded_spans); - let keys = pieces + let raw_spans: Vec> = parts.pieces.iter().map(|p| p.raw_span.clone()).collect(); + let decoded_spans: Vec> = parts + .pieces + .iter() + .map(|p| p.decoded_span.clone()) + .collect(); + let mut raw_ranges = occurrences(parts.query, markers); + raw_ranges.extend_from_slice(raw_extra); + let mut dec_ranges = occurrences(&parts.decoded_query, markers); + dec_ranges.extend_from_slice(dec_extra); + let by_raw = touched(&raw_ranges, &raw_spans); + let by_decoded = touched(&dec_ranges, &decoded_spans); + let keys = parts + .pieces .iter() .enumerate() .map(|(i, p)| { if by_raw[i] || by_decoded[i] - || any_hit(p.raw_key.as_bytes(), markers, pair) - || any_hit(&p.key, markers, pair) + || kept_hit(p.raw_key.as_bytes(), markers, pair) + || kept_hit(&p.key, markers, pair) { TYPED.to_owned() } else { @@ -397,7 +582,7 @@ fn strip_query( } }) .collect(); - (keys, u64::try_from(pieces.len()).unwrap_or(u64::MAX)) + (keys, u64::try_from(parts.pieces.len()).unwrap_or(u64::MAX)) } /// scheme = first token, lowercased. For `basic`, the credentials are base64-decoded and the decoded bytes @@ -757,4 +942,80 @@ mod tests { assert_eq!(dedup_hits(vec![h(true), h(false)]), [h(false)]); assert_eq!(dedup_hits(vec![h(false), h(true)]), [h(false)]); } + + #[test] + fn a_value_holding_a_url_delimiter_is_not_split_across_components() { + // `?`: the value is cut into a path segment and a query key. + let m = [user("ab?cd")]; + let s = strip_url("https://ex.test/a/ab?cd=1", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}"); + assert_eq!(s.value.query_keys, ["{typed}"]); + // `#`: the tail is a fragment, which is dropped; the head must not stay. + let m = [user("ab#cd")]; + let s = strip_url("https://ex.test/a/ab#cd", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}"); + // `@`: userinfo and host. + let m = [user("ab@cd")]; + let s = strip_url("https://ab@cd.ex.test/x", &m, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + let s = strip_url("https://ex.test/x/ab@cd", &m, None).unwrap(); + assert_eq!(s.value.path, "/x/{id}"); + let m = [pw("Tr0ub4dor?9x")]; + let s = strip_url("https://ex.test/login/Tr0ub4dor?9x", &m, None).unwrap(); + assert_eq!(s.value.path, "/login/{id}"); + assert_eq!(s.value.query_keys, ["{typed}"]); + } + + #[test] + fn a_match_across_a_delimiter_is_found_in_the_decoded_url_too() { + let m = [user("ab?cd")]; + // Fully encoded: one segment. + let s = strip_url("https://ex.test/a/ab%3Fcd?x=1", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}"); + assert_eq!(s.value.query_keys, ["x"]); + // Only the key is encoded: the decoded URL holds `ab?cd` across path and query. + let s = strip_url("https://ex.test/a/ab?c%64=1", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}"); + assert_eq!(s.value.query_keys, ["{typed}"]); + let m = [user("ab#cd")]; + let s = strip_url("https://ex.test/a/ab#%63d", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}"); + } + + #[test] + fn a_short_marker_matches_kept_text_only_as_a_whole_token() { + let on = [user("on")]; + let s = strip_url("https://ex.test/login/session?json=1", &on, None).unwrap(); + assert_eq!(s.value.path, "/login/session"); + assert_eq!(s.value.query_keys, ["json"]); + assert!(!holds_typed(b"application/json", &on, None)); + let s = strip_url("https://ex.test/u/on/x?on=1", &on, None).unwrap(); + assert_eq!(s.value.path, "/u/{id}/x"); + assert_eq!(s.value.query_keys, ["{typed}"]); + let one = [user("1")]; + let s = strip_url("https://ex.test/v1/login", &one, None).unwrap(); + assert_eq!(s.value.path, "/v1/login"); + let s = strip_url("https://ex.test/item/1", &one, None).unwrap(); + assert_eq!(s.value.path, "/item/{id}"); + let bob = [user("bob")]; + for (url, path, keys) in [ + ("https://ex.test/u/bob/x", "/u/{id}/x", vec![]), + ("https://ex.test/user-bob", "/{id}", vec![]), + ("https://ex.test/bob@ex.test", "/{id}", vec![]), + ("https://ex.test/x?bob=1", "/x", vec!["{typed}"]), + ("https://ex.test/bobby/x?bobby=1", "/bobby/x", vec!["bobby"]), + ] { + let s = strip_url(url, &bob, None).unwrap(); + assert_eq!(s.value.path, path, "{url}"); + assert_eq!(s.value.query_keys, keys, "{url}"); + } + // Six characters or more still match as a substring. + let s = strip_url("https://ex.test/x/zzabcdefzz", &[user("abcdef")], None).unwrap(); + assert_eq!(s.value.path, "/x/{id}"); + // Hosts follow the same rule. + let s = strip_url("https://person.ex.test/", &on, None).unwrap(); + assert_eq!(s.value.origin, "https://person.ex.test"); + let s = strip_url("https://on.ex.test/", &on, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + } } From 035f70e17f8dfcb56c7b5fd28a93314004e1aef1 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:31:26 -0500 Subject: [PATCH 15/36] fix(teach): tighten capture's kept text, username choice and memory bound (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-teach/src/capture/build.rs | 53 +++++- crates/kharon-teach/src/capture/mod.rs | 204 ++++++++++++++++++++- crates/kharon-teach/src/capture/resolve.rs | 44 +++-- crates/kharon-teach/src/capture/types.rs | 1 + crates/kharon-teach/src/site.rs | 15 +- 5 files changed, 297 insertions(+), 20 deletions(-) diff --git a/crates/kharon-teach/src/capture/build.rs b/crates/kharon-teach/src/capture/build.rs index c0583ae..6a76f83 100644 --- a/crates/kharon-teach/src/capture/build.rs +++ b/crates/kharon-teach/src/capture/build.rs @@ -86,6 +86,55 @@ fn selector_is_clean(selector: &str, ctx: Ctx<'_>) -> bool { !ctx.holds_typed(selector) && !ctx.holds_typed(&unescaped) } +const INPUT_TYPES: [&str; 22] = [ + "text", + "password", + "email", + "tel", + "number", + "search", + "url", + "hidden", + "checkbox", + "radio", + "submit", + "button", + "reset", + "file", + "date", + "datetime-local", + "month", + "week", + "time", + "color", + "range", + "image", +]; + +/// An HTML input type keyword, lowercased; anything else is `{other}`. +fn input_type_keyword(raw: &str) -> String { + let lower = raw.to_ascii_lowercase(); + if INPUT_TYPES.contains(&lower.as_str()) { + lower + } else { + "{other}".to_owned() + } +} + +/// A tag name, lowercased, when it is ASCII letters, digits and `-` and holds no typed value. +fn tag_name(raw: &str, ctx: Ctx<'_>) -> String { + let lower = raw.to_ascii_lowercase(); + let plain = !lower.is_empty() + && lower + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-'); + if plain && !ctx.holds_typed(&lower) { + lower + } else { + "{other}".to_owned() + } +} + fn known_method(method: &str, lowercase: bool) -> String { let upper = method.to_ascii_uppercase(); let known = ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]; @@ -122,8 +171,8 @@ pub(super) fn describe( ) -> ElementDescriptor { ElementDescriptor { element_id: raw.element_id, - tag: raw.tag.to_lowercase(), - input_type: raw.input_type.as_deref().map(str::to_lowercase), + tag: tag_name(&raw.tag, ctx), + input_type: raw.input_type.as_deref().map(input_type_keyword), id: clean(raw.id.as_ref(), ctx), name: clean(raw.name.as_ref(), ctx), autocomplete: clean(raw.autocomplete.as_ref(), ctx), diff --git a/crates/kharon-teach/src/capture/mod.rs b/crates/kharon-teach/src/capture/mod.rs index 08d099c..aad2514 100644 --- a/crates/kharon-teach/src/capture/mod.rs +++ b/crates/kharon-teach/src/capture/mod.rs @@ -21,6 +21,9 @@ use crate::site::{is_captcha_host, registrable_domain}; use build::{Ctx, Tally, describe, form_ref, frame_ref, reduce_request, stripped_url}; use resolve::{Elem, resolve}; +/// Once this many body bytes are buffered, every further body is buffered as `Unavailable`. +pub const MAX_BUFFERED_BODY_BYTES: usize = 32 << 20; + /// A frame as it was when an event arrived. struct FrameInfo { main: bool, @@ -84,6 +87,8 @@ pub struct Capture { cross_site: bool, framed_login: bool, body_unavailable: bool, + body_budget: usize, + buffered_body_bytes: usize, } fn host_of(url: &str) -> Option { @@ -138,6 +143,16 @@ impl Capture { cross_site: false, framed_login: false, body_unavailable: false, + body_budget: MAX_BUFFERED_BODY_BYTES, + buffered_body_bytes: 0, + } + } + + #[cfg(test)] + fn with_body_budget(start_url: &str, tool_version: &str, budget: usize) -> Self { + Self { + body_budget: budget, + ..Self::new(start_url, tool_version) } } @@ -268,6 +283,7 @@ impl Capture { has_step: false, first_seq: self.seq + 1, last_seq: 0, + seqs: Vec::new(), }); let i = self.elems.len() - 1; self.note_report(i, &element, &info); @@ -276,6 +292,7 @@ impl Capture { }; self.seq += 1; self.elems[i].last_seq = self.seq; + self.elems[i].seqs.push(self.seq); if let Some(v) = value { self.elems[i].value = v; } @@ -317,8 +334,14 @@ impl Capture { if is_captcha_url(&b.url) { self.captcha = true; } - if matches!(&b.body, RawBody::Bytes(bytes) if bytes.len() > MAX_BODY_BYTES) { - b.body = RawBody::Unavailable; + if let RawBody::Bytes(bytes) = &b.body { + if bytes.len() > MAX_BODY_BYTES + || self.buffered_body_bytes.saturating_add(bytes.len()) > self.body_budget + { + b.body = RawBody::Unavailable; + } else { + self.buffered_body_bytes += bytes.len(); + } } let to_login_site = host_of(&b.url).is_some_and(|h| registrable_domain(&h) == self.login_site); @@ -1427,4 +1450,181 @@ mod tests { assert_eq!(rec.stripped.requests_seen, 5); let _ = rec.to_json(); } + + fn session_with_budget(budget: usize) -> Session { + let mut s = Session::unstarted(LOGIN); + s.cap = Capture::with_body_budget(LOGIN, "0.0.0", budget); + s.start(); + s + } + + #[test] + fn bodies_past_the_buffer_limit_are_unavailable() { + let big = |n: usize| RawBody::Bytes(Zeroizing::new(vec![b'a'; n])); + let rec = session_with_budget(100) + .type_into(&user_el(), USER) + .request( + "POST", + "https://ex.test/a", + ResourceKind::Fetch, + false, + Some("text/plain"), + big(80), + ) + .request( + "POST", + "https://ex.test/b", + ResourceKind::Fetch, + false, + Some("text/plain"), + big(80), + ) + .finish(); + assert_eq!(rec.requests.len(), 2); + assert!(matches!( + rec.requests[0].body, + BodyShape::Opaque { len: 80, .. } + )); + assert_eq!(rec.requests[1].body, BodyShape::Unavailable); + assert!(rec.flags.body_unavailable); + assert_eq!(rec.stripped.request_bodies_reduced, 1); + assert_eq!(MAX_BUFFERED_BODY_BYTES, 32 << 20); + } + + #[test] + fn a_combined_credential_alone_picks_the_username() { + let combined = Base64::encode_string(format!("{USER}:{PASS}").as_bytes()); + let tenant = no_form(el(4, "tenant", "text")); + let rec = Session::new(LOGIN) + .type_into(&no_form(user_el()), USER) + .type_into(&tenant, TENANT) + .type_into(&no_form(pw_el()), PASS) + .post_json( + "https://ex.test/api/login", + &format!(r#"{{"auth":"{combined}"}}"#), + ) + .finish(); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 4), Some(Role::None)); + assert_holds_no_typed_value(&rec.to_json(), &ALL_VALUES); + } + + #[test] + fn the_same_form_beats_a_later_typed_candidate() { + let tenant = no_form(el(4, "tenant", "text")); + let rec = Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&tenant, TENANT) + .type_into(&pw_el(), PASS) + .post_form("https://ex.test/login", &format!("t={TENANT}&p={PASS}")) + .finish(); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 4), Some(Role::None)); + } + + #[test] + fn a_value_found_in_any_request_beats_a_later_typed_one() { + let tenant = no_form(el(4, "tenant", "text")); + let rec = Session::new(LOGIN) + .type_into(&no_form(user_el()), USER) + .type_into(&tenant, TENANT) + .type_into(&no_form(pw_el()), PASS) + .post_form("https://ex.test/identify", &format!("u={USER}")) + .post_form("https://ex.test/login", &format!("p={PASS}")) + .finish(); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 4), Some(Role::None)); + } + + #[test] + fn with_no_other_evidence_the_last_typed_before_the_password_wins() { + let tenant = no_form(el(4, "tenant", "text")); + let rec = Session::new(LOGIN) + .type_into(&tenant, TENANT) + .type_into(&no_form(user_el()), USER) + .type_into(&no_form(pw_el()), PASS) + .finish(); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 4), Some(Role::None)); + } + + #[test] + fn an_edit_after_the_password_does_not_beat_the_last_field_typed_before_it() { + let first = no_form(el(4, "tenant", "text")); + let rec = Session::new(LOGIN) + .type_into(&first, TENANT) + .type_into(&no_form(user_el()), USER) + .type_into(&no_form(pw_el()), PASS) + .type_into(&first, TENANT) + .finish(); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 4), Some(Role::None)); + } + + #[test] + fn typed_values_in_every_kept_attribute_are_dropped() { + let mut e = user_el(); + e.name = Some(format!("n-{USER}")); + e.autocomplete = Some(format!("x-{USER}")); + e.aria_label = Some(format!("for {USER}")); + e.form = Some(RawForm { + index: 0, + action: Some(format!("https://ex.test/login?who={USER}&{USER}=1")), + method: "post".into(), + }); + let mut clicked = button(); + clicked.id = Some(format!("go-{USER}")); + clicked.selectors = vec![format!("#go-{USER}"), "button[type=submit]".into()]; + let rec = Session::new(LOGIN) + .type_into(&e, USER) + .click(&clicked) + .finish(); + let (d, _, _) = inputs(&rec)[0]; + assert_eq!( + (&d.name, &d.autocomplete, &d.aria_label), + (&None, &None, &None) + ); + let action = d.form.as_ref().and_then(|f| f.action.as_ref()); + assert!(action.is_some_and(|a| a.query_keys.iter().all(|k| k != USER))); + match &rec.steps[2] { + Step::Click { element, .. } => { + assert_eq!(element.id, None); + assert_eq!(element.selectors, vec!["button[type=submit]".to_owned()]); + } + other => panic!("expected a click, got {other:?}"), + } + assert_holds_no_typed_value(&rec.to_json(), &[USER]); + } + + #[test] + fn page_derived_tag_and_type_are_checked() { + let mut odd = user_el(); + odd.tag = "x.weird".into(); + odd.input_type = Some("weird".into()); + let mut custom = el(6, "c", "TEL"); + custom.tag = "My-Widget".into(); + let mut typed = el(7, "d", "text"); + typed.tag = format!("x-{USER}"); + let rec = Session::new(LOGIN) + .type_into(&odd, USER) + .type_into(&custom, "1") + .type_into(&typed, "2") + .finish(); + let ins = inputs(&rec); + assert_eq!(ins[0].0.tag, "{other}"); + assert_eq!(ins[0].0.input_type.as_deref(), Some("{other}")); + assert_eq!(ins[1].0.tag, "my-widget"); + assert_eq!(ins[1].0.input_type.as_deref(), Some("tel")); + assert_eq!(ins[2].0.tag, "{other}"); + } + + #[test] + fn the_show_password_scenario_keeps_both_lengths() { + let rec = scenario("show_toggle"); + let lens: Vec<(u32, u32)> = inputs(&rec) + .into_iter() + .map(|(e, len, _)| (e.element_id, len)) + .collect(); + assert_eq!(lens, vec![(1, 13), (2, 12)]); + } } diff --git a/crates/kharon-teach/src/capture/resolve.rs b/crates/kharon-teach/src/capture/resolve.rs index 8e28fb7..b1d3b32 100644 --- a/crates/kharon-teach/src/capture/resolve.rs +++ b/crates/kharon-teach/src/capture/resolve.rs @@ -17,6 +17,8 @@ pub(super) struct Elem { /// Sequence numbers of the first and the latest value-bearing event. pub first_seq: u64, pub last_seq: u64, + /// Sequence numbers of every value-bearing event. + pub seqs: Vec, } impl Elem { @@ -50,24 +52,37 @@ fn plain_marker(value: &str, role: Role) -> Marker { Marker::new(Zeroizing::new(value.to_owned()), role) } -/// Whether `marker` occurs in the request's URL, authorization header or body, under any encoding. -fn carries(req: &Buffered, marker: &Marker) -> bool { +/// Whether `marker` (or, with `pair`, the combined credential) occurs in the request's URL, +/// authorization header or body, under any encoding. +fn carries_with(req: &Buffered, marker: &Marker, pair: Option<(&Marker, &Marker)>) -> bool { let one = std::slice::from_ref(marker); - any_hit(req.url.as_bytes(), one, None) + any_hit(req.url.as_bytes(), one, pair) || req .authorization .as_ref() - .is_some_and(|a| any_hit(a, one, None)) - || matches!(&req.body, RawBody::Bytes(b) if any_hit(b, one, None)) + .is_some_and(|a| any_hit(a, one, pair)) + || matches!(&req.body, RawBody::Bytes(b) if any_hit(b, one, pair)) } -/// The candidate typed in most recently before the first password input; failing that, the latest. +fn carries(req: &Buffered, marker: &Marker) -> bool { + carries_with(req, marker, None) +} + +/// The candidate whose last input event before the first password input is the latest; failing +/// that (none typed before it), the one with the latest input of any kind. fn pick(cands: &[usize], elems: &[Elem], first_password_seq: Option) -> Option { + let last_before = |i: usize| -> Option { + match first_password_seq { + None => Some(elems[i].last_seq), + Some(s) => elems[i].seqs.iter().copied().filter(|q| *q < s).max(), + } + }; cands .iter() .copied() - .filter(|&i| first_password_seq.is_none_or(|s| elems[i].first_seq < s)) - .max_by_key(|&i| elems[i].last_seq) + .filter_map(|i| last_before(i).map(|q| (q, i))) + .max() + .map(|(_, i)| i) .or_else(|| cands.iter().copied().max_by_key(|&i| elems[i].last_seq)) } @@ -116,11 +131,18 @@ pub(super) fn resolve(elems: &[Elem], reqs: &[Buffered]) -> Resolved { .any(|p| p.was_password && p.form_key() == Some(k)) }) }); + // A login that sends only a combined credential matches when probed with the pair. + let combined = |r: &Buffered, m: &Marker| { + password_markers + .iter() + .any(|p| carries_with(r, m, Some((m, p)))) + }; let with_password = narrowed(&|_, m| { - reqs.iter() - .any(|r| carries(r, m) && password_markers.iter().any(|p| carries(r, p))) + reqs.iter().any(|r| { + (carries(r, m) && password_markers.iter().any(|p| carries(r, p))) || combined(r, m) + }) }); - let in_any_request = narrowed(&|_, m| reqs.iter().any(|r| carries(r, m))); + let in_any_request = narrowed(&|_, m| reqs.iter().any(|r| carries(r, m) || combined(r, m))); let username_elem = [same_form, with_password, in_any_request, cands] .iter() diff --git a/crates/kharon-teach/src/capture/types.rs b/crates/kharon-teach/src/capture/types.rs index 6955cc1..5efd2cf 100644 --- a/crates/kharon-teach/src/capture/types.rs +++ b/crates/kharon-teach/src/capture/types.rs @@ -25,6 +25,7 @@ pub struct RawElement { #[derive(Clone)] pub struct RawForm { pub index: u32, + /// The resolved absolute `form.action` DOM property, not the attribute text. pub action: Option, pub method: String, } diff --git a/crates/kharon-teach/src/site.rs b/crates/kharon-teach/src/site.rs index de6fbee..ded3e60 100644 --- a/crates/kharon-teach/src/site.rs +++ b/crates/kharon-teach/src/site.rs @@ -23,20 +23,22 @@ fn psl() -> Option<&'static List> { .as_ref() } -/// The registrable domain of `host` (its eTLD+1), lowercased. An IP address, a single-label host -/// such as `localhost`, or a host the list has no answer for is returned unchanged. +/// The registrable domain of `host` (its eTLD+1), lowercased, without a trailing dot. An IP +/// address, a single-label host such as `localhost`, or a host the list has no answer for is +/// returned lowercased and otherwise unchanged. pub fn registrable_domain(host: &str) -> String { + let host = host.strip_suffix('.').unwrap_or(host).to_ascii_lowercase(); let bare = host .strip_prefix('[') .and_then(|h| h.strip_suffix(']')) - .unwrap_or(host); + .unwrap_or(&host); if bare.parse::().is_ok() { - return host.to_owned(); + return host; } psl() .and_then(|list| list.domain(host.as_bytes())) .map_or_else( - || host.to_owned(), + || host.clone(), |d| String::from_utf8_lossy(d.as_bytes()).into_owned(), ) } @@ -61,6 +63,9 @@ mod tests { assert_eq!(registrable_domain("127.0.0.1"), "127.0.0.1"); assert_eq!(registrable_domain("localhost"), "localhost"); assert_eq!(registrable_domain("[::1]"), "[::1]"); + assert_eq!(registrable_domain("Login.Ex.Co.UK."), "ex.co.uk"); + assert_eq!(registrable_domain("ex.test."), "ex.test"); + assert_eq!(registrable_domain("LOCALHOST."), "localhost"); } #[test] From ea31220be7a62303d45220d3aec79a1ca4f0d898 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:38:13 -0500 Subject: [PATCH 16/36] feat(teach): launch a browser on a private throwaway profile (#361) browser.rs finds a Chromium-family browser, creates a 0700 profile under the temp dir (random suffix, never reuses a name), writes Preferences that turn off the password manager and autofill, launches with a loopback DevTools port, and deletes the profile on remove or drop. sweep_stale removes leftover profiles owned by this user and skips symlinks and any profile whose SingletonLock names a live pid. All flags in the brief were accepted by the installed Chromium; none dropped. Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- Cargo.lock | 4 + crates/kharon-teach/Cargo.toml | 4 + crates/kharon-teach/src/browser.rs | 458 ++++++++++++++++++ crates/kharon-teach/src/lib.rs | 1 + .../tests/req_tch_001_own_browser.rs | 135 ++++++ crates/kharon-teach/tests/support/mod.rs | 33 ++ 6 files changed, 635 insertions(+) create mode 100644 crates/kharon-teach/src/browser.rs create mode 100644 crates/kharon-teach/tests/req_tch_001_own_browser.rs create mode 100644 crates/kharon-teach/tests/support/mod.rs diff --git a/Cargo.lock b/Cargo.lock index caa5f4e..fa07ed7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1090,9 +1090,13 @@ dependencies = [ "memchr", "percent-encoding", "publicsuffix", + "rand", + "serde", "serde_json", "sha1", "sha2", + "thiserror 2.0.21", + "tokio", "url", "zeroize", ] diff --git a/crates/kharon-teach/Cargo.toml b/crates/kharon-teach/Cargo.toml index 69fcf09..7dcdc9e 100644 --- a/crates/kharon-teach/Cargo.toml +++ b/crates/kharon-teach/Cargo.toml @@ -27,4 +27,8 @@ percent-encoding = "2" memchr = "2" url = "2" serde_json = "1" +serde = { version = "1", features = ["derive"] } +thiserror = "2" +rand = "0.8" +tokio = { version = "1", features = ["fs", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time", "io-std", "io-util"] } publicsuffix = "2.3" diff --git a/crates/kharon-teach/src/browser.rs b/crates/kharon-teach/src/browser.rs new file mode 100644 index 0000000..215a698 --- /dev/null +++ b/crates/kharon-teach/src/browser.rs @@ -0,0 +1,458 @@ +//! Find a Chromium-family browser, give it a private throwaway profile, and +//! guarantee the profile is deleted. + +use serde::Serialize; +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +#[derive(Debug, thiserror::Error)] +pub enum BrowserError { + #[error("no Chromium-family browser found; pass --browser ")] + NotFound, + #[error("browser not found at {0}")] + BadPath(PathBuf), + #[error("the browser did not start: {0}")] + Launch(String), +} + +pub const PROFILE_PREFIX: &str = "kharon-teach-"; + +const SEARCH_ORDER: [&str; 6] = [ + "chromium", + "chromium-browser", + "google-chrome", + "google-chrome-stable", + "microsoft-edge", + "brave-browser", +]; + +/// The browser to launch: `explicit` if given, else the first of the known +/// names found on `$PATH`. +pub fn find_browser(explicit: Option<&Path>) -> Result { + let path_var = std::env::var_os("PATH").unwrap_or_default(); + find_browser_in(explicit, &path_var) +} + +pub fn find_browser_in(explicit: Option<&Path>, path_var: &OsStr) -> Result { + if let Some(p) = explicit { + return if p.is_file() { + Ok(p.to_owned()) + } else { + Err(BrowserError::BadPath(p.to_owned())) + }; + } + for name in SEARCH_ORDER { + for dir in std::env::split_paths(path_var) { + let cand = dir.join(name); + if is_executable_file(&cand) { + return Ok(cand); + } + } + } + Err(BrowserError::NotFound) +} + +fn is_executable_file(p: &Path) -> bool { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(p) + .map(|m| m.is_file() && m.permissions().mode() & 0o111 != 0) + .unwrap_or(false) +} + +/// A private throwaway browser profile directory, deleted on drop. +pub struct Profile { + path: PathBuf, +} + +#[derive(Serialize)] +struct Preferences { + credentials_enable_service: bool, + profile: ProfilePrefs, + autofill: AutofillPrefs, +} + +#[derive(Serialize)] +struct ProfilePrefs { + password_manager_enabled: bool, +} + +#[derive(Serialize)] +struct AutofillPrefs { + profile_enabled: bool, + credit_card_enabled: bool, +} + +const PREFERENCES: Preferences = Preferences { + credentials_enable_service: false, + profile: ProfilePrefs { + password_manager_enabled: false, + }, + autofill: AutofillPrefs { + profile_enabled: false, + credit_card_enabled: false, + }, +}; + +fn private_dir(path: &Path) -> std::io::Result<()> { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new().mode(0o700).create(path) +} + +impl Profile { + /// `/kharon-teach-`, created atomically with mode 0700. + pub fn create() -> std::io::Result { + use rand::Rng; + use rand::distributions::Alphanumeric; + let temp = std::env::temp_dir(); + for _ in 0..16 { + let suffix: String = rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(20) + .map(char::from) + .collect(); + let path = temp.join(format!("{PROFILE_PREFIX}{suffix}")); + match private_dir(&path) { + Ok(()) => return Ok(Profile { path }), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e), + } + } + Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "could not find an unused profile directory name", + )) + } + + pub fn path(&self) -> &Path { + &self.path + } + + /// Delete the profile now and report a failure. Also runs on drop, + /// where errors are ignored. + pub fn remove(self) -> std::io::Result<()> { + remove_tree(&self.path) + } + + /// Removes leftover PROFILE_PREFIX directories in the temp location owned by this user. Returns how many. + pub fn sweep_stale() -> std::io::Result { + use std::os::unix::fs::MetadataExt; + // /proc/self is owned by this process's effective uid. + let me = std::fs::metadata("/proc/self")?.uid(); + let mut n = 0; + for entry in std::fs::read_dir(std::env::temp_dir())? { + let Ok(entry) = entry else { continue }; + if !entry + .file_name() + .to_string_lossy() + .starts_with(PROFILE_PREFIX) + { + continue; + } + let path = entry.path(); + let Ok(meta) = std::fs::symlink_metadata(&path) else { + continue; + }; + if !meta.file_type().is_dir() || meta.uid() != me || browser_is_alive(&path) { + continue; + } + if remove_tree(&path).is_ok() { + n += 1; + } + } + Ok(n) + } +} + +impl Drop for Profile { + fn drop(&mut self) { + let _ = remove_tree(&self.path); + } +} + +/// `remove_dir_all` does not follow symlinks. A just-killed browser's helper +/// processes can still be writing, so retry a few times. +fn remove_tree(path: &Path) -> std::io::Result<()> { + let mut last = Ok(()); + for _ in 0..5 { + match std::fs::remove_dir_all(path) { + Ok(()) => return Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => last = Err(e), + } + std::thread::sleep(Duration::from_millis(50)); + } + last +} + +/// Chromium's `SingletonLock` is a symlink whose target text is +/// `-`; the profile is live when that pid exists. +fn browser_is_alive(dir: &Path) -> bool { + let Ok(target) = std::fs::read_link(dir.join("SingletonLock")) else { + return false; + }; + target + .to_string_lossy() + .rsplit('-') + .next() + .and_then(|p| p.parse::().ok()) + .is_some_and(|pid| Path::new(&format!("/proc/{pid}")).exists()) +} + +pub struct Launch<'a> { + pub browser: &'a Path, + pub profile: &'a Profile, + pub first_url: &'a str, + pub headless: bool, +} + +/// The browser process. Killed when dropped. +pub struct Browser { + child: tokio::process::Child, +} + +pub struct DevTools { + pub port: u16, + pub ws_path: String, +} + +const START_TIMEOUT: Duration = Duration::from_secs(20); +const POLL: Duration = Duration::from_millis(50); + +impl Browser { + /// Start the browser on the profile and wait up to 20 s for its + /// `DevToolsActivePort` file. + pub async fn launch(l: Launch<'_>) -> Result<(Browser, DevTools), BrowserError> { + use std::process::Stdio; + let launch_err = |e: std::io::Error| BrowserError::Launch(e.to_string()); + let profile = l.profile.path(); + let default = profile.join("Default"); + private_dir(&default).map_err(launch_err)?; + let prefs = + serde_json::to_string(&PREFERENCES).map_err(|e| BrowserError::Launch(e.to_string()))?; + std::fs::write(default.join("Preferences"), prefs).map_err(launch_err)?; + + let mut cmd = tokio::process::Command::new(l.browser); + cmd.arg(format!("--user-data-dir={}", profile.display())) + .arg("--remote-debugging-port=0") + .arg("--no-first-run") + .arg("--no-default-browser-check") + .arg("--disk-cache-size=1") + .arg("--disable-sync") + .arg("--password-store=basic") + .arg("--disable-features=AutofillServerCommunication,PasswordManagerOnboarding"); + if l.headless { + cmd.arg("--headless=new"); + } + cmd.arg(l.first_url) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true); + let mut child = cmd.spawn().map_err(launch_err)?; + + let file = profile.join("DevToolsActivePort"); + let deadline = tokio::time::Instant::now() + START_TIMEOUT; + loop { + if let Some(status) = child.try_wait().map_err(launch_err)? { + return Err(BrowserError::Launch(status.to_string())); + } + if let Some(dt) = std::fs::read_to_string(&file) + .ok() + .and_then(|t| parse_devtools_active_port(&t)) + { + return Ok((Browser { child }, dt)); + } + if tokio::time::Instant::now() >= deadline { + let _ = child.kill().await; + return Err(BrowserError::Launch( + "no DevTools port after 20 s".to_owned(), + )); + } + tokio::time::sleep(POLL).await; + } + } + + pub fn pid(&self) -> u32 { + self.child.id().unwrap_or(0) + } + + pub async fn wait(&mut self) -> std::io::Result { + self.child.wait().await + } + + /// Kill the browser and wait until it is reaped. + pub async fn close(mut self) -> std::io::Result<()> { + self.child.kill().await + } +} + +/// A copy of the rules in `parse_devtools_active_port` in +/// `crates/kharon-connector/src/devtools.rs`: two lines, a port of 1 to 5 +/// digits that is not 0, and a `/devtools/browser/` path. The host is +/// never taken from the file. +pub fn parse_devtools_active_port(text: &str) -> Option { + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + let mut lines = text.lines(); + let port_line = lines.next()?.trim(); + let path = lines.next()?.trim(); + if port_line.is_empty() || port_line.len() > 5 || !port_line.bytes().all(|b| b.is_ascii_digit()) + { + return None; + } + let port: u16 = port_line.parse().ok()?; + if port == 0 { + return None; + } + let id = path.strip_prefix("/devtools/browser/")?; + if id.is_empty() + || id.len() > 128 + || !id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') + { + return None; + } + Some(DevTools { + port, + ws_path: path.to_owned(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + fn unique(tag: &str) -> String { + format!( + "{PROFILE_PREFIX}{tag}-{}-{:08x}", + std::process::id(), + rand::random::() + ) + } + + #[test] + fn profile_is_private_and_removed() { + let p = Profile::create().unwrap(); + let path = p.path().to_owned(); + assert!( + path.file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with(PROFILE_PREFIX) + ); + assert_eq!(std::fs::metadata(&path).unwrap().mode() & 0o777, 0o700); + p.remove().unwrap(); + assert!(!path.exists()); + } + + #[test] + fn profile_is_removed_on_drop() { + let p = Profile::create().unwrap(); + let path = p.path().to_owned(); + std::fs::write(path.join("f"), b"x").unwrap(); + drop(p); + assert!(!path.exists()); + } + + #[test] + fn sweep_removes_a_leftover_and_counts_it() { + let d = std::env::temp_dir().join(unique("stale")); + std::fs::create_dir(&d).unwrap(); + std::fs::write(d.join("f"), b"x").unwrap(); + assert!(Profile::sweep_stale().unwrap() >= 1); + assert!(!d.exists()); + } + + #[test] + fn sweep_leaves_other_directories() { + let d = std::env::temp_dir().join(format!("other-{}", unique("keep"))); + std::fs::create_dir(&d).unwrap(); + Profile::sweep_stale().unwrap(); + assert!(d.exists()); + std::fs::remove_dir(&d).unwrap(); + } + + #[test] + fn sweep_leaves_a_symlink_and_its_target() { + let target = std::env::temp_dir().join(format!("other-{}", unique("tgt"))); + std::fs::create_dir(&target).unwrap(); + std::fs::write(target.join("f"), b"x").unwrap(); + let link = std::env::temp_dir().join(unique("link")); + std::os::unix::fs::symlink(&target, &link).unwrap(); + Profile::sweep_stale().unwrap(); + assert!(target.join("f").exists()); + assert!(std::fs::symlink_metadata(&link).is_ok()); + std::fs::remove_file(&link).unwrap(); + std::fs::remove_dir_all(&target).unwrap(); + } + + // An addition to the brief: a profile whose SingletonLock names a live + // process is a running recorder's profile and must survive a sweep. + #[test] + fn sweep_skips_a_profile_whose_browser_is_alive() { + let d = std::env::temp_dir().join(unique("live")); + std::fs::create_dir(&d).unwrap(); + std::os::unix::fs::symlink( + format!("host-{}", std::process::id()), + d.join("SingletonLock"), + ) + .unwrap(); + Profile::sweep_stale().unwrap(); + assert!(d.exists()); + std::fs::remove_dir_all(&d).unwrap(); + } + + #[test] + fn no_browser_on_an_empty_path_is_not_found() { + assert!(matches!( + find_browser_in(None, OsStr::new("")), + Err(BrowserError::NotFound) + )); + } + + #[test] + fn finds_the_first_name_in_search_order() { + let dir = std::env::temp_dir().join(unique("bin")); + std::fs::create_dir(&dir).unwrap(); + for n in ["google-chrome", "chromium"] { + let f = dir.join(n); + std::fs::write(&f, b"#!/bin/sh\n").unwrap(); + std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let found = find_browser_in(None, dir.as_os_str()).unwrap(); + assert_eq!(found, dir.join("chromium")); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn explicit_missing_browser_is_bad_path() { + assert!(matches!( + find_browser(Some(Path::new("/nonexistent/chrome"))), + Err(BrowserError::BadPath(_)) + )); + } + + #[test] + fn parses_the_two_line_file() { + let d = parse_devtools_active_port("45123\n/devtools/browser/abc-123\n").unwrap(); + assert_eq!(d.port, 45123); + assert_eq!(d.ws_path, "/devtools/browser/abc-123"); + } + + #[test] + fn rejects_port_zero_garbage_and_a_foreign_path() { + for bad in [ + "0\n/devtools/browser/abc\n", + "123456\n/devtools/browser/abc\n", + "12x\n/devtools/browser/abc\n", + "", + "45123\n", + "45123\n/evil/browser/abc\n", + "45123\n/devtools/browser/\n", + "45123\n/devtools/browser/a/b\n", + ] { + assert!(parse_devtools_active_port(bad).is_none(), "{bad:?}"); + } + } +} diff --git a/crates/kharon-teach/src/lib.rs b/crates/kharon-teach/src/lib.rs index 2bc7e71..fe5f3de 100644 --- a/crates/kharon-teach/src/lib.rs +++ b/crates/kharon-teach/src/lib.rs @@ -1,6 +1,7 @@ //! Teach-mode recorder for kharon (#361). pub mod body; +pub mod browser; pub mod capture; pub mod marker; pub mod site; diff --git a/crates/kharon-teach/tests/req_tch_001_own_browser.rs b/crates/kharon-teach/tests/req_tch_001_own_browser.rs new file mode 100644 index 0000000..d8aaf1e --- /dev/null +++ b/crates/kharon-teach/tests/req_tch_001_own_browser.rs @@ -0,0 +1,135 @@ +//! SPEC req_tch_001: the recorder launches its own browser on its own +//! throwaway profile and the DevTools port is loopback only. + +mod support; + +use kharon_teach::browser::{Browser, Launch, Profile}; +use std::path::Path; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +/// Local addresses (hex ip) of every listening row for `port` in a +/// `/proc/net/tcp*` file. +fn listeners(file: &str, port: u16) -> Vec { + let text = std::fs::read_to_string(file).unwrap_or_default(); + let want = format!(":{port:04X}"); + text.lines() + .skip(1) + .filter_map(|l| { + let cols: Vec<&str> = l.split_whitespace().collect(); + let local = *cols.get(1)?; + let state = *cols.get(3)?; + (state == "0A" && local.ends_with(&want)) + .then(|| local[..local.len() - want.len()].to_owned()) + }) + .collect() +} + +#[tokio::test] +async fn launches_on_its_own_profile_and_binds_loopback_only() { + let Some(bin) = support::chromium_or_skip("launches_on_its_own_profile") else { + return; + }; + let profile = Profile::create().unwrap(); + let profile_path = profile.path().to_owned(); + let (browser, dt) = Browser::launch(Launch { + browser: &bin, + profile: &profile, + first_url: "about:blank", + headless: true, + }) + .await + .unwrap(); + + let cmdline = std::fs::read(format!("/proc/{}/cmdline", browser.pid())).unwrap(); + let cmdline = String::from_utf8_lossy(&cmdline).replace('\0', " "); + assert!( + cmdline.contains(&format!("--user-data-dir={}", profile_path.display())), + "{cmdline}" + ); + + let mut s = tokio::net::TcpStream::connect(("127.0.0.1", dt.port)) + .await + .unwrap(); + s.write_all(b"GET /json/version HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + // Chromium keeps the connection open, so read until the document is complete. + let mut body = String::new(); + let mut buf = [0u8; 4096]; + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while !body.trim_end().ends_with('}') { + let n = s.read(&mut buf).await.unwrap(); + assert!(n > 0, "closed early: {body}"); + body.push_str(&String::from_utf8_lossy(&buf[..n])); + } + }) + .await + .expect("no /json/version answer"); + assert!(body.starts_with("HTTP/1.1 200"), "{body}"); + assert!(body.contains("webSocketDebuggerUrl"), "{body}"); + + let v4 = listeners("/proc/net/tcp", dt.port); + let v6 = listeners("/proc/net/tcp6", dt.port); + assert!(!v4.is_empty() || !v6.is_empty(), "no listener found"); + assert!(v4.iter().all(|a| a == "0100007F"), "{v4:?}"); + assert!( + v6.iter() + .all(|a| a == "00000000000000000000000001000000" + || a == "0000000000000000FFFF00000100007F"), + "{v6:?}" + ); + + browser.close().await.unwrap(); + profile.remove().unwrap(); + assert!(!Path::new(&profile_path).exists()); +} + +#[test] +fn the_library_has_no_way_to_attach_elsewhere() { + fn shipped_files(dir: &Path, out: &mut Vec) { + for e in std::fs::read_dir(dir).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + shipped_files(&p, out); + } else if p.extension().is_some_and(|x| x == "rs") { + out.push(p); + } + } + } + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let mut files = Vec::new(); + shipped_files(&src, &mut files); + assert!(!files.is_empty()); + let mut launch_src = None; + for f in files { + let text = std::fs::read_to_string(&f).unwrap(); + let shipped = text.split("#[cfg(test)]").next().unwrap(); + for line in shipped.lines() { + for lit in ["ws://", "http://"] { + assert!( + !line.contains(lit) || line.contains("127.0.0.1"), + "{}: a {lit} literal that is not the loopback CDP URL: {line}", + f.display() + ); + } + } + if let Some(at) = shipped.find("pub struct Launch<") { + let rest = &shipped[at..]; + let open = rest.find('{').unwrap(); + let close = rest.find('}').unwrap(); + launch_src = Some(rest[open + 1..close].to_owned()); + } + } + let body = launch_src.expect("Launch struct not found"); + let mut names: Vec = body + .split(',') + .map(str::trim) + .filter(|f| !f.is_empty()) + .map(|f| { + let f = f.strip_prefix("pub ").unwrap_or(f); + f.split(':').next().unwrap().trim().to_owned() + }) + .collect(); + names.sort(); + assert_eq!(names, ["browser", "first_url", "headless", "profile"]); +} diff --git a/crates/kharon-teach/tests/support/mod.rs b/crates/kharon-teach/tests/support/mod.rs new file mode 100644 index 0000000..e36ff4e --- /dev/null +++ b/crates/kharon-teach/tests/support/mod.rs @@ -0,0 +1,33 @@ +// Shared by several test files; each uses only some of these items. +#![allow(dead_code)] + +use std::path::PathBuf; + +pub fn chromium_bin() -> Option { + if let Ok(p) = std::env::var("KHARON_CHROMIUM") { + return Some(PathBuf::from(p)); + } + [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/local/bin/google-chrome", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + ] + .into_iter() + .map(PathBuf::from) + .find(|p| p.exists()) +} + +/// The Chromium binary, or `None` to skip (unless a run requires it). +/// Same search as `chromium_bin` in `crates/kharond/tests/req_arch_003_org_chromium.rs`. +pub fn chromium_or_skip(test: &str) -> Option { + let bin = chromium_bin(); + if bin.is_none() { + if std::env::var_os("KHARON_REQUIRE_CHROMIUM").is_some() { + panic!("KHARON_REQUIRE_CHROMIUM is set but no Chromium binary was found"); + } + eprintln!("SKIP {test}: no Chromium binary"); + } + bin +} From b5e63895ceee86d866d7359f3597457a99209189 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:39:10 -0500 Subject: [PATCH 17/36] fix(teach): a profile names its owner so a sweep never takes a live one (#361) Profile::create builds the directory under a name the sweep never matches, writes kharon-teach.owner (this pid), then renames it into place. sweep_stale skips a directory whose owner file names a live pid, so sibling tests and concurrent recorders never sweep each other's fresh profiles. A missing, symlinked, oversized or unparseable owner file counts as no live owner. Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-teach/src/browser.rs | 119 +++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 5 deletions(-) diff --git a/crates/kharon-teach/src/browser.rs b/crates/kharon-teach/src/browser.rs index 215a698..61cb21a 100644 --- a/crates/kharon-teach/src/browser.rs +++ b/crates/kharon-teach/src/browser.rs @@ -17,6 +17,8 @@ pub enum BrowserError { } pub const PROFILE_PREFIX: &str = "kharon-teach-"; +/// Written into every profile: the pid of the recorder that owns it. +pub const OWNER_FILE: &str = "kharon-teach.owner"; const SEARCH_ORDER: [&str; 6] = [ "chromium", @@ -100,10 +102,16 @@ fn private_dir(path: &Path) -> std::io::Result<()> { } impl Profile { - /// `/kharon-teach-`, created atomically with mode 0700. + /// `/kharon-teach-`, mode 0700, holding an owner file with + /// this process's pid. It is built under a name the sweep never matches + /// and renamed into place, so a sweep cannot see it half-made. A crash + /// between the mkdir and the rename leaves a `.kharon-teach-new-*` + /// directory that nothing sweeps; the window is microseconds. pub fn create() -> std::io::Result { use rand::Rng; use rand::distributions::Alphanumeric; + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; let temp = std::env::temp_dir(); for _ in 0..16 { let suffix: String = rand::thread_rng() @@ -111,12 +119,41 @@ impl Profile { .take(20) .map(char::from) .collect(); - let path = temp.join(format!("{PROFILE_PREFIX}{suffix}")); - match private_dir(&path) { - Ok(()) => return Ok(Profile { path }), + let staging = temp.join(format!(".{PROFILE_PREFIX}new-{suffix}")); + match private_dir(&staging) { + Ok(()) => {} Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, Err(e) => return Err(e), } + let made = (|| { + let mut f = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(staging.join(OWNER_FILE))?; + f.write_all(std::process::id().to_string().as_bytes()) + })(); + if let Err(e) = made { + let _ = std::fs::remove_dir_all(&staging); + return Err(e); + } + let path = temp.join(format!("{PROFILE_PREFIX}{suffix}")); + // rename onto an existing non-empty directory fails; an existing + // empty one would be replaced, so check first. + if std::fs::symlink_metadata(&path).is_ok() { + let _ = std::fs::remove_dir_all(&staging); + continue; + } + match std::fs::rename(&staging, &path) { + Ok(()) => return Ok(Profile { path }), + Err(e) => { + let _ = std::fs::remove_dir_all(&staging); + if path.exists() { + continue; + } + return Err(e); + } + } } Err(std::io::Error::new( std::io::ErrorKind::AlreadyExists, @@ -153,7 +190,11 @@ impl Profile { let Ok(meta) = std::fs::symlink_metadata(&path) else { continue; }; - if !meta.file_type().is_dir() || meta.uid() != me || browser_is_alive(&path) { + if !meta.file_type().is_dir() + || meta.uid() != me + || owner_is_alive(&path) + || browser_is_alive(&path) + { continue; } if remove_tree(&path).is_ok() { @@ -185,6 +226,22 @@ fn remove_tree(path: &Path) -> std::io::Result<()> { last } +/// The owner file names a live pid. A missing, symlinked, non-regular, +/// over-32-byte or unparseable owner file counts as no live owner. +fn owner_is_alive(dir: &Path) -> bool { + let file = dir.join(OWNER_FILE); + let Ok(meta) = std::fs::symlink_metadata(&file) else { + return false; + }; + if !meta.file_type().is_file() || meta.len() > 32 { + return false; + } + std::fs::read_to_string(&file) + .ok() + .and_then(|t| t.trim().parse::().ok()) + .is_some_and(|pid| Path::new(&format!("/proc/{pid}")).exists()) +} + /// Chromium's `SingletonLock` is a symlink whose target text is /// `-`; the profile is live when that pid exists. fn browser_is_alive(dir: &Path) -> bool { @@ -342,6 +399,10 @@ mod tests { .starts_with(PROFILE_PREFIX) ); assert_eq!(std::fs::metadata(&path).unwrap().mode() & 0o777, 0o700); + assert_eq!( + std::fs::read_to_string(path.join(OWNER_FILE)).unwrap(), + std::process::id().to_string() + ); p.remove().unwrap(); assert!(!path.exists()); } @@ -403,6 +464,54 @@ mod tests { std::fs::remove_dir_all(&d).unwrap(); } + fn dead_pid() -> u32 { + let mut c = std::process::Command::new("true").spawn().unwrap(); + let pid = c.id(); + c.wait().unwrap(); + pid + } + + fn dir_with_owner(tag: &str, owner: &str) -> PathBuf { + let d = std::env::temp_dir().join(unique(tag)); + std::fs::create_dir(&d).unwrap(); + std::fs::write(d.join(OWNER_FILE), owner).unwrap(); + d + } + + #[test] + fn sweep_skips_a_profile_owned_by_this_process() { + let d = dir_with_owner("mine", &std::process::id().to_string()); + Profile::sweep_stale().unwrap(); + assert!(d.exists()); + std::fs::remove_dir_all(&d).unwrap(); + } + + #[test] + fn sweep_removes_a_profile_owned_by_a_dead_process() { + let d = dir_with_owner("dead", &dead_pid().to_string()); + assert!(Profile::sweep_stale().unwrap() >= 1); + assert!(!d.exists()); + } + + #[test] + fn sweep_treats_an_owner_symlink_as_no_owner() { + let d = std::env::temp_dir().join(unique("ownlink")); + std::fs::create_dir(&d).unwrap(); + let real = d.join("real"); + std::fs::write(&real, std::process::id().to_string()).unwrap(); + std::os::unix::fs::symlink(&real, d.join(OWNER_FILE)).unwrap(); + assert!(Profile::sweep_stale().unwrap() >= 1); + assert!(!d.exists()); + } + + #[test] + fn sweep_treats_a_junk_or_oversized_owner_as_no_owner() { + let a = dir_with_owner("junk", "not a pid"); + let b = dir_with_owner("big", &format!("{}{}", std::process::id(), " ".repeat(40))); + assert!(Profile::sweep_stale().unwrap() >= 2); + assert!(!a.exists() && !b.exists()); + } + #[test] fn no_browser_on_an_empty_path_is_not_found() { assert!(matches!( From ebf70f1ccd8cfea8b7bc7d670c173377e84f658e Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:43:34 -0500 Subject: [PATCH 18/36] feat(teach): a CDP client that subscribes to events (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- Cargo.lock | 2 + crates/kharon-teach/Cargo.toml | 6 +- crates/kharon-teach/src/cdp.rs | 645 ++++++++++++++++++++++++++++++ crates/kharon-teach/src/events.rs | 532 ++++++++++++++++++++++++ crates/kharon-teach/src/lib.rs | 2 + 5 files changed, 1185 insertions(+), 2 deletions(-) create mode 100644 crates/kharon-teach/src/cdp.rs create mode 100644 crates/kharon-teach/src/events.rs diff --git a/Cargo.lock b/Cargo.lock index fa07ed7..6884278 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1084,6 +1084,7 @@ name = "kharon-teach" version = "0.0.0" dependencies = [ "base64ct", + "futures-util", "hex", "kharon-recording", "md-5", @@ -1097,6 +1098,7 @@ dependencies = [ "sha2", "thiserror 2.0.21", "tokio", + "tokio-tungstenite", "url", "zeroize", ] diff --git a/crates/kharon-teach/Cargo.toml b/crates/kharon-teach/Cargo.toml index 7dcdc9e..3ea1bcb 100644 --- a/crates/kharon-teach/Cargo.toml +++ b/crates/kharon-teach/Cargo.toml @@ -17,7 +17,7 @@ path = "src/main.rs" [dependencies] kharon-recording = { path = "../kharon-recording" } -zeroize = "1" +zeroize = { version = "1", features = ["serde"] } sha1 = "0.10" sha2 = "0.10" md-5 = "0.10" @@ -26,9 +26,11 @@ hex = "0.4" percent-encoding = "2" memchr = "2" url = "2" -serde_json = "1" +serde_json = { version = "1", features = ["raw_value"] } serde = { version = "1", features = ["derive"] } thiserror = "2" rand = "0.8" tokio = { version = "1", features = ["fs", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time", "io-std", "io-util"] } publicsuffix = "2.3" +tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "handshake"] } +futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } diff --git a/crates/kharon-teach/src/cdp.rs b/crates/kharon-teach/src/cdp.rs new file mode 100644 index 0000000..ef9a67b --- /dev/null +++ b/crates/kharon-teach/src/cdp.rs @@ -0,0 +1,645 @@ +//! A small Chrome DevTools Protocol client over a loopback WebSocket that makes +//! calls and receives events. +//! +//! Everything arriving on this socket may contain what a person typed into a +//! login page. Nothing from a frame reaches an error message, a log line or a +//! `Debug` output: errors are built from fixed words and numbers only. + +use crate::browser::DevTools; +use futures_util::{SinkExt, StreamExt}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; +use tokio::net::TcpStream; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; +use tokio::sync::oneshot; +use tokio_tungstenite::tungstenite::protocol::WebSocketConfig; +use tokio_tungstenite::tungstenite::{self, Message}; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async_with_config}; + +const MAX_REMOTE_MESSAGE_CHARS: usize = 200; +/// CDP frames that carry request bodies can be large. +const MAX_FRAME_BYTES: usize = 64 * 1024 * 1024; + +#[derive(Debug, thiserror::Error)] +pub enum CdpError { + #[error("cdp connection closed")] + Closed, + #[error("cdp error {code}: {message}")] + Remote { code: i64, message: String }, + #[error("cdp transport: {0}")] + Transport(String), + #[error("cdp reply did not parse ({category} at line {line} column {column})")] + Parse { + category: &'static str, + line: usize, + column: usize, + }, +} + +impl From for CdpError { + /// Keeps the error's category and position only: its message can quote + /// the offending value. + fn from(e: serde_json::Error) -> Self { + use serde_json::error::Category; + let category = match e.classify() { + Category::Io => "io", + Category::Syntax => "syntax", + Category::Data => "data", + Category::Eof => "eof", + }; + CdpError::Parse { + category, + line: e.line(), + column: e.column(), + } + } +} + +/// Parse browser-derived JSON without ever letting the serde error text (which +/// can quote the offending value) out. +pub fn parse_quiet(raw: &str) -> Result { + serde_json::from_str(raw).map_err(CdpError::from) +} + +/// An event pushed by the browser. No `Debug`: `params` can hold page data. +pub struct Event { + pub method: String, + pub session_id: Option, + pub params: Box, +} + +/// What we send. Typed, never `json!`. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct Outgoing<'a, P: Serialize> { + id: u64, + method: &'a str, + params: &'a P, + #[serde(skip_serializing_if = "Option::is_none")] + session_id: Option<&'a str>, +} + +/// Anything the browser sends: a reply (`id`) or an event (`method`). +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Incoming { + #[serde(default)] + id: Option, + #[serde(default)] + method: Option, + #[serde(default)] + session_id: Option, + #[serde(default)] + params: Option>, + #[serde(default)] + result: Option>, + #[serde(default)] + error: Option>, +} + +#[derive(Deserialize)] +struct WireError { + #[serde(default)] + code: i64, + #[serde(default)] + message: String, +} + +type Reply = Result, CdpError>; + +#[derive(Default)] +struct State { + pending: HashMap>, + closed: bool, +} + +struct Shared { + state: Mutex, +} + +impl Shared { + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Mark closed and drop every waiter: their receivers see `Closed`. + fn close(&self) { + let mut st = self.lock(); + st.closed = true; + st.pending.clear(); + } +} + +struct Inner { + shared: Arc, + next_id: AtomicU64, + /// When the last clone drops, this closes, the task ends and the socket closes. + out: UnboundedSender, +} + +#[derive(Clone)] +pub struct Cdp { + inner: Arc, +} + +fn transport(e: &tungstenite::Error) -> CdpError { + use tungstenite::Error as E; + let words = match e { + E::Io(io) if io.kind() == std::io::ErrorKind::ConnectionRefused => "connection refused", + E::Io(_) => "io error", + E::Url(_) => "bad url", + E::Http(_) | E::HttpFormat(_) => "handshake failed", + E::ConnectionClosed | E::AlreadyClosed => "connection closed", + _ => "protocol error", + }; + CdpError::Transport(words.to_owned()) +} + +impl Cdp { + /// Dials ws://127.0.0.1:. The host is a constant. + pub async fn connect(dev: &DevTools) -> Result<(Cdp, UnboundedReceiver), CdpError> { + if !dev.ws_path.starts_with("/devtools/") + || dev.ws_path.chars().any(|c| c.is_control() || c == ' ') + { + return Err(CdpError::Transport("bad websocket path".to_owned())); + } + let url = format!("ws://127.0.0.1:{}{}", dev.port, dev.ws_path); + let config = WebSocketConfig { + max_message_size: Some(MAX_FRAME_BYTES), + max_frame_size: Some(MAX_FRAME_BYTES), + ..WebSocketConfig::default() + }; + let (ws, _resp) = connect_async_with_config(url, Some(config), false) + .await + .map_err(|e| transport(&e))?; + + let shared = Arc::new(Shared { + state: Mutex::new(State::default()), + }); + let (out, out_rx) = unbounded_channel(); + let (ev_tx, ev_rx) = unbounded_channel(); + tokio::spawn(run(ws, out_rx, Arc::clone(&shared), ev_tx)); + let cdp = Cdp { + inner: Arc::new(Inner { + shared, + next_id: AtomicU64::new(1), + out, + }), + }; + Ok((cdp, ev_rx)) + } + + pub async fn call( + &self, + method: &str, + params: &P, + session_id: Option<&str>, + ) -> Result { + let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed); + let frame = serde_json::to_string(&Outgoing { + id, + method, + params, + session_id, + }) + // The serde message could quote a value; say only that it failed. + .map_err(|_| CdpError::Transport("could not encode call".to_owned()))?; + + let (tx, rx) = oneshot::channel(); + { + let mut st = self.inner.shared.lock(); + if st.closed { + return Err(CdpError::Closed); + } + st.pending.insert(id, tx); + } + if self.inner.out.send(frame).is_err() { + self.inner.shared.lock().pending.remove(&id); + return Err(CdpError::Closed); + } + let raw = rx.await.map_err(|_| CdpError::Closed)??; + parse_quiet(raw.get()) + } +} + +/// The one task: reads frames, routes them, and writes queued calls. +async fn run( + mut ws: WebSocketStream>, + mut out: UnboundedReceiver, + shared: Arc, + events: UnboundedSender, +) { + loop { + tokio::select! { + msg = ws.next() => match msg { + Some(Ok(Message::Text(t))) => route(&t, &shared, &events), + Some(Ok(Message::Close(_))) | Some(Err(_)) | None => break, + // Binary, ping, pong: dropped. + Some(Ok(_)) => {} + }, + queued = out.recv() => match queued { + Some(text) => { + if ws.send(Message::Text(text)).await.is_err() { + break; + } + } + None => break, + }, + } + } + shared.close(); + let _ = tokio::time::timeout(Duration::from_millis(500), ws.close(None)).await; +} + +/// A frame that does not parse is dropped silently (never logged). +fn route(text: &str, shared: &Shared, events: &UnboundedSender) { + let Ok(m) = serde_json::from_str::(text) else { + return; + }; + if let Some(id) = m.id { + let Some(waiter) = shared.lock().pending.remove(&id) else { + return; // matches no pending call + }; + let reply = match (m.error, m.result) { + (Some(raw), _) => { + let w = parse_quiet::(raw.get()).unwrap_or(WireError { + code: 0, + message: String::new(), + }); + Err(CdpError::Remote { + code: w.code, + message: w.message.chars().take(MAX_REMOTE_MESSAGE_CHARS).collect(), + }) + } + (None, Some(result)) => Ok(result), + (None, None) => RawValue::from_string("{}".to_owned()) + .map_err(|_| CdpError::Transport("protocol error".to_owned())), + }; + let _ = waiter.send(reply); + } else if let Some(method) = m.method { + let params = match m.params { + Some(p) => p, + None => match RawValue::from_string("{}".to_owned()) { + Ok(p) => p, + Err(_) => return, + }, + }; + let _ = events.send(Event { + method, + session_id: m.session_id, + params, + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::{Empty, FrameNavigated}; + use futures_util::{SinkExt, StreamExt}; + use serde::Deserialize; + use std::future::Future; + use std::time::Duration; + use tokio::net::{TcpListener, TcpStream}; + use tokio::task::JoinHandle; + use tokio::time::timeout; + use tokio_tungstenite::{WebSocketStream, accept_async, tungstenite::Message}; + + type Ws = WebSocketStream; + const T: Duration = Duration::from_secs(5); + + /// Serve one connection with `f`; returns the DevTools to dial and the + /// server task's handle. + async fn serve(f: F) -> (DevTools, JoinHandle) + where + F: FnOnce(Ws) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + O: Send + 'static, + { + let l = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = l.local_addr().unwrap().port(); + let h = tokio::spawn(async move { + let (s, _) = l.accept().await.unwrap(); + f(accept_async(s).await.unwrap()).await + }); + ( + DevTools { + port, + ws_path: "/devtools/browser/test".to_owned(), + }, + h, + ) + } + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct Sent { + id: u64, + method: String, + #[serde(default)] + session_id: Option, + } + + async fn next_text(ws: &mut Ws) -> String { + loop { + match timeout(T, ws.next()).await.unwrap() { + Some(Ok(Message::Text(t))) => return t, + Some(Ok(_)) => continue, + other => panic!("server expected a text frame, got end: {}", other.is_none()), + } + } + } + + #[derive(Deserialize)] + struct Reply { + v: String, + } + + #[tokio::test] + async fn a_call_gets_its_own_reply() { + let (dev, srv) = serve(|mut ws| async move { + let a: Sent = serde_json::from_str(&next_text(&mut ws).await).unwrap(); + let b: Sent = serde_json::from_str(&next_text(&mut ws).await).unwrap(); + for s in [b, a] { + let reply = format!(r#"{{"id":{},"result":{{"v":"{}"}}}}"#, s.id, s.method); + ws.send(Message::Text(reply)).await.unwrap(); + } + ws + }) + .await; + let (cdp, _rx) = Cdp::connect(&dev).await.unwrap(); + let (ra, rb) = timeout(T, async { + tokio::join!( + cdp.call::<_, Reply>("A.first", &Empty {}, None), + cdp.call::<_, Reply>("B.second", &Empty {}, None) + ) + }) + .await + .unwrap(); + assert_eq!(ra.unwrap().v, "A.first"); + assert_eq!(rb.unwrap().v, "B.second"); + drop(srv); + } + + #[tokio::test] + async fn a_remote_error_is_returned() { + let (dev, _srv) = serve(|mut ws| async move { + let s: Sent = serde_json::from_str(&next_text(&mut ws).await).unwrap(); + let long = "n".repeat(500); + let r = format!( + r#"{{"id":{},"error":{{"code":-32000,"message":"{long}"}}}}"#, + s.id + ); + ws.send(Message::Text(r)).await.unwrap(); + ws + }) + .await; + let (cdp, _rx) = Cdp::connect(&dev).await.unwrap(); + let r = timeout(T, cdp.call::<_, Empty2>("X.y", &Empty {}, None)) + .await + .unwrap(); + match r { + Err(CdpError::Remote { code, message }) => { + assert_eq!(code, -32000); + assert_eq!(message.chars().count(), 200); + } + _ => panic!("expected Remote"), + } + } + + #[derive(Deserialize)] + struct Empty2 {} + + #[tokio::test] + async fn a_short_remote_message_is_kept_whole() { + let (dev, _srv) = serve(|mut ws| async move { + let s: Sent = serde_json::from_str(&next_text(&mut ws).await).unwrap(); + let r = format!( + r#"{{"id":{},"error":{{"code":-32601,"message":"nope"}}}}"#, + s.id + ); + ws.send(Message::Text(r)).await.unwrap(); + ws + }) + .await; + let (cdp, _rx) = Cdp::connect(&dev).await.unwrap(); + let r = timeout(T, cdp.call::<_, Empty2>("X.y", &Empty {}, None)) + .await + .unwrap(); + assert!( + matches!(r, Err(CdpError::Remote { code: -32601, ref message }) if message == "nope") + ); + } + + #[tokio::test] + async fn events_are_delivered_with_their_session() { + let (dev, _srv) = serve(|mut ws| async move { + let ev = r#"{"method":"Page.frameNavigated","sessionId":"S1","params":{"frame":{"id":"F","url":"https://a.test/","extra":1}}}"#; + ws.send(Message::Text(ev.to_owned())).await.unwrap(); + let ev2 = r#"{"method":"Target.targetDestroyed","params":{"targetId":"T"}}"#; + ws.send(Message::Text(ev2.to_owned())).await.unwrap(); + ws + }) + .await; + let (_cdp, mut rx) = Cdp::connect(&dev).await.unwrap(); + let e = timeout(T, rx.recv()).await.unwrap().unwrap(); + assert_eq!(e.method, "Page.frameNavigated"); + assert_eq!(e.session_id.as_deref(), Some("S1")); + let f: FrameNavigated = parse_quiet(e.params.get()).unwrap(); + assert_eq!( + (f.frame.id.as_str(), f.frame.url.as_str()), + ("F", "https://a.test/") + ); + assert!(f.frame.parent_id.is_none()); + let e = timeout(T, rx.recv()).await.unwrap().unwrap(); + assert_eq!(e.method, "Target.targetDestroyed"); + assert!(e.session_id.is_none()); + } + + #[tokio::test] + async fn session_id_is_sent_at_top_level() { + let (dev, srv) = serve(|mut ws| async move { + let mut seen = Vec::new(); + for _ in 0..2 { + let t = next_text(&mut ws).await; + let s: Sent = serde_json::from_str(&t).unwrap(); + ws.send(Message::Text(format!(r#"{{"id":{},"result":{{}}}}"#, s.id))) + .await + .unwrap(); + seen.push(t); + } + (seen, ws) + }) + .await; + let (cdp, _rx) = Cdp::connect(&dev).await.unwrap(); + timeout( + T, + cdp.call::<_, Empty2>("Runtime.enable", &Empty {}, Some("S1")), + ) + .await + .unwrap() + .unwrap(); + timeout( + T, + cdp.call::<_, Empty2>("Browser.getVersion", &Empty {}, None), + ) + .await + .unwrap() + .unwrap(); + let (seen, _ws) = timeout(T, srv).await.unwrap().unwrap(); + let first: Sent = serde_json::from_str(&seen[0]).unwrap(); + assert_eq!(first.id, 1); + assert_eq!(first.session_id.as_deref(), Some("S1")); + assert!(seen[0].contains(r#""params":{}"#)); + let second: Sent = serde_json::from_str(&seen[1]).unwrap(); + assert_eq!(second.id, 2); + assert!(second.session_id.is_none()); + assert!(!seen[1].contains("sessionId")); + } + + #[tokio::test] + async fn calls_fail_after_the_socket_closes() { + let (dev, _srv) = serve(|mut ws| async move { + // Read the first call, then close without answering. + let _ = next_text(&mut ws).await; + let _ = ws.close(None).await; + }) + .await; + let (cdp, mut rx) = Cdp::connect(&dev).await.unwrap(); + // The pending call fails. + let r = timeout(T, cdp.call::<_, Empty2>("A.a", &Empty {}, None)) + .await + .unwrap(); + assert!(matches!(r, Err(CdpError::Closed))); + // A later call fails too, without hanging. + let r = timeout(T, cdp.call::<_, Empty2>("A.b", &Empty {}, None)) + .await + .unwrap(); + assert!(matches!(r, Err(CdpError::Closed))); + // The event channel ends. + assert!(timeout(T, rx.recv()).await.unwrap().is_none()); + } + + #[tokio::test] + async fn stray_replies_and_junk_frames_are_dropped_silently() { + let (dev, _srv) = serve(|mut ws| async move { + let s: Sent = serde_json::from_str(&next_text(&mut ws).await).unwrap(); + ws.send(Message::Text( + r#"{"id":999,"result":{"v":"stray"}}"#.to_owned(), + )) + .await + .unwrap(); + ws.send(Message::Text("not json at all {{".to_owned())) + .await + .unwrap(); + ws.send(Message::Binary(vec![1, 2, 3])).await.unwrap(); + ws.send(Message::Text(format!( + r#"{{"id":{},"result":{{"v":"ok"}}}}"#, + s.id + ))) + .await + .unwrap(); + ws + }) + .await; + let (cdp, mut rx) = Cdp::connect(&dev).await.unwrap(); + let r = timeout(T, cdp.call::<_, Reply>("A.a", &Empty {}, None)) + .await + .unwrap() + .unwrap(); + assert_eq!(r.v, "ok"); + assert!(rx.try_recv().is_err(), "junk must not surface as an event"); + } + + #[tokio::test] + async fn dropping_the_last_clone_closes_the_socket() { + let (dev, srv) = serve(|mut ws| async move { + loop { + match ws.next().await { + None | Some(Err(_)) | Some(Ok(Message::Close(_))) => return, + Some(Ok(_)) => {} + } + } + }) + .await; + let (cdp, rx) = Cdp::connect(&dev).await.unwrap(); + let other = cdp.clone(); + drop(cdp); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!srv.is_finished(), "one clone is still alive"); + drop(other); + drop(rx); + timeout(T, srv).await.unwrap().unwrap(); + } + + #[tokio::test] + async fn a_bad_ws_path_is_refused() { + for bad in [ + "", + "/json/version", + "devtools/x", + "/devtools/\r\nHost: evil", + "//evil/devtools/x", + ] { + let dev = DevTools { + port: 1, + ws_path: bad.to_owned(), + }; + match Cdp::connect(&dev).await { + Err(CdpError::Transport(_)) => {} + _ => panic!("expected Transport for {bad:?}"), + } + } + } + + #[tokio::test] + async fn a_refused_connection_is_a_transport_error_in_fixed_words() { + let l = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = l.local_addr().unwrap().port(); + drop(l); + let dev = DevTools { + port, + ws_path: "/devtools/browser/x".to_owned(), + }; + match Cdp::connect(&dev).await { + Err(CdpError::Transport(w)) => assert_eq!(w, "connection refused"), + _ => panic!("expected Transport"), + } + } + + #[test] + fn a_parse_error_never_carries_the_value() { + #[derive(Debug, Deserialize)] + struct Wants { + #[allow(dead_code)] + value: u32, + } + let e = parse_quiet::(r#"{"value": "Tr0ub4dor-9x"}"#) + .map(|_| ()) + .unwrap_err(); + assert!(matches!( + e, + CdpError::Parse { + category: "data", + .. + } + )); + assert!(!e.to_string().contains("Tr0ub4dor-9x")); + assert!(!format!("{e:?}").contains("Tr0ub4dor-9x")); + let e = parse_quiet::(r#"{"value": 1"#) + .map(|_| ()) + .unwrap_err(); + assert!(matches!( + e, + CdpError::Parse { + category: "eof", + .. + } + )); + } +} diff --git a/crates/kharon-teach/src/events.rs b/crates/kharon-teach/src/events.rs new file mode 100644 index 0000000..c5b5eff --- /dev/null +++ b/crates/kharon-teach/src/events.rs @@ -0,0 +1,532 @@ +//! Typed CDP messages: the event parameters the recorder reads, the parameters +//! it sends, and the report the page script passes through the binding. +//! +//! Anything that can hold page data (what a person typed, a request body, a +//! header value) is held in `Zeroizing` and has no `Debug` impl: a derive would +//! print the content. + +use serde::de::{IgnoredAny, MapAccess, Visitor}; +use serde::{Deserialize, Deserializer, Serialize}; +use std::collections::HashMap; +use std::fmt; +use zeroize::Zeroizing; + +// ---- events the recorder reads ---- + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AttachedToTarget { + pub session_id: String, + pub target_info: TargetInfo, + pub waiting_for_debugger: bool, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TargetInfo { + pub target_id: String, + #[serde(rename = "type")] + pub kind: String, + pub url: String, + #[serde(default)] + pub opener_id: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TargetDestroyed { + pub target_id: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetachedFromTarget { + pub session_id: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FrameNavigated { + pub frame: Frame, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Frame { + pub id: String, + #[serde(default)] + pub parent_id: Option, + pub url: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BindingCalled { + pub name: String, + pub payload: Zeroizing, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RequestWillBeSent { + pub request_id: String, + pub loader_id: String, + #[serde(default)] + pub frame_id: Option, + #[serde(rename = "documentURL")] + pub document_url: String, + pub request: Request, + #[serde(rename = "type", default)] + pub resource: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Request { + pub url: String, + pub method: String, + #[serde(default)] + pub headers: Headers, + #[serde(default)] + pub post_data: Option>, + #[serde(default)] + pub has_post_data: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResponseReceived { + pub request_id: String, + pub response: Response, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Response { + #[serde(deserialize_with = "lenient_status")] + pub status: u16, + #[serde(default)] + pub headers: Headers, +} + +/// A status outside `u16` (or fractional) must not fail the whole event. +fn lenient_status<'de, D: Deserializer<'de>>(d: D) -> Result { + let v = f64::deserialize(d)?; + // `as` saturates and maps NaN to 0. + Ok(v.clamp(0.0, f64::from(u16::MAX)) as u16) +} + +// ---- results ---- + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VersionInfo { + pub product: String, + pub user_agent: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LayoutMetrics { + pub css_visual_viewport: VisualViewport, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VisualViewport { + pub client_width: f64, + pub client_height: f64, +} + +/// The result of `Network.getRequestPostData`. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PostData { + pub post_data: Zeroizing, +} + +// ---- headers ---- + +/// Header map. Values are `Zeroizing` and there is no `Debug`. +#[derive(Default)] +pub struct Headers(HashMap>); + +impl Headers { + fn get(&self, name: &str) -> Option<&str> { + self.0 + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case(name)) + .map(|(_, v)| v.as_str()) + } + + pub fn content_type(&self) -> Option<&str> { + self.get("content-type") + } + + pub fn authorization(&self) -> Option<&str> { + self.get("authorization") + } + + pub fn has_set_cookie(&self) -> bool { + self.0.keys().any(|k| k.eq_ignore_ascii_case("set-cookie")) + } +} + +/// A header value that is a string, or anything else (skipped). +struct MaybeString(Option>); + +impl<'de> Deserialize<'de> for MaybeString { + fn deserialize>(d: D) -> Result { + struct V; + impl<'de> Visitor<'de> for V { + type Value = MaybeString; + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("a header value") + } + fn visit_str(self, v: &str) -> Result { + Ok(MaybeString(Some(Zeroizing::new(v.to_owned())))) + } + fn visit_string(self, v: String) -> Result { + Ok(MaybeString(Some(Zeroizing::new(v)))) + } + fn visit_bool(self, _: bool) -> Result { + Ok(MaybeString(None)) + } + fn visit_i64(self, _: i64) -> Result { + Ok(MaybeString(None)) + } + fn visit_u64(self, _: u64) -> Result { + Ok(MaybeString(None)) + } + fn visit_f64(self, _: f64) -> Result { + Ok(MaybeString(None)) + } + fn visit_unit(self) -> Result { + Ok(MaybeString(None)) + } + fn visit_none(self) -> Result { + Ok(MaybeString(None)) + } + fn visit_seq>( + self, + mut seq: A, + ) -> Result { + while seq.next_element::()?.is_some() {} + Ok(MaybeString(None)) + } + fn visit_map>(self, mut map: A) -> Result { + while map.next_entry::()?.is_some() {} + Ok(MaybeString(None)) + } + } + d.deserialize_any(V) + } +} + +impl<'de> Deserialize<'de> for Headers { + fn deserialize>(d: D) -> Result { + struct V; + impl<'de> Visitor<'de> for V { + type Value = Headers; + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("a header object") + } + fn visit_map>(self, mut map: A) -> Result { + let mut out = HashMap::new(); + while let Some((k, v)) = map.next_entry::()? { + if let Some(v) = v.0 { + out.insert(k, v); + } + } + Ok(Headers(out)) + } + } + d.deserialize_map(V) + } +} + +// ---- parameters the recorder sends ---- + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SetAutoAttach { + pub auto_attach: bool, + pub wait_for_debugger_on_start: bool, + pub flatten: bool, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AddBinding<'a> { + pub name: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + pub execution_context_name: Option<&'a str>, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AddScriptToEvaluateOnNewDocument<'a> { + pub source: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + pub world_name: Option<&'a str>, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GetRequestPostData<'a> { + pub request_id: &'a str, +} + +/// Parameters for a call that takes none: serializes as `{}`. +#[derive(Serialize)] +pub struct Empty {} + +// ---- the page script's report ---- + +const MAX_FIELD_CHARS: usize = 4096; +const MAX_SELECTORS: usize = 16; + +#[derive(Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReportKind { + Input, + Change, + Click, + Submit, + EnterKey, +} + +#[derive(Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct ListenerReport { + pub kind: ReportKind, + pub document_id: String, + pub frame_url: String, + pub is_top: bool, + pub element: ReportElement, + #[serde(default)] + pub value: Option>, +} + +#[derive(Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct ReportElement { + pub element_id: u32, + pub tag: String, + #[serde(default)] + pub input_type: Option, + #[serde(default)] + pub id: Option, + #[serde(default)] + pub name: Option, + #[serde(default)] + pub autocomplete: Option, + #[serde(default)] + pub placeholder: Option, + #[serde(default)] + pub aria_label: Option, + #[serde(default)] + pub is_password_now: bool, + #[serde(default)] + pub form: Option, + #[serde(default)] + pub selectors: Vec, + #[serde(default)] + pub in_shadow_root: bool, +} + +#[derive(Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct ReportForm { + pub index: u32, + #[serde(default)] + pub action: Option, + pub method: String, +} + +fn short(s: &str) -> bool { + s.chars().take(MAX_FIELD_CHARS + 1).count() <= MAX_FIELD_CHARS +} + +fn short_opt(s: &Option) -> bool { + s.as_deref().is_none_or(short) +} + +impl ListenerReport { + /// False when a hostile page sent more than the recorder will hold: any + /// string over 4,096 characters, or more than 16 selectors. + pub fn within_limits(&self) -> bool { + let e = &self.element; + short(&self.document_id) + && short(&self.frame_url) + && self.value.as_ref().is_none_or(|v| short(v)) + && short(&e.tag) + && short_opt(&e.input_type) + && short_opt(&e.id) + && short_opt(&e.name) + && short_opt(&e.autocomplete) + && short_opt(&e.placeholder) + && short_opt(&e.aria_label) + && e.selectors.len() <= MAX_SELECTORS + && e.selectors.iter().all(|s| short(s)) + && e.form + .as_ref() + .is_none_or(|f| short_opt(&f.action) && short(&f.method)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cdp::parse_quiet; + + const REPORT: &str = r##"{ + "kind": "input", "document_id": "d1", "frame_url": "https://a.test/login", + "is_top": true, "value": "Tr0ub4dor-9x", + "element": { + "element_id": 3, "tag": "input", "input_type": "password", "id": "pw", + "name": "password", "autocomplete": "current-password", + "is_password_now": true, + "form": {"index": 0, "action": "/session", "method": "post"}, + "selectors": ["#pw", "input[name=password]"], "in_shadow_root": false + } + }"##; + + #[test] + fn listener_report_parses_and_value_is_zeroizing() { + let r: ListenerReport = parse_quiet(REPORT).unwrap(); + assert!(matches!(r.kind, ReportKind::Input)); + let v: &Zeroizing = r.value.as_ref().unwrap(); + assert_eq!(v.as_str(), "Tr0ub4dor-9x"); + assert_eq!(r.element.element_id, 3); + assert_eq!(r.element.input_type.as_deref(), Some("password")); + assert!(r.element.is_password_now); + let f = r.element.form.as_ref().unwrap(); + assert_eq!( + (f.index, f.action.as_deref(), f.method.as_str()), + (0, Some("/session"), "post") + ); + assert_eq!(r.element.selectors.len(), 2); + assert!(r.within_limits()); + } + + #[test] + fn a_sparse_report_parses() { + let raw = r#"{"kind":"enter_key","document_id":"d","frame_url":"u","is_top":false, + "element":{"element_id":1,"tag":"input"}}"#; + let r: ListenerReport = parse_quiet(raw).unwrap(); + assert!(matches!(r.kind, ReportKind::EnterKey)); + assert!(r.value.is_none() && r.element.selectors.is_empty() && r.element.form.is_none()); + assert!(!r.element.is_password_now && !r.element.in_shadow_root); + } + + #[test] + fn oversized_reports_are_out_of_limits() { + let mut r: ListenerReport = parse_quiet(REPORT).unwrap(); + r.value = Some(Zeroizing::new("x".repeat(4097))); + assert!(!r.within_limits()); + r.value = Some(Zeroizing::new("x".repeat(4096))); + assert!(r.within_limits()); + + let mut r: ListenerReport = parse_quiet(REPORT).unwrap(); + r.element.selectors = vec!["a".to_owned(); 17]; + assert!(!r.within_limits()); + r.element.selectors = vec!["a".to_owned(); 16]; + assert!(r.within_limits()); + + let mut r: ListenerReport = parse_quiet(REPORT).unwrap(); + r.element.name = Some("n".repeat(5000)); + assert!(!r.within_limits()); + + let mut r: ListenerReport = parse_quiet(REPORT).unwrap(); + r.frame_url = "u".repeat(4097); + assert!(!r.within_limits()); + + let mut r: ListenerReport = parse_quiet(REPORT).unwrap(); + r.element.form.as_mut().unwrap().action = Some("a".repeat(4097)); + assert!(!r.within_limits()); + } + + #[test] + fn headers_are_read_case_insensitively() { + let h: Headers = parse_quiet( + r#"{"Content-Type":"application/json","AUTHORIZATION":"Bearer x","Set-Cookie":"a=b"}"#, + ) + .unwrap(); + assert_eq!(h.content_type(), Some("application/json")); + assert_eq!(h.authorization(), Some("Bearer x")); + assert!(h.has_set_cookie()); + + let h: Headers = parse_quiet(r#"{"content-type":"text/html"}"#).unwrap(); + assert_eq!(h.content_type(), Some("text/html")); + assert_eq!(h.authorization(), None); + assert!(!h.has_set_cookie()); + } + + #[test] + fn a_non_string_header_is_skipped_not_fatal() { + let h: Headers = + parse_quiet(r#"{"x-n":5,"x-a":["a"],"x-o":{"k":1},"x-z":null,"content-type":"t"}"#) + .unwrap(); + assert_eq!(h.content_type(), Some("t")); + } + + #[test] + fn a_wild_status_does_not_fail_the_event() { + let r: ResponseReceived = + parse_quiet(r#"{"requestId":"1","response":{"status":99999,"headers":{}}}"#).unwrap(); + assert_eq!(r.response.status, u16::MAX); + let r: ResponseReceived = + parse_quiet(r#"{"requestId":"1","response":{"status":-3,"headers":{}}}"#).unwrap(); + assert_eq!(r.response.status, 0); + let r: ResponseReceived = parse_quiet( + r#"{"requestId":"1","response":{"status":302,"headers":{"Set-Cookie":"s"}}}"#, + ) + .unwrap(); + assert_eq!(r.response.status, 302); + assert!(r.response.headers.has_set_cookie()); + } + + #[test] + fn a_request_event_parses_with_unknown_fields_ignored() { + let raw = r#"{"requestId":"r","loaderId":"l","frameId":"f","documentURL":"https://a.test/", + "request":{"url":"https://a.test/s","method":"POST","headers":{"Content-Type":"x"}, + "postData":"user=a&pass=b","hasPostData":true,"initialPriority":"High"}, + "type":"Document","timestamp":1.5}"#; + let e: RequestWillBeSent = parse_quiet(raw).unwrap(); + assert_eq!(e.request.method, "POST"); + assert_eq!( + e.request.post_data.as_ref().map(|p| p.as_str()), + Some("user=a&pass=b") + ); + assert_eq!(e.resource.as_deref(), Some("Document")); + assert_eq!(e.request.headers.content_type(), Some("x")); + } + + #[test] + fn sent_params_serialize_in_camel_case_and_skip_none() { + let s = serde_json::to_string(&AddBinding { + name: "b", + execution_context_name: None, + }) + .unwrap(); + assert_eq!(s, r#"{"name":"b"}"#); + let s = serde_json::to_string(&SetAutoAttach { + auto_attach: true, + wait_for_debugger_on_start: true, + flatten: true, + }) + .unwrap(); + assert_eq!( + s, + r#"{"autoAttach":true,"waitForDebuggerOnStart":true,"flatten":true}"# + ); + assert_eq!(serde_json::to_string(&Empty {}).unwrap(), "{}"); + let s = serde_json::to_string(&AddScriptToEvaluateOnNewDocument { + source: "1", + world_name: Some("w"), + }) + .unwrap(); + assert_eq!(s, r#"{"source":"1","worldName":"w"}"#); + } +} diff --git a/crates/kharon-teach/src/lib.rs b/crates/kharon-teach/src/lib.rs index fe5f3de..f558891 100644 --- a/crates/kharon-teach/src/lib.rs +++ b/crates/kharon-teach/src/lib.rs @@ -3,6 +3,8 @@ pub mod body; pub mod browser; pub mod capture; +pub mod cdp; +pub mod events; pub mod marker; pub mod site; pub mod strip; From 0d3491db8caf1cad5cdec9d1556b72c6e1cab147 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:44:48 -0500 Subject: [PATCH 19/36] fix(teach): sweep tests use private directories; guard the first URL (#361) Profile::create_in and sweep_stale_in take the location, so each unit test builds and sweeps inside its own private directory and asserts exact counts; no unit test sweeps the shared temp dir any more. Browser::launch refuses a first URL that starts with '-' (it would be read as a switch) without echoing it, kills and reaps the child if try_wait errors, and passes --user-data-dir as an OsString. Tests added for the Preferences file, the BOM and an oversized port. Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-teach/src/browser.rs | 203 +++++++++++++++++++++-------- 1 file changed, 150 insertions(+), 53 deletions(-) diff --git a/crates/kharon-teach/src/browser.rs b/crates/kharon-teach/src/browser.rs index 61cb21a..00f46b5 100644 --- a/crates/kharon-teach/src/browser.rs +++ b/crates/kharon-teach/src/browser.rs @@ -108,11 +108,15 @@ impl Profile { /// between the mkdir and the rename leaves a `.kharon-teach-new-*` /// directory that nothing sweeps; the window is microseconds. pub fn create() -> std::io::Result { + Self::create_in(&std::env::temp_dir()) + } + + /// [`Profile::create`] in `temp`. + pub(crate) fn create_in(temp: &Path) -> std::io::Result { use rand::Rng; use rand::distributions::Alphanumeric; use std::io::Write; use std::os::unix::fs::OpenOptionsExt; - let temp = std::env::temp_dir(); for _ in 0..16 { let suffix: String = rand::thread_rng() .sample_iter(&Alphanumeric) @@ -173,11 +177,16 @@ impl Profile { /// Removes leftover PROFILE_PREFIX directories in the temp location owned by this user. Returns how many. pub fn sweep_stale() -> std::io::Result { + Self::sweep_stale_in(&std::env::temp_dir()) + } + + /// [`Profile::sweep_stale`] over `dir`. + pub(crate) fn sweep_stale_in(dir: &Path) -> std::io::Result { use std::os::unix::fs::MetadataExt; // /proc/self is owned by this process's effective uid. let me = std::fs::metadata("/proc/self")?.uid(); let mut n = 0; - for entry in std::fs::read_dir(std::env::temp_dir())? { + for entry in std::fs::read_dir(dir)? { let Ok(entry) = entry else { continue }; if !entry .file_name() @@ -211,6 +220,14 @@ impl Drop for Profile { } } +/// `/Default/Preferences`: password manager and autofill off. +fn write_preferences(profile: &Path) -> std::io::Result<()> { + let default = profile.join("Default"); + private_dir(&default)?; + let prefs = serde_json::to_string(&PREFERENCES).map_err(std::io::Error::other)?; + std::fs::write(default.join("Preferences"), prefs) +} + /// `remove_dir_all` does not follow symlinks. A just-killed browser's helper /// processes can still be writing, so retry a few times. fn remove_tree(path: &Path) -> std::io::Result<()> { @@ -282,15 +299,19 @@ impl Browser { pub async fn launch(l: Launch<'_>) -> Result<(Browser, DevTools), BrowserError> { use std::process::Stdio; let launch_err = |e: std::io::Error| BrowserError::Launch(e.to_string()); + if l.first_url.starts_with('-') { + // It would be read as a browser switch. Do not echo the value. + return Err(BrowserError::Launch( + "the first URL must not start with '-'".to_owned(), + )); + } let profile = l.profile.path(); - let default = profile.join("Default"); - private_dir(&default).map_err(launch_err)?; - let prefs = - serde_json::to_string(&PREFERENCES).map_err(|e| BrowserError::Launch(e.to_string()))?; - std::fs::write(default.join("Preferences"), prefs).map_err(launch_err)?; + write_preferences(profile).map_err(launch_err)?; + let mut user_data_dir = std::ffi::OsString::from("--user-data-dir="); + user_data_dir.push(profile.as_os_str()); let mut cmd = tokio::process::Command::new(l.browser); - cmd.arg(format!("--user-data-dir={}", profile.display())) + cmd.arg(user_data_dir) .arg("--remote-debugging-port=0") .arg("--no-first-run") .arg("--no-default-browser-check") @@ -311,8 +332,13 @@ impl Browser { let file = profile.join("DevToolsActivePort"); let deadline = tokio::time::Instant::now() + START_TIMEOUT; loop { - if let Some(status) = child.try_wait().map_err(launch_err)? { - return Err(BrowserError::Launch(status.to_string())); + match child.try_wait() { + Ok(Some(status)) => return Err(BrowserError::Launch(status.to_string())), + Ok(None) => {} + Err(e) => { + let _ = child.kill().await; + return Err(launch_err(e)); + } } if let Some(dt) = std::fs::read_to_string(&file) .ok() @@ -379,18 +405,42 @@ mod tests { use super::*; use std::os::unix::fs::{MetadataExt, PermissionsExt}; - fn unique(tag: &str) -> String { - format!( - "{PROFILE_PREFIX}{tag}-{}-{:08x}", - std::process::id(), - rand::random::() - ) + /// A private directory for one test, removed when dropped. Its name does + /// not start with the profile prefix, and nothing else sweeps it. + struct Sandbox(PathBuf); + + impl Sandbox { + fn new(tag: &str) -> Self { + let d = std::env::temp_dir().join(format!( + "kharon-sandbox-{tag}-{}-{:08x}", + std::process::id(), + rand::random::() + )); + std::fs::create_dir(&d).unwrap(); + Sandbox(d) + } + fn path(&self) -> &Path { + &self.0 + } + fn mkdir(&self, name: &str) -> PathBuf { + let d = self.0.join(name); + std::fs::create_dir(&d).unwrap(); + d + } + } + + impl Drop for Sandbox { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } } #[test] fn profile_is_private_and_removed() { - let p = Profile::create().unwrap(); + let sb = Sandbox::new("create"); + let p = Profile::create_in(sb.path()).unwrap(); let path = p.path().to_owned(); + assert_eq!(path.parent().unwrap(), sb.path()); assert!( path.file_name() .unwrap() @@ -403,13 +453,16 @@ mod tests { std::fs::read_to_string(path.join(OWNER_FILE)).unwrap(), std::process::id().to_string() ); + // No staging directory is left behind. + assert_eq!(std::fs::read_dir(sb.path()).unwrap().count(), 1); p.remove().unwrap(); assert!(!path.exists()); } #[test] fn profile_is_removed_on_drop() { - let p = Profile::create().unwrap(); + let sb = Sandbox::new("drop"); + let p = Profile::create_in(sb.path()).unwrap(); let path = p.path().to_owned(); std::fs::write(path.join("f"), b"x").unwrap(); drop(p); @@ -418,50 +471,47 @@ mod tests { #[test] fn sweep_removes_a_leftover_and_counts_it() { - let d = std::env::temp_dir().join(unique("stale")); - std::fs::create_dir(&d).unwrap(); + let sb = Sandbox::new("stale"); + let d = sb.mkdir("kharon-teach-stale"); std::fs::write(d.join("f"), b"x").unwrap(); - assert!(Profile::sweep_stale().unwrap() >= 1); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 1); assert!(!d.exists()); } #[test] fn sweep_leaves_other_directories() { - let d = std::env::temp_dir().join(format!("other-{}", unique("keep"))); - std::fs::create_dir(&d).unwrap(); - Profile::sweep_stale().unwrap(); - assert!(d.exists()); - std::fs::remove_dir(&d).unwrap(); + let sb = Sandbox::new("other"); + let d = sb.mkdir("other-keep"); + let staging = sb.mkdir(".kharon-teach-new-x"); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); + assert!(d.exists() && staging.exists()); } #[test] fn sweep_leaves_a_symlink_and_its_target() { - let target = std::env::temp_dir().join(format!("other-{}", unique("tgt"))); - std::fs::create_dir(&target).unwrap(); + let sb = Sandbox::new("link"); + let target = sb.mkdir("other-tgt"); std::fs::write(target.join("f"), b"x").unwrap(); - let link = std::env::temp_dir().join(unique("link")); + let link = sb.path().join("kharon-teach-link"); std::os::unix::fs::symlink(&target, &link).unwrap(); - Profile::sweep_stale().unwrap(); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); assert!(target.join("f").exists()); assert!(std::fs::symlink_metadata(&link).is_ok()); - std::fs::remove_file(&link).unwrap(); - std::fs::remove_dir_all(&target).unwrap(); } // An addition to the brief: a profile whose SingletonLock names a live // process is a running recorder's profile and must survive a sweep. #[test] fn sweep_skips_a_profile_whose_browser_is_alive() { - let d = std::env::temp_dir().join(unique("live")); - std::fs::create_dir(&d).unwrap(); + let sb = Sandbox::new("live"); + let d = sb.mkdir("kharon-teach-live"); std::os::unix::fs::symlink( format!("host-{}", std::process::id()), d.join("SingletonLock"), ) .unwrap(); - Profile::sweep_stale().unwrap(); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); assert!(d.exists()); - std::fs::remove_dir_all(&d).unwrap(); } fn dead_pid() -> u32 { @@ -471,47 +521,84 @@ mod tests { pid } - fn dir_with_owner(tag: &str, owner: &str) -> PathBuf { - let d = std::env::temp_dir().join(unique(tag)); - std::fs::create_dir(&d).unwrap(); + fn dir_with_owner(sb: &Sandbox, name: &str, owner: &str) -> PathBuf { + let d = sb.mkdir(name); std::fs::write(d.join(OWNER_FILE), owner).unwrap(); d } #[test] fn sweep_skips_a_profile_owned_by_this_process() { - let d = dir_with_owner("mine", &std::process::id().to_string()); - Profile::sweep_stale().unwrap(); + let sb = Sandbox::new("mine"); + let d = dir_with_owner(&sb, "kharon-teach-mine", &std::process::id().to_string()); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); assert!(d.exists()); - std::fs::remove_dir_all(&d).unwrap(); } #[test] fn sweep_removes_a_profile_owned_by_a_dead_process() { - let d = dir_with_owner("dead", &dead_pid().to_string()); - assert!(Profile::sweep_stale().unwrap() >= 1); + let sb = Sandbox::new("dead"); + let d = dir_with_owner(&sb, "kharon-teach-dead", &dead_pid().to_string()); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 1); assert!(!d.exists()); } #[test] fn sweep_treats_an_owner_symlink_as_no_owner() { - let d = std::env::temp_dir().join(unique("ownlink")); - std::fs::create_dir(&d).unwrap(); + let sb = Sandbox::new("ownlink"); + let d = sb.mkdir("kharon-teach-ownlink"); let real = d.join("real"); std::fs::write(&real, std::process::id().to_string()).unwrap(); std::os::unix::fs::symlink(&real, d.join(OWNER_FILE)).unwrap(); - assert!(Profile::sweep_stale().unwrap() >= 1); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 1); assert!(!d.exists()); } #[test] fn sweep_treats_a_junk_or_oversized_owner_as_no_owner() { - let a = dir_with_owner("junk", "not a pid"); - let b = dir_with_owner("big", &format!("{}{}", std::process::id(), " ".repeat(40))); - assert!(Profile::sweep_stale().unwrap() >= 2); + let sb = Sandbox::new("junk"); + let a = dir_with_owner(&sb, "kharon-teach-junk", "not a pid"); + let big = format!("{}{}", std::process::id(), " ".repeat(40)); + let b = dir_with_owner(&sb, "kharon-teach-big", &big); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 2); assert!(!a.exists() && !b.exists()); } + #[test] + fn preferences_turn_off_the_password_manager_and_autofill() { + let sb = Sandbox::new("prefs"); + write_preferences(sb.path()).unwrap(); + assert_eq!( + std::fs::metadata(sb.path().join("Default")).unwrap().mode() & 0o777, + 0o700 + ); + assert_eq!( + std::fs::read_to_string(sb.path().join("Default/Preferences")).unwrap(), + r#"{"credentials_enable_service":false,"profile":{"password_manager_enabled":false},"autofill":{"profile_enabled":false,"credit_card_enabled":false}}"# + ); + } + + #[tokio::test] + async fn a_first_url_that_looks_like_a_switch_is_refused() { + let sb = Sandbox::new("url"); + let profile = Profile::create_in(sb.path()).unwrap(); + let err = match Browser::launch(Launch { + browser: Path::new("/nonexistent/chrome"), + profile: &profile, + first_url: "--secret-switch", + headless: true, + }) + .await + { + Err(e) => e, + Ok(_) => panic!("launched"), + }; + let BrowserError::Launch(msg) = err else { + panic!("wrong error"); + }; + assert_eq!(msg, "the first URL must not start with '-'"); + } + #[test] fn no_browser_on_an_empty_path_is_not_found() { assert!(matches!( @@ -522,8 +609,8 @@ mod tests { #[test] fn finds_the_first_name_in_search_order() { - let dir = std::env::temp_dir().join(unique("bin")); - std::fs::create_dir(&dir).unwrap(); + let sb = Sandbox::new("bin"); + let dir = sb.path().to_owned(); for n in ["google-chrome", "chromium"] { let f = dir.join(n); std::fs::write(&f, b"#!/bin/sh\n").unwrap(); @@ -531,7 +618,6 @@ mod tests { } let found = find_browser_in(None, dir.as_os_str()).unwrap(); assert_eq!(found, dir.join("chromium")); - std::fs::remove_dir_all(&dir).unwrap(); } #[test] @@ -549,6 +635,17 @@ mod tests { assert_eq!(d.ws_path, "/devtools/browser/abc-123"); } + #[test] + fn accepts_a_bom_and_rejects_a_port_over_u16() { + assert_eq!( + parse_devtools_active_port("\u{feff}45123\n/devtools/browser/abc\n") + .unwrap() + .port, + 45123 + ); + assert!(parse_devtools_active_port("99999\n/devtools/browser/abc\n").is_none()); + } + #[test] fn rejects_port_zero_garbage_and_a_foreign_path() { for bad in [ From 9711369dfc9d8d340a8af7e46a1b7c1fd18729d5 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 05:48:08 -0500 Subject: [PATCH 20/36] fix(teach): zeroize CDP frame text and bound every wait (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-teach/Cargo.toml | 3 + crates/kharon-teach/src/cdp.rs | 233 +++++++++++++++++++++++------- crates/kharon-teach/src/events.rs | 27 +++- 3 files changed, 206 insertions(+), 57 deletions(-) diff --git a/crates/kharon-teach/Cargo.toml b/crates/kharon-teach/Cargo.toml index 3ea1bcb..c2f7017 100644 --- a/crates/kharon-teach/Cargo.toml +++ b/crates/kharon-teach/Cargo.toml @@ -34,3 +34,6 @@ tokio = { version = "1", features = ["fs", "macros", "net", "process", "rt-multi publicsuffix = "2.3" tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "handshake"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } + +[dev-dependencies] +tokio = { version = "1", features = ["test-util"] } diff --git a/crates/kharon-teach/src/cdp.rs b/crates/kharon-teach/src/cdp.rs index ef9a67b..f95d6bf 100644 --- a/crates/kharon-teach/src/cdp.rs +++ b/crates/kharon-teach/src/cdp.rs @@ -14,21 +14,27 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; -use tokio::net::TcpStream; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; -use tokio::sync::oneshot; +use tokio::sync::{oneshot, watch}; +use tokio_tungstenite::connect_async_with_config; use tokio_tungstenite::tungstenite::protocol::WebSocketConfig; use tokio_tungstenite::tungstenite::{self, Message}; -use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async_with_config}; +use zeroize::Zeroizing; const MAX_REMOTE_MESSAGE_CHARS: usize = 200; /// CDP frames that carry request bodies can be large. const MAX_FRAME_BYTES: usize = 64 * 1024 * 1024; +/// How long the WebSocket handshake may take. +pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// How long a call may wait for its reply. +pub const CALL_TIMEOUT: Duration = Duration::from_secs(30); #[derive(Debug, thiserror::Error)] pub enum CdpError { #[error("cdp connection closed")] Closed, + #[error("cdp call timed out")] + Timeout, #[error("cdp error {code}: {message}")] Remote { code: i64, message: String }, #[error("cdp transport: {0}")] @@ -66,14 +72,25 @@ pub fn parse_quiet(raw: &str) -> Result { serde_json::from_str(raw).map_err(CdpError::from) } -/// An event pushed by the browser. No `Debug`: `params` can hold page data. +/// An event pushed by the browser. `params` is the params JSON text (`{}` when +/// absent); parse it with [`parse_quiet`]. +/// +/// Held in `Zeroizing` and with no `Debug`, because it can hold page data. +/// Not covered, and not coverable here: tungstenite's own read buffers and +/// serde's scratch buffers while a frame is parsed. +/// +/// ```compile_fail +/// fn needs_debug() {} +/// needs_debug::(); +/// ``` pub struct Event { pub method: String, pub session_id: Option, - pub params: Box, + pub params: Zeroizing, } -/// What we send. Typed, never `json!`. +/// What we send. Typed, never `json!`. (A call can carry a script, not typed +/// values, so this text is not zeroized.) #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct Outgoing<'a, P: Serialize> { @@ -84,22 +101,23 @@ struct Outgoing<'a, P: Serialize> { session_id: Option<&'a str>, } -/// Anything the browser sends: a reply (`id`) or an event (`method`). +/// Anything the browser sends: a reply (`id`) or an event (`method`). It +/// borrows from the frame text, which is held in `Zeroizing`. #[derive(Deserialize)] #[serde(rename_all = "camelCase")] -struct Incoming { +struct Incoming<'a> { #[serde(default)] id: Option, #[serde(default)] method: Option, #[serde(default)] session_id: Option, - #[serde(default)] - params: Option>, - #[serde(default)] - result: Option>, - #[serde(default)] - error: Option>, + #[serde(default, borrow)] + params: Option<&'a RawValue>, + #[serde(default, borrow)] + result: Option<&'a RawValue>, + #[serde(default, borrow)] + error: Option<&'a RawValue>, } #[derive(Deserialize)] @@ -110,7 +128,7 @@ struct WireError { message: String, } -type Reply = Result, CdpError>; +type Reply = Result, CdpError>; #[derive(Default)] struct State { @@ -120,6 +138,7 @@ struct State { struct Shared { state: Mutex, + done: watch::Sender, } impl Shared { @@ -127,18 +146,36 @@ impl Shared { self.state.lock().unwrap_or_else(PoisonError::into_inner) } - /// Mark closed and drop every waiter: their receivers see `Closed`. + /// Mark closed, drop every waiter (their receivers see `Closed`) and tell + /// both socket tasks to stop. fn close(&self) { - let mut st = self.lock(); - st.closed = true; - st.pending.clear(); + { + let mut st = self.lock(); + st.closed = true; + st.pending.clear(); + } + self.done.send_replace(true); + } +} + +/// Removes a call's pending entry however the call ends (reply, timeout, or +/// the caller dropping the future). +struct PendingGuard<'a> { + shared: &'a Shared, + id: u64, +} + +impl Drop for PendingGuard<'_> { + fn drop(&mut self) { + self.shared.lock().pending.remove(&self.id); } } struct Inner { shared: Arc, next_id: AtomicU64, - /// When the last clone drops, this closes, the task ends and the socket closes. + /// When the last clone drops, this closes, the writer ends and closes the + /// socket, and the reader follows. out: UnboundedSender, } @@ -174,16 +211,24 @@ impl Cdp { max_frame_size: Some(MAX_FRAME_BYTES), ..WebSocketConfig::default() }; - let (ws, _resp) = connect_async_with_config(url, Some(config), false) - .await - .map_err(|e| transport(&e))?; + let dialed = tokio::time::timeout( + CONNECT_TIMEOUT, + connect_async_with_config(url, Some(config), false), + ) + .await + .map_err(|_| CdpError::Transport("handshake timed out".to_owned()))?; + let (ws, _resp) = dialed.map_err(|e| transport(&e))?; + let (sink, stream) = ws.split(); + let (done, _) = watch::channel(false); let shared = Arc::new(Shared { state: Mutex::new(State::default()), + done, }); let (out, out_rx) = unbounded_channel(); let (ev_tx, ev_rx) = unbounded_channel(); - tokio::spawn(run(ws, out_rx, Arc::clone(&shared), ev_tx)); + tokio::spawn(read_loop(stream, Arc::clone(&shared), ev_tx)); + tokio::spawn(write_loop(sink, out_rx, Arc::clone(&shared))); let cdp = Cdp { inner: Arc::new(Inner { shared, @@ -218,47 +263,80 @@ impl Cdp { } st.pending.insert(id, tx); } + let _guard = PendingGuard { + shared: &self.inner.shared, + id, + }; if self.inner.out.send(frame).is_err() { - self.inner.shared.lock().pending.remove(&id); return Err(CdpError::Closed); } - let raw = rx.await.map_err(|_| CdpError::Closed)??; - parse_quiet(raw.get()) + let raw = tokio::time::timeout(CALL_TIMEOUT, rx) + .await + .map_err(|_| CdpError::Timeout)? + .map_err(|_| CdpError::Closed)??; + parse_quiet(&raw) } } -/// The one task: reads frames, routes them, and writes queued calls. -async fn run( - mut ws: WebSocketStream>, - mut out: UnboundedReceiver, +type WsStream = + tokio_tungstenite::WebSocketStream>; + +/// Resolves once the connection is marked closed. The watch guard is dropped +/// before this returns, so it never blocks `Shared::close`. +async fn closed(done: &mut watch::Receiver) { + let _ = done.wait_for(|d| *d).await; +} + +/// Reads frames and routes them. The frame text is `Zeroizing` from the moment +/// it leaves tungstenite. +async fn read_loop( + mut stream: futures_util::stream::SplitStream, shared: Arc, events: UnboundedSender, ) { + let mut done = shared.done.subscribe(); loop { tokio::select! { - msg = ws.next() => match msg { - Some(Ok(Message::Text(t))) => route(&t, &shared, &events), + msg = stream.next() => match msg { + Some(Ok(Message::Text(t))) => route(&Zeroizing::new(t), &shared, &events), Some(Ok(Message::Close(_))) | Some(Err(_)) | None => break, // Binary, ping, pong: dropped. Some(Ok(_)) => {} }, - queued = out.recv() => match queued { - Some(text) => { - if ws.send(Message::Text(text)).await.is_err() { - break; - } - } - None => break, - }, + () = closed(&mut done) => break, + } + } + shared.close(); +} + +/// Writes queued calls. A blocked write never stops the reader. +async fn write_loop( + mut sink: futures_util::stream::SplitSink, + mut out: UnboundedReceiver, + shared: Arc, +) { + let mut done = shared.done.subscribe(); + loop { + let queued = tokio::select! { + q = out.recv() => q, + () = closed(&mut done) => None, + }; + let Some(text) = queued else { break }; + let sent = tokio::select! { + r = sink.send(Message::Text(text)) => r.is_ok(), + () = closed(&mut done) => false, + }; + if !sent { + break; } } shared.close(); - let _ = tokio::time::timeout(Duration::from_millis(500), ws.close(None)).await; + let _ = tokio::time::timeout(Duration::from_millis(500), sink.close()).await; } /// A frame that does not parse is dropped silently (never logged). -fn route(text: &str, shared: &Shared, events: &UnboundedSender) { - let Ok(m) = serde_json::from_str::(text) else { +fn route(text: &Zeroizing, shared: &Shared, events: &UnboundedSender) { + let Ok(m) = serde_json::from_str::>(text) else { return; }; if let Some(id) = m.id { @@ -276,23 +354,16 @@ fn route(text: &str, shared: &Shared, events: &UnboundedSender) { message: w.message.chars().take(MAX_REMOTE_MESSAGE_CHARS).collect(), }) } - (None, Some(result)) => Ok(result), - (None, None) => RawValue::from_string("{}".to_owned()) - .map_err(|_| CdpError::Transport("protocol error".to_owned())), + (None, Some(result)) => Ok(Zeroizing::new(result.get().to_owned())), + (None, None) => Ok(Zeroizing::new("{}".to_owned())), }; let _ = waiter.send(reply); } else if let Some(method) = m.method { - let params = match m.params { - Some(p) => p, - None => match RawValue::from_string("{}".to_owned()) { - Ok(p) => p, - Err(_) => return, - }, - }; + let params = m.params.map_or("{}", |p| p.get()); let _ = events.send(Event { method, session_id: m.session_id, - params, + params: Zeroizing::new(params.to_owned()), }); } } @@ -450,7 +521,7 @@ mod tests { let e = timeout(T, rx.recv()).await.unwrap().unwrap(); assert_eq!(e.method, "Page.frameNavigated"); assert_eq!(e.session_id.as_deref(), Some("S1")); - let f: FrameNavigated = parse_quiet(e.params.get()).unwrap(); + let f: FrameNavigated = parse_quiet(&e.params).unwrap(); assert_eq!( (f.frame.id.as_str(), f.frame.url.as_str()), ("F", "https://a.test/") @@ -642,4 +713,54 @@ mod tests { } )); } + + #[tokio::test(start_paused = true)] + async fn a_call_with_no_reply_times_out_and_leaves_no_entry() { + let (dev, _srv) = serve(|mut ws| async move { + let _ = next_text(&mut ws).await; + tokio::time::sleep(Duration::from_secs(3600)).await; + ws + }) + .await; + let (cdp, _rx) = Cdp::connect(&dev).await.unwrap(); + let r = cdp.call::<_, Empty2>("A.a", &Empty {}, None).await; + assert!(matches!(r, Err(CdpError::Timeout))); + assert!(cdp.inner.shared.lock().pending.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn a_dropped_call_leaves_no_pending_entry() { + let (dev, _srv) = serve(|mut ws| async move { + let _ = next_text(&mut ws).await; + tokio::time::sleep(Duration::from_secs(3600)).await; + ws + }) + .await; + let (cdp, _rx) = Cdp::connect(&dev).await.unwrap(); + let r = tokio::time::timeout( + Duration::from_secs(1), + cdp.call::<_, Empty2>("A.a", &Empty {}, None), + ) + .await; + assert!(r.is_err(), "the caller's own timeout fires first"); + assert!(cdp.inner.shared.lock().pending.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn a_handshake_that_never_finishes_times_out() { + let l = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = l.local_addr().unwrap().port(); + let _hold = tokio::spawn(async move { + let _conn = l.accept().await; + tokio::time::sleep(Duration::from_secs(3600)).await; + }); + let dev = DevTools { + port, + ws_path: "/devtools/browser/x".to_owned(), + }; + match Cdp::connect(&dev).await { + Err(CdpError::Transport(w)) => assert_eq!(w, "handshake timed out"), + _ => panic!("expected a handshake timeout"), + } + } } diff --git a/crates/kharon-teach/src/events.rs b/crates/kharon-teach/src/events.rs index c5b5eff..d9061c9 100644 --- a/crates/kharon-teach/src/events.rs +++ b/crates/kharon-teach/src/events.rs @@ -148,6 +148,11 @@ pub struct PostData { // ---- headers ---- /// Header map. Values are `Zeroizing` and there is no `Debug`. +/// +/// ```compile_fail +/// fn needs_debug() {} +/// needs_debug::(); +/// ``` #[derive(Default)] pub struct Headers(HashMap>); @@ -231,6 +236,12 @@ impl<'de> Deserialize<'de> for Headers { fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("a header object") } + fn visit_unit(self) -> Result { + Ok(Headers::default()) + } + fn visit_none(self) -> Result { + Ok(Headers::default()) + } fn visit_map>(self, mut map: A) -> Result { let mut out = HashMap::new(); while let Some((k, v)) = map.next_entry::()? { @@ -241,7 +252,7 @@ impl<'de> Deserialize<'de> for Headers { Ok(Headers(out)) } } - d.deserialize_map(V) + d.deserialize_any(V) } } @@ -296,6 +307,12 @@ pub enum ReportKind { EnterKey, } +/// What the page script reported. No `Debug`: `value` is what a person typed. +/// +/// ```compile_fail +/// fn needs_debug() {} +/// needs_debug::(); +/// ``` #[derive(Deserialize)] #[serde(rename_all = "snake_case")] pub struct ListenerReport { @@ -463,6 +480,14 @@ mod tests { assert!(!h.has_set_cookie()); } + #[test] + fn null_headers_are_empty_not_fatal() { + let r: ResponseReceived = + parse_quiet(r#"{"requestId":"1","response":{"status":200,"headers":null}}"#).unwrap(); + assert!(r.response.headers.content_type().is_none()); + assert!(!r.response.headers.has_set_cookie()); + } + #[test] fn a_non_string_header_is_skipped_not_fatal() { let h: Headers = From 63dfdda2556b5605d5173de86f51b7b78739a10f Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 06:06:24 -0500 Subject: [PATCH 21/36] feat(teach): record a login from a live browser session (#361) The in-page listener, the session that wires a launched browser, the CDP client and the capture state machine together (record()), the notice page, and real-browser tests that classify the four fixture logins and check that nothing typed reaches the recording or the profile. Script world: /usr/bin/chromium delivers the binding from the isolated world ("kharon-teach"), so listener_world is isolated. Runtime.enable is required on each page session or Runtime.bindingCalled is never sent. Set-Cookie is only in Network.responseReceivedExtraInfo, which can arrive after the next hop's requestWillBeSent, so hops are matched in order. Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- Cargo.lock | 1 + crates/kharon-teach/Cargo.toml | 1 + crates/kharon-teach/src/events.rs | 84 ++ crates/kharon-teach/src/lib.rs | 2 + crates/kharon-teach/src/listener.js | 234 +++++ crates/kharon-teach/src/notice.rs | 99 ++ crates/kharon-teach/src/session.rs | 894 ++++++++++++++++++ .../tests/req_tch_002_no_typed_value.rs | 24 + .../kharon-teach/tests/req_tch_005_shapes.rs | 118 +++ .../tests/req_tch_006_profile_and_visible.rs | 201 ++++ crates/kharon-teach/tests/support/mod.rs | 567 +++++++++++ 11 files changed, 2225 insertions(+) create mode 100644 crates/kharon-teach/src/listener.js create mode 100644 crates/kharon-teach/src/notice.rs create mode 100644 crates/kharon-teach/src/session.rs create mode 100644 crates/kharon-teach/tests/req_tch_002_no_typed_value.rs create mode 100644 crates/kharon-teach/tests/req_tch_005_shapes.rs create mode 100644 crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs diff --git a/Cargo.lock b/Cargo.lock index 6884278..5f50570 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1097,6 +1097,7 @@ dependencies = [ "sha1", "sha2", "thiserror 2.0.21", + "time", "tokio", "tokio-tungstenite", "url", diff --git a/crates/kharon-teach/Cargo.toml b/crates/kharon-teach/Cargo.toml index c2f7017..07bbad5 100644 --- a/crates/kharon-teach/Cargo.toml +++ b/crates/kharon-teach/Cargo.toml @@ -33,6 +33,7 @@ rand = "0.8" tokio = { version = "1", features = ["fs", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time", "io-std", "io-util"] } publicsuffix = "2.3" tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "handshake"] } +time = { version = "0.3", features = ["formatting"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } [dev-dependencies] diff --git a/crates/kharon-teach/src/events.rs b/crates/kharon-teach/src/events.rs index d9061c9..6448a43 100644 --- a/crates/kharon-teach/src/events.rs +++ b/crates/kharon-teach/src/events.rs @@ -78,6 +78,9 @@ pub struct RequestWillBeSent { pub request: Request, #[serde(rename = "type", default)] pub resource: Option, + /// Set when this request is the next hop of a redirect: the previous hop's response. + #[serde(default)] + pub redirect_response: Option, } #[derive(Deserialize)] @@ -116,6 +119,23 @@ fn lenient_status<'de, D: Deserializer<'de>>(d: D) -> Result { Ok(v.clamp(0.0, f64::from(u16::MAX)) as u16) } +/// The raw response headers, which Chromium reports apart from `Network.responseReceived` because +/// some of them (`Set-Cookie`) are only in this event. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResponseReceivedExtraInfo { + pub request_id: String, + #[serde(default, deserialize_with = "lenient_opt_status")] + pub status_code: Option, + #[serde(default)] + pub headers: Headers, +} + +fn lenient_opt_status<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let v = Option::::deserialize(d)?; + Ok(v.map(|v| v.clamp(0.0, f64::from(u16::MAX)) as u16)) +} + // ---- results ---- #[derive(Deserialize)] @@ -288,12 +308,40 @@ pub struct GetRequestPostData<'a> { pub request_id: &'a str, } +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SetDiscoverTargets { + pub discover: bool, +} + +#[derive(Serialize)] +pub struct Navigate<'a> { + pub url: &'a str, +} + +#[derive(Serialize)] +pub struct RemoveScript<'a> { + pub identifier: &'a str, +} + +/// The result of `Page.addScriptToEvaluateOnNewDocument`. +#[derive(Deserialize)] +pub struct ScriptId { + pub identifier: String, +} + /// Parameters for a call that takes none: serializes as `{}`. #[derive(Serialize)] pub struct Empty {} // ---- the page script's report ---- +/// What the listener sends once when it starts in a top-level document. +#[derive(Deserialize)] +pub struct Hello { + pub hello: bool, +} + const MAX_FIELD_CHARS: usize = 4096; const MAX_SELECTORS: usize = 16; @@ -554,4 +602,40 @@ mod tests { .unwrap(); assert_eq!(s, r#"{"source":"1","worldName":"w"}"#); } + + #[test] + fn a_redirect_event_carries_the_previous_hops_response() { + let raw = r#"{"requestId":"r","loaderId":"r","documentURL":"https://a.test/", + "request":{"url":"https://a.test/home","method":"GET"}, + "redirectResponse":{"status":302,"headers":{"Location":"/home"}},"type":"Document"}"#; + let e: RequestWillBeSent = parse_quiet(raw).unwrap(); + assert_eq!(e.redirect_response.map(|r| r.status), Some(302)); + let raw = r#"{"requestId":"r","loaderId":"r","documentURL":"https://a.test/", + "request":{"url":"https://a.test/","method":"GET"}}"#; + assert!( + parse_quiet::(raw) + .unwrap() + .redirect_response + .is_none() + ); + } + + #[test] + fn raw_response_headers_carry_set_cookie() { + let e: ResponseReceivedExtraInfo = parse_quiet( + r#"{"requestId":"r","statusCode":302,"headers":{"set-cookie":"sid=x"},"blockedCookies":[]}"#, + ) + .unwrap(); + assert_eq!(e.status_code, Some(302)); + assert!(e.headers.has_set_cookie()); + let e: ResponseReceivedExtraInfo = parse_quiet(r#"{"requestId":"r"}"#).unwrap(); + assert!(e.status_code.is_none() && !e.headers.has_set_cookie()); + } + + #[test] + fn hello_is_not_a_report() { + assert!(parse_quiet::(r#"{"hello":true}"#).is_ok_and(|h| h.hello)); + assert!(parse_quiet::(REPORT).is_err()); + assert!(parse_quiet::(r#"{"hello":true}"#).is_err()); + } } diff --git a/crates/kharon-teach/src/lib.rs b/crates/kharon-teach/src/lib.rs index f558891..d63df93 100644 --- a/crates/kharon-teach/src/lib.rs +++ b/crates/kharon-teach/src/lib.rs @@ -6,6 +6,8 @@ pub mod capture; pub mod cdp; pub mod events; pub mod marker; +pub mod notice; +pub mod session; pub mod site; pub mod strip; #[doc(hidden)] diff --git a/crates/kharon-teach/src/listener.js b/crates/kharon-teach/src/listener.js new file mode 100644 index 0000000..e0dfec5 --- /dev/null +++ b/crates/kharon-teach/src/listener.js @@ -0,0 +1,234 @@ +// kharon-teach page listener. Runs at document start in every frame. +// +// It reports what a person did (which element, what kind of event, and the +// text of a text-entry control) through the one binding the recorder added. +// It keeps no state except an element counter, and leaves no global. +(() => { + "use strict"; + try { + const BINDING = "__kharonTeach"; + const GUARD = Symbol.for("kharon-teach.listener"); + if (Object.prototype.hasOwnProperty.call(document, GUARD)) return; + Object.defineProperty(document, GUARD, { value: true, enumerable: false }); + + const MAX_STR = 2048; + const MAX_VALUE = 4096; + const MAX_SELECTORS = 5; + const TEXTISH = ["text", "email", "password", "tel", "number", "search", "url"]; + + const cut = (s) => (s === null || s === undefined ? null : String(s).slice(0, MAX_STR)); + const ids = new WeakMap(); + let counter = 0; + let documentId = ""; + try { + const bytes = new Uint8Array(8); + crypto.getRandomValues(bytes); + documentId = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join(""); + } catch (_) { + documentId = String(Math.random()).slice(2) + String(Date.now()); + } + + const send = (report) => { + try { + const bind = window[BINDING]; + if (typeof bind === "function") bind(JSON.stringify(report)); + } catch (_) { + /* never throw into the page */ + } + }; + + const isTop = () => { + try { + return window.top === window; + } catch (_) { + return false; + } + }; + + const elementId = (el) => { + let n = ids.get(el); + if (n === undefined) { + n = ++counter; + ids.set(el, n); + } + return n; + }; + + const attrEscape = (s) => String(s).replace(/\\/g, "\\\\").replace(/"/g, '\\"'); + + const structural = (el, scope) => { + const parts = []; + let node = el; + for (let depth = 0; node && node.nodeType === 1 && depth < 6; depth++) { + const tag = node.tagName.toLowerCase(); + let part = tag; + const parent = node.parentElement; + if (parent) { + const same = Array.from(parent.children).filter((c) => c.tagName === node.tagName); + if (same.length > 1) part += ":nth-of-type(" + (same.indexOf(node) + 1) + ")"; + } + parts.unshift(part); + const candidate = parts.join(" > "); + try { + if (scope.querySelector(candidate) === el) return candidate; + } catch (_) { + /* try a longer path */ + } + node = parent; + } + return null; + }; + + const selectorsFor = (el, typed) => { + const out = []; + let root = null; + try { + root = el.getRootNode(); + } catch (_) { + root = document; + } + const inShadow = typeof ShadowRoot !== "undefined" && root instanceof ShadowRoot; + const scope = inShadow ? root : document; + const add = (s, unique) => { + if (!s || s.length > MAX_STR || out.includes(s)) return; + // A value this short is contained in an ordinary selector by chance. + if (typed && typed.length >= 3 && s.includes(typed)) return; + try { + if (scope.querySelector(s) !== el) return; + if (unique && scope.querySelectorAll(s).length !== 1) return; + out.push(s); + } catch (_) { + /* an invalid candidate is dropped */ + } + }; + const tag = el.tagName.toLowerCase(); + if (el.id) add("#" + CSS.escape(el.id)); + const name = el.getAttribute("name"); + if (name) add(tag + '[name="' + attrEscape(name) + '"]'); + const auto = el.getAttribute("autocomplete"); + if (auto) add(tag + '[autocomplete="' + attrEscape(auto) + '"]'); + const type = el.getAttribute("type"); + if (type) add(tag + '[type="' + attrEscape(type) + '"]', true); + add(structural(el, scope)); + return { selectors: out.slice(0, MAX_SELECTORS), inShadow }; + }; + + const formInfo = (form) => { + if (!form) return null; + let index = 0; + try { + index = Math.max(0, Array.prototype.indexOf.call(document.forms, form)); + } catch (_) { + index = 0; + } + return { + index, + action: cut(form.action), + method: String(form.method || "get").toLowerCase(), + }; + }; + + const textEntry = (el) => { + const tag = el.tagName.toLowerCase(); + if (tag === "textarea") return true; + if (tag !== "input") return false; + const type = String(el.type || "text").toLowerCase(); + return TEXTISH.includes(type); + }; + + const describe = (el, typed) => { + const tag = el.tagName.toLowerCase(); + const isInput = tag === "input"; + const sel = selectorsFor(el, typed); + const form = tag === "form" ? el : el.form || null; + return { + element_id: elementId(el), + tag, + input_type: isInput ? String(el.type || "text").toLowerCase() : null, + id: cut(el.id || null), + name: cut(el.getAttribute("name")), + autocomplete: cut(el.getAttribute("autocomplete")), + placeholder: cut(el.getAttribute("placeholder")), + aria_label: cut(el.getAttribute("aria-label")), + is_password_now: isInput && String(el.type).toLowerCase() === "password", + form: formInfo(form), + selectors: sel.selectors, + in_shadow_root: sel.inShadow, + }; + }; + + const originOf = (e) => { + let t = null; + try { + const path = e.composedPath ? e.composedPath() : []; + t = path.length > 0 ? path[0] : e.target; + } catch (_) { + t = e.target; + } + return t && t.nodeType === 1 ? t : null; + }; + + const report = (kind, el, withValue) => { + let value = null; + if (withValue && textEntry(el)) { + const v = String(el.value === undefined ? "" : el.value); + if (v.length <= MAX_VALUE) value = v; + } + const body = { + kind, + document_id: documentId, + frame_url: cut(location.href) || "", + is_top: isTop(), + element: describe(el, value), + }; + if (value !== null) body.value = value; + send(body); + }; + + const CLICKABLE = 'button, a, input[type="submit"], input[type="button"], [role="button"]'; + + const handlers = { + input: (e) => { + const el = originOf(e); + if (el && el.matches("input, textarea, select")) report("input", el, true); + }, + change: (e) => { + const el = originOf(e); + if (el && el.matches("input, textarea, select")) report("change", el, true); + }, + click: (e) => { + const el = originOf(e); + const hit = el && el.closest ? el.closest(CLICKABLE) : null; + if (hit) report("click", hit, false); + }, + submit: (e) => { + const el = originOf(e); + if (el && el.tagName.toLowerCase() === "form") report("submit", el, false); + }, + keydown: (e) => { + if (e.key !== "Enter") return; + const el = originOf(e); + if (el && el.tagName.toLowerCase() === "input") report("enter_key", el, false); + }, + }; + + for (const type of Object.keys(handlers)) { + const fn = handlers[type]; + document.addEventListener( + type, + (e) => { + try { + fn(e); + } catch (_) { + /* never throw into the page */ + } + }, + true, + ); + } + + if (isTop()) send({ hello: true }); + } catch (_) { + /* never throw into the page */ + } +})(); diff --git a/crates/kharon-teach/src/notice.rs b/crates/kharon-teach/src/notice.rs new file mode 100644 index 0000000..5583449 --- /dev/null +++ b/crates/kharon-teach/src/notice.rs @@ -0,0 +1,99 @@ +//! The page the browser opens first: what the recorder does with what a person types, and one +//! link that starts the recording. + +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode}; + +pub const NOTICE: &str = "kharon-teach records how a login works so kharon can learn it.\n\ +Use a THROWAWAY account. Everything you type in the window that\n\ +opens is read by this program. Nothing you type is saved, and\n\ +nothing is sent anywhere. The result is one file on this computer.\n"; + +const KEEP: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~'); + +fn escape_html(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + c => out.push(c), + } + } + out +} + +/// The notice page as HTML text. It holds no script. +pub fn notice_page_html(login_url: &url::Url) -> String { + let host = login_url.host_str().unwrap_or("the login page"); + format!( + "kharon-teach\ +
{notice}

Start recording and open {host}

", + notice = escape_html(NOTICE), + href = escape_html(login_url.as_str()), + host = escape_html(host), + ) +} + +/// A `data:` URL page that shows [`NOTICE`] and one link, "Start recording and open ", to +/// `login_url`. +pub fn notice_page_url(login_url: &url::Url) -> String { + let html = notice_page_html(login_url); + format!( + "data:text/html;charset=utf-8,{}", + utf8_percent_encode(&html, KEEP) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_page_shows_the_notice_and_one_link_to_the_login_url() { + let url = url::Url::parse("https://login.example.test/sign-in?next=a").unwrap(); + let html = notice_page_html(&url); + assert!(html.contains("THROWAWAY")); + assert!(html.contains("Start recording and open login.example.test")); + assert_eq!(html.matches("\"q\"&w=%3Cscript%3E").unwrap(); + let html = notice_page_html(&url); + let start = html.find("href=\"").unwrap() + 6; + let end = start + html[start..].find('"').unwrap(); + let attr = &html[start..end]; + assert!(!attr.contains('<') && !attr.contains('\'') && !attr.contains('>')); + let back = attr + .replace("<", "<") + .replace(">", ">") + .replace("'", "'") + .replace(""", "\"") + .replace("&", "&"); + assert_eq!(back, url.as_str()); + assert_eq!(html.matches("")); + } + + #[test] + fn the_url_is_a_data_url_with_no_raw_markup() { + let url = url::Url::parse("https://a.test/").unwrap(); + let page = notice_page_url(&url); + let body = page.strip_prefix("data:text/html;charset=utf-8,").unwrap(); + assert!(!body.contains('<') && !body.contains(' ') && !body.contains('"')); + let decoded = percent_encoding::percent_decode_str(body) + .decode_utf8() + .unwrap(); + assert_eq!(decoded, notice_page_html(&url)); + } +} diff --git a/crates/kharon-teach/src/session.rs b/crates/kharon-teach/src/session.rs new file mode 100644 index 0000000..5868931 --- /dev/null +++ b/crates/kharon-teach/src/session.rs @@ -0,0 +1,894 @@ +//! One recording session: a launched browser, a CDP connection and the capture state machine, +//! wired so that what a person does in the window becomes a [`Recording`]. +//! +//! Nothing in this file writes to stdout or stderr, and nothing from a frame reaches an error. + +use crate::browser::{Browser, BrowserError, Launch, Profile}; +use crate::capture::{ + Capture, CaptureEvent, EndInfo, PageEventKind, RawBody, RawElement, RawForm, RawFrame, +}; +use crate::cdp::{Cdp, CdpError, Event, parse_quiet}; +use crate::events::{ + AddBinding, AddScriptToEvaluateOnNewDocument, AttachedToTarget, BindingCalled, Empty, + FrameNavigated, GetRequestPostData, Hello, LayoutMetrics, ListenerReport, Navigate, PostData, + RemoveScript, ReportKind, RequestWillBeSent, ResponseReceived, ResponseReceivedExtraInfo, + ScriptId, SetAutoAttach, SetDiscoverTargets, TargetDestroyed, VersionInfo, +}; +use crate::notice::notice_page_url; +use kharon_recording::{BrowserInfo, EndedBy, ListenerWorld, Recording, ResourceKind, Viewport}; +use serde::Serialize; +use serde::de::IgnoredAny; +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use std::time::{Duration, Instant, SystemTime}; +use tokio::sync::{mpsc::UnboundedReceiver, oneshot}; +use zeroize::Zeroizing; + +/// The page script, run in every frame at document start. +pub const LISTENER_JS: &str = include_str!("listener.js"); +const BINDING: &str = "__kharonTeach"; +const WORLD: &str = "kharon-teach"; +/// How long the listener may take to announce itself on the notice page. +const HELLO_WAIT: Duration = Duration::from_secs(3); +/// How long to keep taking events after the person confirms. +const LATE_WAIT: Duration = Duration::from_millis(300); +/// How long the browser may take to show its first page. +const FIRST_PAGE_WAIT: Duration = Duration::from_secs(10); + +pub enum End { + Confirmed, + Discard, +} + +pub struct RecordOptions<'a> { + pub login_url: &'a url::Url, + pub browser: &'a std::path::Path, + pub headless: bool, + pub tool_version: &'a str, +} + +pub struct Ready { + pub devtools_port: u16, + pub profile_dir: PathBuf, + pub stale_profiles_removed: usize, +} + +#[derive(Debug, thiserror::Error)] +pub enum SessionError { + #[error(transparent)] + Browser(#[from] BrowserError), + #[error(transparent)] + Cdp(#[from] CdpError), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("the browser showed no page")] + NoPage, + #[error("the page script did not start in the browser")] + ListenerNotRunning, +} + +pub enum Outcome { + Recorded(Box), + Discarded, + NeverStarted, +} + +/// RFC 3339, UTC. +pub fn utc_rfc3339(t: SystemTime) -> String { + use time::format_description::well_known::Rfc3339; + time::OffsetDateTime::from(t) + .format(&Rfc3339) + .unwrap_or_default() +} + +struct Sess { + target_id: String, + /// The id of the injected script, so the world can be changed once. + script_id: Option, +} + +/// One hop of a request. Chromium reuses a request id across redirects, and the raw headers of a +/// hop (`Set-Cookie`) can arrive after the next hop's `requestWillBeSent`. +#[derive(Default)] +struct Hop { + /// The raw-header event for this hop has been taken. + extra_seen: bool, + status: Option, + content_type: Option, + set_cookie: bool, +} + +enum Flow { + Continue, + WindowClosed, +} + +struct Rec { + cdp: Cdp, + notice: String, + world: ListenerWorld, + sessions: HashMap, + page_targets: HashSet, + first_session: Option, + first_main_frame: Option, + capture: Capture, + clock: Option<(Instant, SystemTime)>, + hops: HashMap>, + hello: bool, +} + +async fn send( + cdp: &Cdp, + method: &str, + params: &P, + session: Option<&str>, +) -> Result<(), CdpError> { + cdp.call::(method, params, session) + .await + .map(|_| ()) +} + +/// Sweeps stale profiles, launches, records until `end` fires or the window closes, cleans up. +/// `ready` fires once the browser is up and the notice page is showing. Dropping the future +/// (Ctrl-C) still removes the profile. +pub async fn record( + opts: RecordOptions<'_>, + ready: oneshot::Sender, + end: oneshot::Receiver, +) -> Result { + let stale = Profile::sweep_stale().unwrap_or(0); + let profile = Profile::create()?; + let profile_dir = profile.path().to_owned(); + let notice = notice_page_url(opts.login_url); + let (browser, dev) = Browser::launch(Launch { + browser: opts.browser, + profile: &profile, + first_url: "about:blank", + headless: opts.headless, + }) + .await?; + let mut guard = Cleanup { + browser: Some(browser), + profile: Some(profile), + }; + let Some(browser) = guard.browser.as_mut() else { + return Err(SessionError::NoPage); + }; + let result = run( + browser, + &dev, + &opts, + ¬ice, + Ready { + devtools_port: dev.port, + profile_dir, + stale_profiles_removed: stale, + }, + ready, + end, + ) + .await; + guard.finish().await; + result +} + +/// Owns the browser and its profile. However the session ends, including the future being +/// dropped, the browser is killed and the profile removed. +struct Cleanup { + browser: Option, + profile: Option, +} + +/// A just-killed browser's helper processes can still be writing into the profile for a moment. +const REMOVE_TRIES: u32 = 60; +const REMOVE_PAUSE: Duration = Duration::from_millis(50); + +/// Removes the profile; true once it is gone. +fn try_remove(path: &std::path::Path) -> bool { + let _ = std::fs::remove_dir_all(path); + !path.exists() +} + +/// The browser's process and everything below it. A killed browser's helper processes (the +/// network service above all) outlive it for a moment and recreate files in a profile that was +/// just removed, so the profile is removed only after they are gone. +#[cfg(target_os = "linux")] +fn family(root: u32) -> Vec { + let mut parent_of: Vec<(u32, u32)> = Vec::new(); + if let Ok(procs) = std::fs::read_dir("/proc") { + for e in procs.flatten() { + let Ok(pid) = e.file_name().to_string_lossy().parse::() else { + continue; + }; + if let Some((_, ppid)) = stat_of(pid) { + parent_of.push((pid, ppid)); + } + } + } + let mut out = vec![root]; + let mut i = 0; + while i < out.len() { + let at = out[i]; + out.extend( + parent_of + .iter() + .filter(|(_, pp)| *pp == at) + .map(|(p, _)| *p), + ); + i += 1; + } + out +} + +/// A process's state letter and parent pid, from `/proc//stat`. +#[cfg(target_os = "linux")] +fn stat_of(pid: u32) -> Option<(char, u32)> { + let text = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + // The command name is in parentheses and may hold spaces and parentheses itself. + let rest = &text[text.rfind(')')? + 1..]; + let mut f = rest.split_whitespace(); + let state = f.next()?.chars().next()?; + let ppid = f.next()?.parse().ok()?; + Some((state, ppid)) +} + +#[cfg(target_os = "linux")] +fn any_alive(pids: &[u32]) -> bool { + pids.iter() + .any(|p| stat_of(*p).is_some_and(|(state, _)| state != 'Z' && state != 'X')) +} + +#[cfg(not(target_os = "linux"))] +fn family(root: u32) -> Vec { + vec![root] +} + +#[cfg(not(target_os = "linux"))] +fn any_alive(_pids: &[u32]) -> bool { + false +} + +impl Cleanup { + fn family(&self) -> Vec { + self.browser + .as_ref() + .map(|b| family(b.pid())) + .unwrap_or_default() + } + + /// The normal end: wait for the browser to be gone, then remove the profile. + async fn finish(&mut self) { + let procs = self.family(); + if let Some(b) = self.browser.take() { + let _ = b.close().await; + } + for _ in 0..REMOVE_TRIES { + if !any_alive(&procs) { + break; + } + tokio::time::sleep(REMOVE_PAUSE).await; + } + if let Some(p) = self.profile.take() { + let path = p.path().to_owned(); + for _ in 0..REMOVE_TRIES { + if try_remove(&path) { + break; + } + tokio::time::sleep(REMOVE_PAUSE).await; + } + } + } +} + +impl Drop for Cleanup { + fn drop(&mut self) { + let procs = self.family(); + // Dropping the browser kills it. + drop(self.browser.take()); + for _ in 0..REMOVE_TRIES { + if !any_alive(&procs) { + break; + } + std::thread::sleep(REMOVE_PAUSE); + } + if let Some(p) = self.profile.take() { + let path = p.path().to_owned(); + for _ in 0..REMOVE_TRIES { + if try_remove(&path) { + break; + } + std::thread::sleep(REMOVE_PAUSE); + } + } + } +} + +async fn run( + browser: &mut Browser, + dev: &crate::browser::DevTools, + opts: &RecordOptions<'_>, + notice: &str, + info: Ready, + ready: oneshot::Sender, + mut end: oneshot::Receiver, +) -> Result { + let (cdp, mut rx) = Cdp::connect(dev).await?; + let mut rec = Rec { + cdp: cdp.clone(), + notice: notice.to_owned(), + world: ListenerWorld::Isolated, + sessions: HashMap::new(), + page_targets: HashSet::new(), + first_session: None, + first_main_frame: None, + capture: Capture::new(opts.login_url.as_str(), opts.tool_version), + clock: None, + hops: HashMap::new(), + hello: false, + }; + + send( + &cdp, + "Target.setDiscoverTargets", + &SetDiscoverTargets { discover: true }, + None, + ) + .await?; + send(&cdp, "Target.setAutoAttach", &auto_attach(), None).await?; + + // The browser's first page. + let deadline = tokio::time::Instant::now() + FIRST_PAGE_WAIT; + while rec.first_session.is_none() { + match tokio::time::timeout_at(deadline, rx.recv()).await { + Ok(Some(ev)) => { + if let Flow::WindowClosed = rec.on_event(ev).await { + return Ok(Outcome::NeverStarted); + } + } + Ok(None) | Err(_) => return Err(SessionError::NoPage), + } + } + + match rec.choose_world(&mut rx).await? { + Choice::Done => {} + Choice::Closed => return Ok(Outcome::NeverStarted), + } + let _ = ready.send(info); + + let mut closed = false; + let confirmed = loop { + tokio::select! { + biased; + e = &mut end => break matches!(e, Ok(End::Confirmed)), + ev = rx.recv() => match ev { + Some(ev) => { + if let Flow::WindowClosed = rec.on_event(ev).await { + closed = true; + break false; + } + } + None => { closed = true; break false; } + }, + _ = browser.wait() => { closed = true; break false; } + } + }; + + if !confirmed && !closed { + return Ok(Outcome::Discarded); + } + // Everything already queued, then a short wait for the last navigation and its response. + while let Ok(ev) = rx.try_recv() { + if closed { + rec.on_event(ev).await; + } else if let Flow::WindowClosed = rec.on_event(ev).await { + closed = true; + } + } + if !closed { + let deadline = tokio::time::Instant::now() + LATE_WAIT; + while let Ok(Some(ev)) = tokio::time::timeout_at(deadline, rx.recv()).await { + rec.on_event(ev).await; + } + } + Ok(rec.finish(confirmed)) +} + +fn auto_attach() -> SetAutoAttach { + SetAutoAttach { + auto_attach: true, + wait_for_debugger_on_start: true, + flatten: true, + } +} + +enum Choice { + Done, + Closed, +} + +enum Wait { + Hello, + Timeout, + Closed, +} + +impl Rec { + fn t_ms(&self) -> u64 { + self.clock.map_or(0, |(t0, _)| { + u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX) + }) + } + + /// Show the notice page and find out which world the listener runs in. The world is never + /// switched after this. + async fn choose_world( + &mut self, + rx: &mut UnboundedReceiver, + ) -> Result { + let Some(first) = self.first_session.clone() else { + return Err(SessionError::NoPage); + }; + let notice = self.notice.clone(); + send( + &self.cdp, + "Page.navigate", + &Navigate { url: ¬ice }, + Some(&first), + ) + .await?; + match self.wait_hello(rx).await { + Wait::Hello => return Ok(Choice::Done), + Wait::Closed => return Ok(Choice::Closed), + Wait::Timeout => {} + } + // The isolated world did not deliver: install in the main world instead. + if let Some(id) = self + .sessions + .get_mut(&first) + .and_then(|s| s.script_id.take()) + { + send( + &self.cdp, + "Page.removeScriptToEvaluateOnNewDocument", + &RemoveScript { identifier: &id }, + Some(&first), + ) + .await?; + } + self.world = ListenerWorld::Main; + self.install(&first).await?; + send( + &self.cdp, + "Page.navigate", + &Navigate { url: ¬ice }, + Some(&first), + ) + .await?; + match self.wait_hello(rx).await { + Wait::Hello => Ok(Choice::Done), + Wait::Closed => Ok(Choice::Closed), + Wait::Timeout => Err(SessionError::ListenerNotRunning), + } + } + + async fn wait_hello(&mut self, rx: &mut UnboundedReceiver) -> Wait { + self.hello = false; + let deadline = tokio::time::Instant::now() + HELLO_WAIT; + while !self.hello { + match tokio::time::timeout_at(deadline, rx.recv()).await { + Ok(Some(ev)) => { + if let Flow::WindowClosed = self.on_event(ev).await { + return Wait::Closed; + } + } + Ok(None) => return Wait::Closed, + Err(_) => return Wait::Timeout, + } + } + Wait::Hello + } + + /// The binding and the script, in the world chosen now. + async fn install(&mut self, session: &str) -> Result<(), CdpError> { + let isolated = self.world == ListenerWorld::Isolated; + send( + &self.cdp, + "Runtime.addBinding", + &AddBinding { + name: BINDING, + execution_context_name: isolated.then_some(WORLD), + }, + Some(session), + ) + .await?; + let id: ScriptId = self + .cdp + .call( + "Page.addScriptToEvaluateOnNewDocument", + &AddScriptToEvaluateOnNewDocument { + source: LISTENER_JS, + world_name: isolated.then_some(WORLD), + }, + Some(session), + ) + .await?; + if let Some(s) = self.sessions.get_mut(session) { + s.script_id = Some(id.identifier); + } + Ok(()) + } + + async fn setup(&mut self, session: &str) -> Result<(), CdpError> { + send( + &self.cdp, + "Target.setAutoAttach", + &auto_attach(), + Some(session), + ) + .await?; + send(&self.cdp, "Page.enable", &Empty {}, Some(session)).await?; + send(&self.cdp, "Network.enable", &Empty {}, Some(session)).await?; + send(&self.cdp, "Runtime.enable", &Empty {}, Some(session)).await?; + self.install(session).await?; + Ok(()) + } + + async fn on_event(&mut self, ev: Event) -> Flow { + let session = ev.session_id.as_deref(); + match ev.method.as_str() { + "Target.attachedToTarget" => { + if let Ok(p) = parse_quiet::(&ev.params) { + self.attached(p).await; + } + } + "Target.targetDestroyed" => { + if let Ok(p) = parse_quiet::(&ev.params) { + self.sessions.retain(|_, s| s.target_id != p.target_id); + if self.page_targets.remove(&p.target_id) && self.page_targets.is_empty() { + return Flow::WindowClosed; + } + } + } + _ => { + let Some(session) = session else { + return Flow::Continue; + }; + if !self.sessions.contains_key(session) { + return Flow::Continue; + } + match ev.method.as_str() { + "Page.frameNavigated" => { + if let Ok(p) = parse_quiet::(&ev.params) { + self.navigated(session, p).await; + } + } + "Runtime.bindingCalled" => { + if let Ok(p) = parse_quiet::(&ev.params) { + self.binding(session, &p); + } + } + "Network.requestWillBeSent" => { + if let Ok(p) = parse_quiet::(&ev.params) { + self.request(session, p).await; + } + } + "Network.responseReceived" => { + if let Ok(p) = parse_quiet::(&ev.params) { + let hops = self.hops.entry(p.request_id.clone()).or_default(); + if hops.is_empty() { + hops.push(Hop::default()); + } + let n = hops.len() - 1; + let hop = &mut hops[n]; + hop.status = Some(p.response.status); + hop.content_type = p.response.headers.content_type().map(str::to_owned); + hop.set_cookie |= p.response.headers.has_set_cookie(); + self.push_response(&p.request_id, n); + } + } + "Network.responseReceivedExtraInfo" => { + if let Ok(p) = parse_quiet::(&ev.params) { + let hops = self.hops.entry(p.request_id.clone()).or_default(); + if hops.is_empty() { + hops.push(Hop::default()); + } + // One per hop, in order: the first hop that has not had its own and + // does not contradict this one's status. + let n = hops + .iter() + .position(|h| { + !h.extra_seen + && (h.status.is_none() + || p.status_code.is_none() + || h.status == p.status_code) + }) + .unwrap_or(hops.len() - 1); + let hop = &mut hops[n]; + hop.extra_seen = true; + hop.status = hop.status.or(p.status_code); + if hop.content_type.is_none() { + hop.content_type = p.headers.content_type().map(str::to_owned); + } + hop.set_cookie |= p.headers.has_set_cookie(); + self.push_response(&p.request_id, n); + } + } + _ => {} + } + } + } + Flow::Continue + } + + async fn attached(&mut self, p: AttachedToTarget) { + let is_page = match p.target_info.kind.as_str() { + "page" => true, + "iframe" => false, + _ => { + // Workers and the rest: let them run, record nothing from them. + let _ = send( + &self.cdp, + "Runtime.runIfWaitingForDebugger", + &Empty {}, + Some(&p.session_id), + ) + .await; + return; + } + }; + let is_first = is_page && self.first_session.is_none(); + if is_page { + self.page_targets.insert(p.target_info.target_id.clone()); + if is_first { + self.first_session = Some(p.session_id.clone()); + } else { + self.capture.push(CaptureEvent::Popup); + } + } + self.sessions.insert( + p.session_id.clone(), + Sess { + target_id: p.target_info.target_id, + script_id: None, + }, + ); + // A page that vanishes while it is being set up is not an error for the session. + let _ = self.setup(&p.session_id).await; + let _ = send( + &self.cdp, + "Runtime.runIfWaitingForDebugger", + &Empty {}, + Some(&p.session_id), + ) + .await; + } + + fn is_first_main(&self, session: &str, parent: Option<&str>) -> bool { + self.first_session.as_deref() == Some(session) && parent.is_none() + } + + async fn navigated(&mut self, session: &str, p: FrameNavigated) { + let main = self.is_first_main(session, p.frame.parent_id.as_deref()); + if main { + self.first_main_frame = Some(p.frame.id.clone()); + if self.clock.is_none() { + let url = p.frame.url.as_str(); + if url == self.notice || url.starts_with("data:") || url == "about:blank" { + return; + } + self.start(session).await; + } + } + if self.clock.is_some() { + self.capture.push(CaptureEvent::Navigation { + t_ms: self.t_ms(), + frame: RawFrame { + main, + url: p.frame.url, + }, + }); + } + } + + /// The main frame left the notice page. + async fn start(&mut self, session: &str) { + self.clock = Some((Instant::now(), SystemTime::now())); + let version: Result = + self.cdp.call("Browser.getVersion", &Empty {}, None).await; + let metrics: Result = self + .cdp + .call("Page.getLayoutMetrics", &Empty {}, Some(session)) + .await; + let (product, user_agent) = version + .map(|v| (v.product, v.user_agent)) + .unwrap_or_default(); + let (width, height) = metrics.map_or((0, 0), |m| { + ( + m.css_visual_viewport.client_width.round() as u32, + m.css_visual_viewport.client_height.round() as u32, + ) + }); + self.capture.push(CaptureEvent::Started { + browser: BrowserInfo { + product, + user_agent, + viewport: Viewport { width, height }, + }, + listener_world: self.world, + }); + } + + fn binding(&mut self, session: &str, p: &BindingCalled) { + if p.name != BINDING { + return; + } + if parse_quiet::(&p.payload).is_ok_and(|h| h.hello) { + self.hello = true; + return; + } + let Ok(r) = parse_quiet::(&p.payload) else { + return; + }; + if !r.within_limits() || self.clock.is_none() || !self.sessions.contains_key(session) { + return; + } + let kind = match r.kind { + ReportKind::Input => PageEventKind::Input, + ReportKind::Change => PageEventKind::Change, + ReportKind::Click => PageEventKind::Click, + ReportKind::Submit => PageEventKind::Submit, + ReportKind::EnterKey => PageEventKind::EnterKey, + }; + let e = r.element; + let element = RawElement { + element_id: e.element_id, + document_id: r.document_id, + tag: e.tag, + input_type: e.input_type, + id: e.id, + name: e.name, + autocomplete: e.autocomplete, + placeholder: e.placeholder, + aria_label: e.aria_label, + is_password_now: e.is_password_now, + form: e.form.map(|f| RawForm { + index: f.index, + action: f.action, + method: f.method, + }), + selectors: e.selectors, + in_shadow_root: e.in_shadow_root, + }; + let value = match kind { + PageEventKind::Input | PageEventKind::Change => r.value, + _ => None, + }; + self.capture.push(CaptureEvent::Page { + t_ms: self.t_ms(), + kind, + frame: RawFrame { + main: r.is_top, + url: r.frame_url, + }, + element, + value, + }); + } + + fn push_response(&mut self, id: &str, n: usize) { + let Some(hop) = self.hops.get(id).and_then(|h| h.get(n)) else { + return; + }; + let Some(status) = hop.status else { return }; + self.capture.push(CaptureEvent::Response { + request_id: format!("{id}#{n}"), + status, + content_type: hop.content_type.clone(), + set_cookie: hop.set_cookie, + }); + } + + async fn request(&mut self, session: &str, p: RequestWillBeSent) { + let id = p.request_id.clone(); + if let Some(prev) = p.redirect_response { + let hops = self.hops.entry(id.clone()).or_default(); + if hops.is_empty() { + hops.push(Hop::default()); + } + let last = hops.len() - 1; + let hop = &mut hops[last]; + hop.status = Some(prev.status); + if hop.content_type.is_none() { + hop.content_type = prev.headers.content_type().map(str::to_owned); + } + hop.set_cookie |= prev.headers.has_set_cookie(); + hops.push(Hop::default()); + self.push_response(&id, last); + } else { + self.hops.insert(id.clone(), vec![Hop::default()]); + } + let n = self.hops.get(&id).map_or(0, |h| h.len() - 1); + if self.clock.is_none() { + return; + } + + let resource = match p.resource.as_deref() { + Some("Document") => ResourceKind::Document, + Some("XHR") => ResourceKind::Xhr, + Some("Fetch") => ResourceKind::Fetch, + _ => ResourceKind::Other, + }; + let main_frame_navigation = resource == ResourceKind::Document + && p.frame_id.is_some() + && p.frame_id == self.first_main_frame + && p.request_id == p.loader_id; + let body = match (&p.request.post_data, p.request.has_post_data) { + (Some(d), _) => RawBody::Bytes(Zeroizing::new(d.as_bytes().to_vec())), + (None, Some(true)) => { + match self + .cdp + .call::<_, PostData>( + "Network.getRequestPostData", + &GetRequestPostData { request_id: &id }, + Some(session), + ) + .await + { + Ok(d) => RawBody::Bytes(Zeroizing::new(d.post_data.as_bytes().to_vec())), + Err(_) => RawBody::Unavailable, + } + } + _ => RawBody::None, + }; + self.capture.push(CaptureEvent::Request { + request_id: format!("{id}#{n}"), + t_ms: self.t_ms(), + method: p.request.method.clone(), + url: p.request.url.clone(), + resource, + main_frame_navigation, + content_type: p.request.headers.content_type().map(str::to_owned), + authorization: p + .request + .headers + .authorization() + .map(|a| Zeroizing::new(a.as_bytes().to_vec())), + body, + frame_url: p.document_url.clone(), + }); + } + + fn finish(self, confirmed: bool) -> Outcome { + let Some((_, started)) = self.clock else { + return Outcome::NeverStarted; + }; + let info = EndInfo { + ended_by: if confirmed { + EndedBy::Confirmed + } else { + EndedBy::WindowClosed + }, + success_confirmed: confirmed, + started_at: utc_rfc3339(started), + ended_at: utc_rfc3339(SystemTime::now()), + }; + Outcome::Recorded(Box::new(self.capture.finish(info))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn times_are_rfc3339_utc() { + assert_eq!(utc_rfc3339(SystemTime::UNIX_EPOCH), "1970-01-01T00:00:00Z"); + let now = utc_rfc3339(SystemTime::now()); + assert!(now.ends_with('Z') && now.len() >= 20, "{now}"); + } + + #[test] + fn the_listener_script_is_embedded_and_names_the_binding() { + assert!(LISTENER_JS.contains(BINDING)); + } +} diff --git a/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs b/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs new file mode 100644 index 0000000..5a73db1 --- /dev/null +++ b/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs @@ -0,0 +1,24 @@ +//! SPEC req_tch_002: nothing a person typed reaches the recording. + +mod support; + +use kharon_teach::testing::assert_holds_no_typed_value; +use support::{Login, PASS, USER, record_login}; + +#[tokio::test] +async fn no_typed_value_in_the_recording_for_any_fixture() { + for (name, which) in [ + ("form", Login::Form), + ("script", Login::Script), + ("two_step", Login::TwoStep), + ("hashed", Login::Hashed), + ] { + let Some(r) = record_login(&format!("no_typed_value_{name}"), which).await else { + return; + }; + let json = r.to_json(); + // The file is a real one: the typed fields are in it as lengths, not as text. + assert!(json.contains("\"value_len\""), "{name}: no input steps"); + assert_holds_no_typed_value(&json, &[USER, PASS]); + } +} diff --git a/crates/kharon-teach/tests/req_tch_005_shapes.rs b/crates/kharon-teach/tests/req_tch_005_shapes.rs new file mode 100644 index 0000000..2f981be --- /dev/null +++ b/crates/kharon-teach/tests/req_tch_005_shapes.rs @@ -0,0 +1,118 @@ +//! SPEC req_tch_005: the recorder tells the shapes of login apart in a real browser. + +mod support; + +use kharon_recording::{ + EndedBy, ListenerWorld, PasswordLeg, Recording, ResourceKind, RungCeiling, Shape, +}; +use kharon_teach::session::{End, Outcome}; +use support::{COOKIE_VALUE, Login, Run, record_login}; + +#[tokio::test] +async fn form_page_classifies_as_form_post() { + let Some(r) = record_login("form_page", Login::Form).await else { + return; + }; + let c = &r.classification; + assert_eq!(c.shape, Shape::FormPost, "{:?}", c.notes); + assert_eq!(c.rung_ceiling, RungCeiling::Milliseconds); + assert!(c.network_swap_candidate); + assert_eq!(r.ended_by, EndedBy::Confirmed); + assert!(r.success_confirmed); + assert_eq!(r.listener_world, ListenerWorld::Isolated); + assert!(!r.browser.product.is_empty()); + assert!(r.browser.viewport.width > 0 && r.browser.viewport.height > 0); + + // The 302 that sets a cookie is stored with its response, and the cookie's value is not. + let post = r + .requests + .iter() + .find(|q| q.method == "POST" && q.url.path == "/form-login") + .expect("the login POST is stored"); + assert_eq!(post.resource, ResourceKind::Document); + let resp = post.response.as_ref().expect("the 302 is stored with it"); + assert_eq!(resp.status, 302); + assert!(resp.set_cookie); + assert!(!r.to_json().contains(COOKIE_VALUE)); +} + +#[tokio::test] +async fn script_page_classifies_as_script_post() { + let Some(r) = record_login("script_page", Login::Script).await else { + return; + }; + assert_eq!( + r.classification.shape, + Shape::ScriptPost, + "{:?}", + r.classification.notes + ); +} + +#[tokio::test] +async fn two_step_page_classifies_as_two_step() { + let Some(r) = record_login("two_step_page", Login::TwoStep).await else { + return; + }; + let c = &r.classification; + assert_eq!( + c.shape, + Shape::TwoStep { + password_leg: PasswordLeg::FormPost + }, + "{:?}", + c.notes + ); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); +} + +#[tokio::test] +async fn hashed_page_classifies_as_transformed() { + let Some(r) = record_login("hashed_page", Login::Hashed).await else { + return; + }; + let c = &r.classification; + assert_eq!( + c.shape, + Shape::Transformed { + digest: Some("sha256".to_owned()) + }, + "{:?}", + c.notes + ); + assert!(!c.network_swap_candidate); +} + +/// The first selector of an element is one a later visit can use to find the same field. +#[tokio::test] +async fn the_first_selector_of_each_typed_element_matches() { + let Some(r) = record_login("first_selector", Login::Form).await else { + return; + }; + let typed = typed_elements(&r); + assert_eq!(typed.len(), 2, "two typed fields"); + for sel in &typed { + assert!(!sel.is_empty()); + } + // Re-open the fixture and check each selector finds exactly the field it describes. + let Some(run) = Run::start("first_selector_reopen", "/form").await else { + return; + }; + run.begin("/form").await; + for (sel, want_name) in typed.iter().zip(["user", "pass"]) { + let found = run.person.attr_of(sel, "name").await; + assert_eq!(found.as_deref(), Some(want_name), "selector {sel}"); + } + assert!(matches!(run.finish(End::Discard).await, Outcome::Discarded)); +} + +fn typed_elements(r: &Recording) -> Vec { + use kharon_recording::Step; + r.steps + .iter() + .filter_map(|s| match s { + Step::Input { element, .. } => element.selectors.first().cloned(), + _ => None, + }) + .collect() +} diff --git a/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs new file mode 100644 index 0000000..1dfa283 --- /dev/null +++ b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs @@ -0,0 +1,201 @@ +//! SPEC req_tch_006: the browser keeps nothing from a login, and the profile is always removed. + +mod support; + +use kharon_recording::EndedBy; +use kharon_teach::session::{End, Outcome}; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use support::{PASS, Run, USER}; + +/// Row counts of `tables` in a copy of the SQLite file at `db`; a missing table counts as 0. +/// `None` when `python3` is absent. +fn row_counts(db: &Path, tables: &[&str]) -> Option> { + static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + let copy = std::env::temp_dir().join(format!( + "kharon-teach-test-copy-{}-{}", + std::process::id(), + N.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + )); + let _ = std::fs::remove_file(©); + std::fs::copy(db, ©).unwrap(); + let script = [ + "import sqlite3,sys", + "c=sqlite3.connect(sys.argv[1])", + "for t in sys.argv[2:]:", + " try:", + " print(c.execute('select count(*) from ' + t).fetchone()[0])", + " except sqlite3.Error:", + " print(0)", + ] + .join("\n"); + let out = std::process::Command::new("python3") + .args(["-I", "-c", &script]) + .arg(©) + .args(tables) + .output(); + let _ = std::fs::remove_file(©); + let out = match out { + Ok(o) if o.status.success() => o, + Ok(o) => panic!( + "python3 failed reading {}: {} {}", + db.display(), + o.status, + String::from_utf8_lossy(&o.stderr) + ), + Err(_) => { + eprintln!("SKIP: python3 is not installed"); + return None; + } + }; + Some( + String::from_utf8_lossy(&out.stdout) + .lines() + .map(|l| l.trim().parse().unwrap()) + .collect(), + ) +} + +async fn gone(path: &Path) -> bool { + for _ in 0..100 { + if !path.exists() { + return true; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + false +} + +async fn type_a_login(run: &Run) { + run.begin("/form").await; + run.person.type_into("input[name=user]", USER).await; + run.person.type_into("input[name=pass]", PASS).await; +} + +#[tokio::test] +async fn nothing_is_saved_in_the_profile_during_a_login() { + let Some(run) = Run::start("nothing_saved", "/form").await else { + return; + }; + type_a_login(&run).await; + run.person.click("#go").await; + run.person.wait_for_path("/home").await; + let profile = run.ready.profile_dir.clone(); + assert!(profile.is_dir(), "the profile directory exists"); + for (file, tables) in [ + ("Login Data", &["logins"][..]), + ( + "Web Data", + &["autofill", "autofill_profiles", "credit_cards"][..], + ), + ] { + let db = profile.join("Default").join(file); + if !db.exists() { + eprintln!("no {file} in the profile: nothing was saved"); + continue; + } + if let Some(counts) = row_counts(&db, tables) { + assert!(counts.iter().all(|n| *n == 0), "{file}: {counts:?}"); + } + } + let _ = run.finish(End::Discard).await; +} + +#[tokio::test] +async fn profile_is_gone_after_confirm() { + let Some(run) = Run::start("gone_after_confirm", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + type_a_login(&run).await; + run.person.click("#go").await; + run.person.wait_for_path("/home").await; + assert!(matches!( + run.finish(End::Confirmed).await, + Outcome::Recorded(_) + )); + assert!(gone(&profile).await, "the profile is still there"); +} + +#[tokio::test] +async fn profile_is_gone_after_discard() { + let Some(run) = Run::start("gone_after_discard", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + type_a_login(&run).await; + assert!(matches!(run.finish(End::Discard).await, Outcome::Discarded)); + assert!(gone(&profile).await, "the profile is still there"); +} + +#[tokio::test] +async fn closing_the_window_ends_unconfirmed() { + let Some(mut run) = Run::start("close_mid_login", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + type_a_login(&run).await; + let begun = std::time::Instant::now(); + run.person.close_all_pages().await; + match run.wait().await { + Outcome::Recorded(r) => { + assert_eq!(r.ended_by, EndedBy::WindowClosed); + assert!(!r.success_confirmed); + } + _ => panic!("expected a recording"), + } + assert!(begun.elapsed() < Duration::from_secs(10)); + assert!(gone(&profile).await, "the profile is still there"); +} + +#[tokio::test] +async fn closing_before_start_is_never_started() { + let Some(mut run) = Run::start("close_before_start", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + run.person.close_all_pages().await; + assert!(matches!(run.wait().await, Outcome::NeverStarted)); + assert!(gone(&profile).await, "the profile is still there"); +} + +#[tokio::test] +async fn dropping_the_future_removes_the_profile() { + let Some(run) = Run::start("drop_future", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + let port = run.ready.devtools_port; + type_a_login(&run).await; + // The Ctrl-C path: the task is dropped mid-recording. + run.abort(); + assert!(gone(&profile).await, "the profile is still there"); + // The browser went with it: its DevTools port stops answering. + let mut closed = false; + for _ in 0..100 { + if tokio::net::TcpStream::connect(("127.0.0.1", port)) + .await + .is_err() + { + closed = true; + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!(closed, "the browser is still listening"); +} + +#[tokio::test] +async fn a_stale_profile_is_swept_at_the_next_start() { + let stale: PathBuf = + std::env::temp_dir().join(format!("kharon-teach-stale-test-{}", std::process::id())); + std::fs::create_dir_all(stale.join("Default")).unwrap(); + std::fs::write(stale.join("Default").join("Preferences"), "{}").unwrap(); + let Some(run) = Run::start("stale_swept", "/form").await else { + let _ = std::fs::remove_dir_all(&stale); + return; + }; + // Tests in this file start at the same time, and any of them may have done the sweep. + assert!(!stale.exists(), "the stale profile was not swept"); + let _ = run.finish(End::Discard).await; +} diff --git a/crates/kharon-teach/tests/support/mod.rs b/crates/kharon-teach/tests/support/mod.rs index e36ff4e..aaa220c 100644 --- a/crates/kharon-teach/tests/support/mod.rs +++ b/crates/kharon-teach/tests/support/mod.rs @@ -31,3 +31,570 @@ pub fn chromium_or_skip(test: &str) -> Option { } bin } + +// ---- the fixture login server, the person at the keyboard, and one recording run ---- + +use kharon_teach::browser::DevTools; +use kharon_teach::cdp::Cdp; +use kharon_teach::session::{End, Outcome, Ready, RecordOptions, SessionError, record}; +use serde::{Deserialize, Serialize}; +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::{Semaphore, oneshot}; +use tokio::task::JoinHandle; + +pub const USER: &str = "kh.throwaway7"; +pub const PASS: &str = "Tr0ub4dor-9x"; +/// Set by the fixture on a successful login; must never reach a recording. +pub const COOKIE_VALUE: &str = "COOKIEVALUE123"; + +/// Browsers running at once. Parallel Chromium instances on a small machine make timing flaky. +static GATE: Semaphore = Semaphore::const_new(2); + +const FORM: &str = "form
\ +
"; + +const SCRIPT: &str = "script\ +"; + +const HASHED: &str = "hashed\ +"; + +const TWO_STEP: &str = "step one
\ +
"; + +const TWO_STEP_PASSWORD: &str = "step two
\ +
"; + +const HOME: &str = "home

signed in

"; + +fn response(status: &str, extra: &str, body: &str, ctype: &str) -> String { + format!( + "HTTP/1.1 {status}\r\n{extra}Content-Type: {ctype}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) +} + +fn route(method: &str, path: &str) -> String { + let html = "text/html; charset=utf-8"; + match (method, path) { + ("GET", "/form") => response("200 OK", "", FORM, html), + ("POST", "/form-login") => response( + "302 Found", + &format!("Location: /home\r\nSet-Cookie: sid={COOKIE_VALUE}; Path=/\r\n"), + "", + html, + ), + ("GET", "/script") => response("200 OK", "", SCRIPT, html), + ("POST", "/api/login") => response("200 OK", "", "{\"ok\":true}", "application/json"), + ("GET", "/two-step") => response("200 OK", "", TWO_STEP, html), + ("POST", "/two-step/password") => response("200 OK", "", TWO_STEP_PASSWORD, html), + ("GET", "/hashed") => response("200 OK", "", HASHED, html), + ("GET", "/home") => response("200 OK", "", HOME, html), + _ => response("404 Not Found", "", "not found", "text/plain"), + } +} + +async fn serve_one(mut sock: tokio::net::TcpStream) { + let mut buf = Vec::new(); + let mut chunk = [0u8; 4096]; + let head_end = loop { + if let Some(i) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + break i + 4; + } + match sock.read(&mut chunk).await { + Ok(0) | Err(_) => return, + Ok(n) => buf.extend_from_slice(&chunk[..n]), + } + }; + let head = String::from_utf8_lossy(&buf[..head_end]).into_owned(); + let mut first = head.lines().next().unwrap_or("").split_whitespace(); + let (method, path) = (first.next().unwrap_or(""), first.next().unwrap_or("")); + let want: usize = head + .lines() + .find_map(|l| { + l.to_ascii_lowercase() + .strip_prefix("content-length:") + .and_then(|v| v.trim().parse().ok()) + }) + .unwrap_or(0); + while buf.len() < head_end + want { + match sock.read(&mut chunk).await { + Ok(0) | Err(_) => break, + Ok(n) => buf.extend_from_slice(&chunk[..n]), + } + } + let path = path.split('?').next().unwrap_or(""); + let _ = sock.write_all(route(method, path).as_bytes()).await; + let _ = sock.shutdown().await; +} + +pub struct Fixture { + pub port: u16, + task: JoinHandle<()>, +} + +impl Fixture { + pub async fn start() -> Fixture { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let task = tokio::spawn(async move { + while let Ok((sock, _)) = listener.accept().await { + tokio::spawn(serve_one(sock)); + } + }); + Fixture { port, task } + } + + pub fn url(&self, path: &str) -> url::Url { + url::Url::parse(&format!("http://127.0.0.1:{}{path}", self.port)).unwrap() + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + self.task.abort(); + } +} + +// ---- the person ---- + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct VersionDoc { + web_socket_debugger_url: String, +} + +#[derive(Deserialize)] +struct Targets { + #[serde(rename = "targetInfos")] + infos: Vec, +} + +#[derive(Deserialize)] +struct TargetRow { + #[serde(rename = "targetId")] + id: String, + #[serde(rename = "type")] + kind: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct TargetIdParam<'a> { + target_id: &'a str, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct AttachParam<'a> { + target_id: &'a str, + flatten: bool, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Attached { + session_id: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct EvalParam<'a> { + expression: &'a str, + return_by_value: bool, +} + +#[derive(Deserialize)] +struct EvalResult { + result: EvalValue, +} + +#[derive(Deserialize)] +struct EvalValue { + #[serde(default)] + value: Option, +} + +#[derive(Serialize)] +struct TextParam<'a> { + text: &'a str, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct MouseParam { + #[serde(rename = "type")] + kind: &'static str, + x: f64, + y: f64, + button: &'static str, + click_count: u32, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct KeyParam { + #[serde(rename = "type")] + kind: &'static str, + key: &'static str, + code: &'static str, + windows_virtual_key_code: u32, + #[serde(skip_serializing_if = "Option::is_none")] + text: Option<&'static str>, +} + +#[derive(Serialize)] +struct UrlParam<'a> { + url: &'a str, +} + +#[derive(Deserialize)] +struct Ignored {} + +async fn get_version_doc(port: u16) -> VersionDoc { + let mut s = tokio::net::TcpStream::connect(("127.0.0.1", port)) + .await + .unwrap(); + s.write_all(b"GET /json/version HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + let mut text = String::new(); + let mut buf = [0u8; 4096]; + tokio::time::timeout(Duration::from_secs(10), async { + while !text.trim_end().ends_with('}') { + let n = s.read(&mut buf).await.unwrap(); + assert!(n > 0, "closed early"); + text.push_str(&String::from_utf8_lossy(&buf[..n])); + } + }) + .await + .expect("no /json/version answer"); + let body = &text[text.find("\r\n\r\n").unwrap() + 4..]; + serde_json::from_str(body).unwrap() +} + +/// A second DevTools connection that does what a person does in the window. +pub struct Person { + cdp: Cdp, + session: String, +} + +impl Person { + pub async fn connect(port: u16) -> Person { + let doc = get_version_doc(port).await; + let at = doc.web_socket_debugger_url.find("/devtools/").unwrap(); + let dev = DevTools { + port, + ws_path: doc.web_socket_debugger_url[at..].to_owned(), + }; + let (cdp, rx) = Cdp::connect(&dev).await.unwrap(); + // Nobody reads this connection's events; keep the channel open. + std::mem::forget(rx); + let page = Self::first_page(&cdp).await.expect("no page target"); + let session: Attached = cdp + .call( + "Target.attachToTarget", + &AttachParam { + target_id: &page, + flatten: true, + }, + None, + ) + .await + .unwrap(); + Person { + cdp, + session: session.session_id, + } + } + + async fn first_page(cdp: &Cdp) -> Option { + let t: Targets = cdp + .call("Target.getTargets", &Ignored2 {}, None) + .await + .ok()?; + t.infos.into_iter().find(|t| t.kind == "page").map(|t| t.id) + } + + async fn eval(&self, expression: &str) -> Option { + let r: EvalResult = self + .cdp + .call( + "Runtime.evaluate", + &EvalParam { + expression, + return_by_value: true, + }, + Some(&self.session), + ) + .await + .ok()?; + r.result.value + } + + pub async fn goto(&self, url: &str) { + let _: Ignored = self + .cdp + .call("Page.navigate", &UrlParam { url }, Some(&self.session)) + .await + .unwrap(); + } + + pub async fn type_into(&self, css: &str, text: &str) { + let sel = serde_json::to_string(css).unwrap(); + let focus = format!("(()=>{{document.querySelector({sel}).focus();return 'ok'}})()"); + assert_eq!( + self.eval(&focus).await.as_deref(), + Some("ok"), + "focus {css}" + ); + let _: Ignored = self + .cdp + .call("Input.insertText", &TextParam { text }, Some(&self.session)) + .await + .unwrap(); + } + + pub async fn press_enter(&self) { + for (kind, text) in [("keyDown", Some("\r")), ("keyUp", None)] { + let _: Ignored = self + .cdp + .call( + "Input.dispatchKeyEvent", + &KeyParam { + kind, + key: "Enter", + code: "Enter", + windows_virtual_key_code: 13, + text, + }, + Some(&self.session), + ) + .await + .unwrap(); + } + } + + pub async fn click(&self, css: &str) { + let sel = serde_json::to_string(css).unwrap(); + let centre = format!( + "(()=>{{const r=document.querySelector({sel}).getBoundingClientRect();\ + return (r.x+r.width/2)+','+(r.y+r.height/2)}})()" + ); + let at = self.eval(¢re).await.expect("element centre"); + let (x, y) = at.split_once(',').unwrap(); + let (x, y): (f64, f64) = (x.parse().unwrap(), y.parse().unwrap()); + for (kind, count) in [("mousePressed", 1), ("mouseReleased", 1)] { + let _: Ignored = self + .cdp + .call( + "Input.dispatchMouseEvent", + &MouseParam { + kind, + x, + y, + button: "left", + click_count: count, + }, + Some(&self.session), + ) + .await + .unwrap(); + } + } + + pub async fn wait_for_path(&self, path: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + if self.eval("location.pathname").await.as_deref() == Some(path) { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "the page never reached {path}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + pub async fn close_all_pages(&self) { + let t: Targets = self + .cdp + .call("Target.getTargets", &Ignored2 {}, None) + .await + .unwrap(); + for p in t.infos.into_iter().filter(|t| t.kind == "page") { + let _: Result = self + .cdp + .call( + "Target.closeTarget", + &TargetIdParam { target_id: &p.id }, + None, + ) + .await; + } + } +} + +#[derive(Serialize)] +struct Ignored2 {} + +// ---- one recording run ---- + +pub struct Run { + pub fixture: Fixture, + pub person: Person, + pub ready: Ready, + end: Option>, + task: JoinHandle>, + _permit: tokio::sync::SemaphorePermit<'static>, +} + +impl Run { + /// Starts `record` headless on the fixture's `login_path` and waits until the notice page is + /// showing. `None` when there is no browser and the run does not require one. + pub async fn start(test: &str, login_path: &str) -> Option { + let bin = chromium_or_skip(test)?; + let permit = GATE.acquire().await.unwrap(); + let fixture = Fixture::start().await; + let login = fixture.url(login_path); + let (ready_tx, ready_rx) = oneshot::channel(); + let (end_tx, end_rx) = oneshot::channel(); + let task = tokio::spawn(async move { + record( + RecordOptions { + login_url: &login, + browser: &bin, + headless: true, + tool_version: "test", + }, + ready_tx, + end_rx, + ) + .await + }); + let ready = match tokio::time::timeout(Duration::from_secs(40), ready_rx) + .await + .expect("the session was not ready in 40 s") + { + Ok(r) => r, + Err(_) => { + let ended = task.await.expect("the session task panicked"); + match ended { + Err(e) => panic!("the session failed before it was ready: {e:?}"), + Ok(Outcome::NeverStarted) => panic!("the window closed before it was ready"), + Ok(_) => panic!("the session ended before it was ready"), + } + } + }; + let person = Person::connect(ready.devtools_port).await; + Some(Run { + fixture, + person, + ready, + end: Some(end_tx), + task, + _permit: permit, + }) + } + + /// What a person does: click the notice page's link, then the login page is open. + pub async fn begin(&self, login_path: &str) { + self.person.click("a").await; + self.person.wait_for_path(login_path).await; + } + + pub async fn finish(mut self, how: End) -> Outcome { + if let Some(tx) = self.end.take() { + let _ = tx.send(how); + } + self.wait().await + } + + /// The outcome of a session that ends by itself (the window closing). + pub async fn wait(&mut self) -> Outcome { + tokio::time::timeout(Duration::from_secs(10), &mut self.task) + .await + .expect("the session did not end in 10 s") + .expect("the session task panicked") + .expect("the session failed") + } + + pub fn abort(&self) { + self.task.abort(); + } +} + +/// The four fixture logins. +#[derive(Clone, Copy)] +pub enum Login { + Form, + Script, + TwoStep, + Hashed, +} + +impl Login { + pub fn path(self) -> &'static str { + match self { + Login::Form => "/form", + Login::Script => "/script", + Login::TwoStep => "/two-step", + Login::Hashed => "/hashed", + } + } +} + +/// Starts a run, does what a person does on `which`, confirms, and hands back the recording. +/// `None` when the run is skipped for want of a browser. +pub async fn record_login(test: &str, which: Login) -> Option { + let run = Run::start(test, which.path()).await?; + run.begin(which.path()).await; + let p = &run.person; + match which { + Login::Form | Login::Script | Login::Hashed => { + p.type_into("input[name=user]", USER).await; + p.type_into("input[name=pass]", PASS).await; + p.click("#go").await; + } + Login::TwoStep => { + p.type_into("input[name=user]", USER).await; + p.click("#go").await; + p.wait_for_path("/two-step/password").await; + p.type_into("input[name=pass]", PASS).await; + p.click("#go").await; + } + } + p.wait_for_path("/home").await; + match run.finish(End::Confirmed).await { + Outcome::Recorded(r) => Some(*r), + _ => panic!("expected a recording"), + } +} + +impl Person { + /// The `attr` attribute of the element `css` matches, or `None` when it matches nothing. + pub async fn attr_of(&self, css: &str, attr: &str) -> Option { + let sel = serde_json::to_string(css).unwrap(); + let attr = serde_json::to_string(attr).unwrap(); + self.eval(&format!( + "(()=>{{const e=document.querySelector({sel});return e?e.getAttribute({attr}):null}})()" + )) + .await + } +} From 7319ec4ddf422a7428ff202336e2ac6e0e541180 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 06:12:43 -0500 Subject: [PATCH 22/36] feat(teach): kharon-teach command line, consent and the recording file (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- Cargo.lock | 2 + crates/kharon-teach/Cargo.toml | 4 + crates/kharon-teach/src/browser.rs | 37 +- crates/kharon-teach/src/cli.rs | 453 +++++++++++++++ crates/kharon-teach/src/lib.rs | 528 ++++++++++++++++++ crates/kharon-teach/src/main.rs | 22 +- crates/kharon-teach/src/notice.rs | 7 + .../tests/req_tch_002_no_typed_value.rs | 14 + .../tests/req_tch_003_file_contents.rs | 45 ++ .../tests/req_tch_004_no_network.rs | 105 ++++ .../tests/req_tch_006_profile_and_visible.rs | 119 ++++ crates/kharon-teach/tests/support/mod.rs | 113 ++++ 12 files changed, 1445 insertions(+), 4 deletions(-) create mode 100644 crates/kharon-teach/src/cli.rs create mode 100644 crates/kharon-teach/tests/req_tch_003_file_contents.rs create mode 100644 crates/kharon-teach/tests/req_tch_004_no_network.rs diff --git a/Cargo.lock b/Cargo.lock index 5f50570..e5b3331 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1084,6 +1084,7 @@ name = "kharon-teach" version = "0.0.0" dependencies = [ "base64ct", + "clap", "futures-util", "hex", "kharon-recording", @@ -1092,6 +1093,7 @@ dependencies = [ "percent-encoding", "publicsuffix", "rand", + "rustix", "serde", "serde_json", "sha1", diff --git a/crates/kharon-teach/Cargo.toml b/crates/kharon-teach/Cargo.toml index 07bbad5..89f16a5 100644 --- a/crates/kharon-teach/Cargo.toml +++ b/crates/kharon-teach/Cargo.toml @@ -34,7 +34,11 @@ tokio = { version = "1", features = ["fs", "macros", "net", "process", "rt-multi publicsuffix = "2.3" tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "handshake"] } time = { version = "0.3", features = ["formatting"] } +clap = { version = "4", features = ["derive"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } [dev-dependencies] tokio = { version = "1", features = ["test-util"] } + +[target.'cfg(unix)'.dependencies] +rustix = { version = "1", default-features = false, features = ["std", "process"] } diff --git a/crates/kharon-teach/src/browser.rs b/crates/kharon-teach/src/browser.rs index 00f46b5..96e4b5c 100644 --- a/crates/kharon-teach/src/browser.rs +++ b/crates/kharon-teach/src/browser.rs @@ -183,8 +183,8 @@ impl Profile { /// [`Profile::sweep_stale`] over `dir`. pub(crate) fn sweep_stale_in(dir: &Path) -> std::io::Result { use std::os::unix::fs::MetadataExt; - // /proc/self is owned by this process's effective uid. - let me = std::fs::metadata("/proc/self")?.uid(); + // Not /proc/self: a non-dumpable process's /proc entries belong to root. + let me = rustix::process::geteuid().as_raw(); let mut n = 0; for entry in std::fs::read_dir(dir)? { let Ok(entry) = entry else { continue }; @@ -564,6 +564,39 @@ mod tests { assert!(!a.exists() && !b.exists()); } + /// A non-dumpable process's `/proc/self` belongs to root, so a sweep that learned its uid + /// from there would compare against 0 and never remove anything. The check runs in a child + /// process (this test binary again) so the parent stays dumpable. + #[test] + fn sweep_still_removes_a_leftover_when_the_process_is_non_dumpable() { + const MARK: &str = "KHARON_TEACH_SWEEP_NONDUMPABLE_DIR"; + if let Some(dir) = std::env::var_os(MARK) { + use rustix::process::{DumpableBehavior, set_dumpable_behavior}; + set_dumpable_behavior(DumpableBehavior::NotDumpable).unwrap(); + let removed = Profile::sweep_stale_in(Path::new(&dir)).unwrap(); + assert_eq!(removed, 1, "the sweep did not remove the leftover"); + return; + } + let sb = Sandbox::new("nondump"); + let d = sb.mkdir("kharon-teach-leftover"); + std::fs::write(d.join("f"), b"x").unwrap(); + let out = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "browser::tests::sweep_still_removes_a_leftover_when_the_process_is_non_dumpable", + "--test-threads=1", + ]) + .env(MARK, sb.path()) + .output() + .unwrap(); + assert!( + out.status.success(), + "child failed: {}", + String::from_utf8_lossy(&out.stdout) + ); + assert!(!d.exists()); + } + #[test] fn preferences_turn_off_the_password_manager_and_autofill() { let sb = Sandbox::new("prefs"); diff --git a/crates/kharon-teach/src/cli.rs b/crates/kharon-teach/src/cli.rs new file mode 100644 index 0000000..d4f55e4 --- /dev/null +++ b/crates/kharon-teach/src/cli.rs @@ -0,0 +1,453 @@ +//! The command line: arguments, the refusals, the file name, the output path and the plain-words +//! summary of a recording. + +use kharon_recording::{ + EndedBy, ListenerWorld, PasswordLeg, Recording, RungCeiling, Shape, StrippedUrl, +}; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +/// How a recording file's name ends. +pub const FILE_SUFFIX: &str = ".kharon-recording.json"; + +#[derive(clap::Parser)] +#[command( + name = "kharon-teach", + version, + about = "Record how a login works so kharon can learn it", + args_conflicts_with_subcommands = true +)] +pub struct Cli { + #[command(subcommand)] + pub command: Option, + /// The login page to record (https, or http on this computer) + pub login_url: Option, + /// Where to write the recording (must not exist yet) + #[arg(long)] + pub out: Option, + /// The Chromium-family browser to use, instead of searching PATH + #[arg(long)] + pub browser: Option, + /// Skip the question after the notice; the notice is still shown + #[arg(long)] + pub yes: bool, +} + +#[derive(clap::Subcommand)] +pub enum Command { + /// Print what a recording file says, in plain words + Show { file: PathBuf }, +} + +/// Why the program will not start. One line each. +#[derive(Debug, thiserror::Error)] +pub enum Refusal { + #[error("give the login page's address, or `show `")] + NoUrl, + #[error("that is not a web address")] + NotAUrl, + #[error("the address must start with https:// (plain http only for this computer)")] + NotHttps, + #[error("the address must not hold a user name or password")] + Userinfo, + #[error("{0}")] + NoBrowser(String), + #[error("{} already exists; pick another --out", .0.display())] + OutExists(PathBuf), + #[error("cannot write in {}; it must be an existing directory you can write to", .0.display())] + OutDir(PathBuf), + #[error("cannot read {}: {}", .0.display(), .1)] + Unreadable(PathBuf, String), + #[error("{} is larger than 16 MiB, so it is not a recording", .0.display())] + TooLarge(PathBuf), + #[error("{} is not a recording kharon-teach can read: {}", .0.display(), .1)] + NotARecording(PathBuf, String), +} + +/// `https`, or `http` when the host is `localhost` or a loopback address; no user name or +/// password in it. +pub fn validate_login_url(s: &str) -> Result { + let u = url::Url::parse(s).map_err(|_| Refusal::NotAUrl)?; + if !u.username().is_empty() || u.password().is_some() { + return Err(Refusal::Userinfo); + } + let Some(host) = u.host() else { + return Err(Refusal::NotHttps); + }; + match u.scheme() { + "https" => Ok(u), + "http" => { + let local = match host { + url::Host::Domain(d) => d.eq_ignore_ascii_case("localhost"), + url::Host::Ipv4(a) => a.is_loopback(), + url::Host::Ipv6(a) => a.is_loopback(), + }; + if local { Ok(u) } else { Err(Refusal::NotHttps) } + } + _ => Err(Refusal::NotHttps), + } +} + +/// `-.kharon-recording.json`. Every character of the host outside +/// `[a-z0-9.-]` becomes `_`, so an IPv6 host puts no `[`, `]` or `:` in a file name. +pub fn default_file_name(host: &str, utc: SystemTime) -> String { + let host: String = host + .to_ascii_lowercase() + .chars() + .map(|c| { + if c.is_ascii_lowercase() || c.is_ascii_digit() || c == '.' || c == '-' { + c + } else { + '_' + } + }) + .collect(); + let t = time::OffsetDateTime::from(utc); + format!( + "{host}-{:04}{:02}{:02}T{:02}{:02}{:02}Z{FILE_SUFFIX}", + t.year(), + u8::from(t.month()), + t.day(), + t.hour(), + t.minute(), + t.second() + ) +} + +/// `dir` is an existing directory this user can create files in. Tried, not read from +/// permission bits: a temporary file is created and removed. +pub fn check_dir_writable(dir: &Path) -> Result<(), Refusal> { + use std::os::unix::fs::OpenOptionsExt; + let dir = if dir.as_os_str().is_empty() { + Path::new(".") + } else { + dir + }; + let bad = || Refusal::OutDir(dir.to_owned()); + if !std::fs::metadata(dir).is_ok_and(|m| m.is_dir()) { + return Err(bad()); + } + for _ in 0..4 { + let probe = dir.join(format!( + ".kharon-teach-probe-{:016x}", + rand::random::() + )); + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&probe) + { + Ok(_) => { + let _ = std::fs::remove_file(&probe); + return Ok(()); + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(_) => return Err(bad()), + } + } + Err(bad()) +} + +/// The file must not exist (a dangling symlink counts as existing) and its directory must be +/// writable. +pub fn check_out_path(p: &Path) -> Result<(), Refusal> { + if std::fs::symlink_metadata(p).is_ok() { + return Err(Refusal::OutExists(p.to_owned())); + } + check_dir_writable(p.parent().unwrap_or(Path::new(""))) +} + +fn site(u: &StrippedUrl) -> String { + format!("{}{}", u.origin, u.path) +} + +/// What a recording says, in plain words, one fact per line. It is built from the recording +/// only and holds no JSON. +pub fn summary(r: &Recording) -> String { + let mut out: Vec = Vec::new(); + out.push(format!("Site: {}", site(&r.start_url))); + out.push( + match r.ended_by { + EndedBy::Confirmed => "How it ended: you pressed Enter to say the login worked.", + EndedBy::WindowClosed => "How it ended: the window was closed before you confirmed.", + } + .to_owned(), + ); + out.push(format!( + "How the login works: {}", + match &r.classification.shape { + Shape::FormPost => "One page; the form sends the password to the site as typed.".to_owned(), + Shape::ScriptPost => + "One page; the page's script sends the password to the site as typed.".to_owned(), + Shape::TwoStep { password_leg } => format!( + "Two steps; the user name goes first, then the password, which {}.", + match password_leg { + PasswordLeg::FormPost => "a form sends to the site as typed", + PasswordLeg::ScriptPost => "the page's script sends to the site as typed", + PasswordLeg::Transformed => "the page changes before it sends it", + } + ), + Shape::Transformed { digest } => match digest { + Some(d) => format!( + "The page changes the password ({d} digest) before it sends it, so kharon cannot send it as typed." + ), + None => "The page changes the password before it sends it, so kharon cannot send it as typed.".to_owned(), + }, + Shape::Unrecognized => + "kharon-teach did not recognize how this login sends the password.".to_owned(), + } + )); + out.push(format!( + "Strongest protection: {}.", + match r.classification.rung_ceiling { + RungCeiling::Milliseconds => + "kharon can fill and submit in one step, so the password is in the page for milliseconds", + RungCeiling::AgentContinues => "the agent has to continue after the fill", + RungCeiling::NeedsPerson => "a person has to look at this recording", + } + )); + out.push( + if r.classification.network_swap_candidate { + "Network swap: this login looks like one where the password can be put in at the network level." + } else { + "Network swap: this login does not look like a candidate." + } + .to_owned(), + ); + let f = &r.flags; + for (set, words) in [ + (f.captcha, "A captcha was on the page."), + (f.otp_step, "There was a one-time-code step."), + (f.cross_site, "The login went through another site."), + (f.popup, "The login opened a pop-up window."), + (f.framed_login, "The login form was inside a frame."), + (f.shadow_dom, "The login form was inside a shadow root."), + ( + f.multiple_passwords, + "The page had more than one password field.", + ), + ( + f.body_unavailable, + "Some request bodies could not be read, so the shape may be incomplete.", + ), + ] { + if set { + out.push(format!("Flag: {words}")); + } + } + for n in &r.classification.notes { + out.push(format!("Note: {n}")); + } + if r.listener_world == ListenerWorld::Main { + out.push( + "Caution: the page listener ran in the page's own script world, where the site's scripts can see and fake what it reports. Trust this recording less." + .to_owned(), + ); + } + let s = &r.stripped; + out.push(format!( + "Removed before saving: {} typed value(s), {} query value(s), {} path segment(s); {} of {} request(s) kept, {} body(ies) reduced to a shape.", + s.typed_values_removed, + s.query_values_removed, + s.path_segments_replaced, + s.requests_stored, + s.requests_seen, + s.request_bodies_reduced + )); + let mut text = out.join("\n"); + text.push('\n'); + text +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use clap::CommandFactory; + use kharon_recording::{BrowserInfo, Classification, Flags, StrippedCounts, Viewport}; + use std::time::Duration; + + #[test] + fn accepts_https_and_loopback_http() { + for ok in [ + "https://ex.test/login", + "http://localhost:8080/", + "http://127.0.0.1:1/", + "http://[::1]:1/", + ] { + assert!(validate_login_url(ok).is_ok(), "{ok}"); + } + } + + #[test] + fn refuses_plain_http_userinfo_and_other_schemes() { + for bad in [ + "http://ex.test", + "https://u:p@ex.test", + "https://u@ex.test", + "file:///x", + "ex.test", + "ftp://ex.test/", + ] { + assert!(validate_login_url(bad).is_err(), "{bad}"); + } + assert!(matches!( + validate_login_url("https://u:p@ex.test"), + Err(Refusal::Userinfo) + )); + } + + #[test] + fn a_refusal_never_echoes_the_address() { + let e = validate_login_url("https://user:secret-pass@ex.test/").unwrap_err(); + assert!(!e.to_string().contains("secret-pass")); + } + + #[test] + fn file_name_shape() { + assert_eq!( + default_file_name( + "ex.test", + SystemTime::UNIX_EPOCH + Duration::from_secs(1_794_974_640) + ), + "ex.test-20261118T040400Z.kharon-recording.json" + ); + } + + #[test] + fn file_name_sanitizes_the_host() { + let n = default_file_name("[::1]", SystemTime::UNIX_EPOCH); + assert!(n.starts_with("___1_-19700101T000000Z"), "{n}"); + assert!(!n.contains(['[', ']', ':', '/'])); + } + + fn private_dir(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!( + "kharon-cli-test-{tag}-{}-{:08x}", + std::process::id(), + rand::random::() + )); + std::fs::create_dir(&d).unwrap(); + d + } + + #[test] + fn refuses_an_existing_out_file() { + let d = private_dir("exists"); + let f = d.join("a.json"); + assert!(check_out_path(&f).is_ok()); + std::fs::write(&f, b"x").unwrap(); + assert!(matches!(check_out_path(&f), Err(Refusal::OutExists(_)))); + std::fs::remove_dir_all(&d).unwrap(); + } + + #[test] + fn refuses_a_missing_or_read_only_directory() { + use std::os::unix::fs::PermissionsExt; + let d = private_dir("dirs"); + assert!(matches!( + check_out_path(&d.join("nope").join("a.json")), + Err(Refusal::OutDir(_)) + )); + let ro = d.join("ro"); + std::fs::create_dir(&ro).unwrap(); + std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o500)).unwrap(); + assert!(matches!( + check_out_path(&ro.join("a.json")), + Err(Refusal::OutDir(_)) + )); + // A file is not a directory. + let file = d.join("f"); + std::fs::write(&file, b"x").unwrap(); + assert!(matches!( + check_out_path(&file.join("a.json")), + Err(Refusal::OutDir(_)) + )); + std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::remove_dir_all(&d).unwrap(); + } + + pub(crate) fn recording() -> Recording { + Recording { + format_version: 1, + tool_version: "t".into(), + browser: BrowserInfo { + product: "Chrome/1".into(), + user_agent: "UA".into(), + viewport: Viewport { + width: 1, + height: 1, + }, + }, + listener_world: ListenerWorld::Isolated, + started_at: "2026-11-18T00:00:00Z".into(), + ended_at: "2026-11-18T00:01:00Z".into(), + ended_by: EndedBy::Confirmed, + success_confirmed: true, + start_url: StrippedUrl { + origin: "https://ex.test".into(), + path: "/login".into(), + query_keys: vec![], + }, + final_url: None, + steps: vec![], + requests: vec![], + flags: Flags::default(), + classification: Classification { + shape: Shape::FormPost, + rung_ceiling: RungCeiling::Milliseconds, + network_swap_candidate: false, + notes: vec![], + }, + stripped: StrippedCounts { + requests_seen: 9, + requests_stored: 4, + request_bodies_reduced: 2, + query_values_removed: 3, + path_segments_replaced: 1, + typed_values_removed: 5, + }, + } + } + + #[test] + fn summary_names_shape_ceiling_flags_and_counts() { + let mut r = recording(); + r.flags.captcha = true; + r.classification.network_swap_candidate = true; + r.classification.notes = vec!["a classifier note".into()]; + r.listener_world = ListenerWorld::Main; + let s = summary(&r); + assert!(s.contains("https://ex.test/login")); + assert!(s.contains("One page; the form sends the password to the site as typed.")); + assert!(s.contains("password is in the page for milliseconds")); + assert!(s.contains("A captcha")); + assert!(s.contains("Network swap: this login looks like")); + assert!(s.contains("Note: a classifier note")); + assert!(s.contains("Caution")); + assert!(s.contains("5 typed value(s)")); + assert!(s.contains("4 of 9 request(s) kept")); + assert!(!s.contains('{') && !s.contains('"')); + let quiet = summary(&recording()); + assert!(!quiet.contains("Caution") && !quiet.contains("Flag:")); + assert!(quiet.contains("does not look like a candidate")); + } + + #[test] + fn cli_has_no_headless_or_attach_option() { + let names: Vec<_> = Cli::command() + .get_arguments() + .map(|a| a.get_id().to_string()) + .collect(); + for banned in [ + "headless", + "attach", + "port", + "profile", + "user-data-dir", + "endpoint", + ] { + assert!(!names.iter().any(|n| n.contains(banned)), "{banned}"); + } + } +} diff --git a/crates/kharon-teach/src/lib.rs b/crates/kharon-teach/src/lib.rs index d63df93..4544a3d 100644 --- a/crates/kharon-teach/src/lib.rs +++ b/crates/kharon-teach/src/lib.rs @@ -4,6 +4,7 @@ pub mod body; pub mod browser; pub mod capture; pub mod cdp; +pub mod cli; pub mod events; pub mod marker; pub mod notice; @@ -12,3 +13,530 @@ pub mod site; pub mod strip; #[doc(hidden)] pub mod testing; + +use cli::{ + Cli, Command, Refusal, check_dir_writable, check_out_path, default_file_name, summary, + validate_login_url, +}; +use kharon_recording::Recording; +use session::{End, Outcome, Ready, RecordOptions, record}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use tokio::io::{AsyncBufRead, AsyncBufReadExt}; +use tokio::sync::oneshot; + +/// How a run ended; the process exit code is its value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Exit { + /// A file was written, or the recording was discarded, or nothing was asked for. + Done = 0, + /// Anything else. + Failed = 1, + /// A refusal: nothing was started. + Refused = 2, +} + +impl From for std::process::ExitCode { + fn from(e: Exit) -> Self { + std::process::ExitCode::from(e as u8) + } +} + +/// What tests may change. The shipped program never does: there is no flag and no environment +/// variable for it. +#[doc(hidden)] +pub struct Hooks { + pub headless: bool, + /// Receives a copy of the session's `Ready` once the notice page is showing. + pub ready: Option>, +} + +/// The whole program. Prompts and results go to `stdout`; refusals and errors to `stderr`, one +/// line each, starting `kharon-teach: `. +pub async fn run( + cli: Cli, + stdin: impl AsyncBufRead + Unpin, + stdout: impl Write, + stderr: impl Write, +) -> Exit { + run_with( + cli, + stdin, + stdout, + stderr, + Hooks { + headless: false, + ready: None, + }, + ) + .await +} + +fn fail(stderr: &mut impl Write, what: &dyn std::fmt::Display, exit: Exit) -> Exit { + let _ = writeln!(stderr, "kharon-teach: {what}"); + exit +} + +/// Where the recording goes. +enum Target { + /// `--out`: exactly this path. + Explicit(PathBuf), + /// No `--out`: a generated name in the current directory. + Here, +} + +#[doc(hidden)] +#[derive(Clone, Copy)] +pub enum Naming { + Default, + Explicit, +} + +/// Creates `first` (mode 0600, never replacing a file). If it exists, the same name with `.1` +/// to `.9` added (before the `.kharon-recording.json` ending for a generated name, at the end +/// for an `--out` name) is tried, so a demonstration is not lost to a name clash. +#[doc(hidden)] +pub fn write_new(first: &Path, naming: Naming, text: &str) -> std::io::Result { + use std::os::unix::fs::OpenOptionsExt; + let name = first + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(); + for n in 0..10 { + let path = if n == 0 { + first.to_owned() + } else { + let numbered = match (naming, name.strip_suffix(cli::FILE_SUFFIX)) { + (Naming::Default, Some(stem)) => format!("{stem}.{n}{}", cli::FILE_SUFFIX), + _ => format!("{name}.{n}"), + }; + first.with_file_name(numbered) + }; + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + { + Ok(mut f) => { + f.write_all(text.as_bytes())?; + f.flush()?; + return Ok(path); + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(e), + } + } + Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "ten files of that name already exist", + )) +} + +/// `kharon-teach show `. +fn show(file: &Path, stdout: &mut impl Write, stderr: &mut impl Write) -> Exit { + use std::io::Read; + const CAP: u64 = 16 * 1024 * 1024; + let read = || -> Result { + let unreadable = |e: std::io::Error| Refusal::Unreadable(file.to_owned(), e.to_string()); + let f = std::fs::File::open(file).map_err(unreadable)?; + let mut bytes = Vec::new(); + f.take(CAP + 1) + .read_to_end(&mut bytes) + .map_err(unreadable)?; + if bytes.len() as u64 > CAP { + return Err(Refusal::TooLarge(file.to_owned())); + } + let text = String::from_utf8(bytes) + .map_err(|_| Refusal::NotARecording(file.to_owned(), "not text".to_owned()))?; + Recording::from_json(&text) + .map_err(|e| Refusal::NotARecording(file.to_owned(), e.to_string())) + }; + match read() { + Ok(r) => { + let _ = stdout.write_all(summary(&r).as_bytes()); + Exit::Done + } + Err(e) => fail(stderr, &e, Exit::Refused), + } +} + +/// Everything checked before the notice. +struct Prepared { + url: url::Url, + browser: PathBuf, + target: Target, +} + +fn prepare(cli: &Cli) -> Result { + let url = validate_login_url(cli.login_url.as_deref().ok_or(Refusal::NoUrl)?)?; + let browser = browser::find_browser(cli.browser.as_deref()) + .map_err(|e| Refusal::NoBrowser(e.to_string()))?; + let target = match &cli.out { + Some(p) => { + check_out_path(p)?; + Target::Explicit(p.clone()) + } + None => { + check_dir_writable(Path::new("."))?; + Target::Here + } + }; + Ok(Prepared { + url, + browser, + target, + }) +} + +/// Linux: no core dump and no ptrace by another process, as `kharon` does (REQ-MEM-001). +#[cfg(target_os = "linux")] +fn harden_process() -> Result<(), String> { + use rustix::process::{DumpableBehavior, Resource, Rlimit, set_dumpable_behavior, setrlimit}; + set_dumpable_behavior(DumpableBehavior::NotDumpable).map_err(|e| e.to_string())?; + setrlimit( + Resource::Core, + Rlimit { + current: Some(0), + maximum: Some(0), + }, + ) + .map_err(|e| e.to_string()) +} + +#[cfg(not(target_os = "linux"))] +fn harden_process() -> Result<(), String> { + Ok(()) +} + +enum Ended { + Finished(Result), + Interrupted, +} + +#[doc(hidden)] +pub async fn run_with( + cli: Cli, + mut stdin: impl AsyncBufRead + Unpin, + mut stdout: impl Write, + mut stderr: impl Write, + hooks: Hooks, +) -> Exit { + use tokio::signal::unix::{SignalKind, signal}; + if let Some(Command::Show { file }) = &cli.command { + return show(file, &mut stdout, &mut stderr); + } + let prep = match prepare(&cli) { + Ok(p) => p, + Err(r) => return fail(&mut stderr, &r, Exit::Refused), + }; + let _ = stdout.write_all(notice::NOTICE.as_bytes()); + if !cli.yes { + let _ = stdout.write_all(notice::CONSENT_PROMPT.as_bytes()); + let _ = stdout.flush(); + let mut line = String::new(); + let n = stdin.read_line(&mut line).await.unwrap_or(0); + if n == 0 || line.trim() != "yes" { + let _ = writeln!(stdout, "{}", notice::NOTHING_RECORDED); + return Exit::Done; + } + } + if let Err(e) = harden_process() { + return fail( + &mut stderr, + &format_args!("cannot make the process non-dumpable: {e}"), + Exit::Failed, + ); + } + let (Ok(mut sigint), Ok(mut sigterm)) = ( + signal(SignalKind::interrupt()), + signal(SignalKind::terminate()), + ) else { + return fail(&mut stderr, &"cannot watch for Ctrl-C", Exit::Failed); + }; + + let (ready_tx, mut ready_rx) = oneshot::channel(); + let (end_tx, end_rx) = oneshot::channel(); + let mut end_tx = Some(end_tx); + let mut rec = Box::pin(record( + RecordOptions { + login_url: &prep.url, + browser: &prep.browser, + headless: hooks.headless, + tool_version: env!("CARGO_PKG_VERSION"), + }, + ready_tx, + end_rx, + )); + let mut forward = hooks.ready; + let (mut got_ready, mut stdin_done) = (false, false); + let stdin_line = async { + let mut line = String::new(); + let read = stdin.read_line(&mut line).await; + (line, read) + }; + tokio::pin!(stdin_line); + let ended = loop { + tokio::select! { + r = &mut rec => break Ended::Finished(r), + r = &mut ready_rx, if !got_ready => { + got_ready = true; + if let Ok(ready) = r { + if ready.stale_profiles_removed > 0 { + let _ = writeln!( + stdout, + "Removed {} leftover browser profile(s) from an earlier run.", + ready.stale_profiles_removed + ); + } + let _ = writeln!(stdout, "{}", notice::END_PROMPT); + let _ = stdout.flush(); + if let Some(tx) = forward.take() { + let _ = tx.send(Ready { + devtools_port: ready.devtools_port, + profile_dir: ready.profile_dir, + stale_profiles_removed: ready.stale_profiles_removed, + }); + } + } + } + (line, read) = &mut stdin_line, if got_ready && !stdin_done => { + stdin_done = true; + // End of input is neither answer: keep recording until the window closes. + if matches!(read, Ok(n) if n > 0) { + let how = if line.trim().eq_ignore_ascii_case("q") { + End::Discard + } else { + End::Confirmed + }; + if let Some(tx) = end_tx.take() { + let _ = tx.send(how); + } + } + } + _ = sigint.recv() => break Ended::Interrupted, + _ = sigterm.recv() => break Ended::Interrupted, + } + }; + let outcome = match ended { + Ended::Finished(r) => r, + Ended::Interrupted => { + // Dropping the session closes the browser and removes its profile. + drop(rec); + return fail( + &mut stderr, + &"Interrupted. Nothing was saved.", + Exit::Failed, + ); + } + }; + drop(rec); + match outcome { + Ok(Outcome::Recorded(r)) => { + let first = match &prep.target { + Target::Explicit(p) => p.clone(), + Target::Here => PathBuf::from(default_file_name( + prep.url.host_str().unwrap_or("login"), + std::time::SystemTime::now(), + )), + }; + let naming = match prep.target { + Target::Explicit(_) => Naming::Explicit, + Target::Here => Naming::Default, + }; + match write_new(&first, naming, &r.to_json()) { + Ok(path) => { + let _ = writeln!(stdout, "\n{}", summary(&r)); + let _ = writeln!(stdout, "Saved: {}", path.display()); + Exit::Done + } + Err(e) => fail( + &mut stderr, + &format_args!("cannot write {}: {e}", first.display()), + Exit::Failed, + ), + } + } + Ok(Outcome::Discarded) => { + let _ = writeln!(stdout, "Discarded. Nothing was saved."); + Exit::Done + } + Ok(Outcome::NeverStarted) => { + let _ = writeln!( + stdout, + "The window was closed before the login page opened. Nothing was saved." + ); + Exit::Done + } + Err(e) => fail(&mut stderr, &e, Exit::Failed), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::Parser; + + fn cli(args: &[&str]) -> Cli { + Cli::try_parse_from(std::iter::once("kharon-teach").chain(args.iter().copied())).unwrap() + } + + fn temp_listing() -> Vec { + std::fs::read_dir(std::env::temp_dir()) + .unwrap() + .flatten() + .map(|e| e.file_name()) + .filter(|n| n.to_string_lossy().starts_with("kharon-teach-")) + .collect() + } + + struct Ran { + exit: Exit, + out: String, + err: String, + } + + async fn go(args: &[&str], stdin: &str) -> Ran { + let (mut out, mut err) = (Vec::new(), Vec::new()); + let exit = run(cli(args), stdin.as_bytes(), &mut out, &mut err).await; + Ran { + exit, + out: String::from_utf8(out).unwrap(), + err: String::from_utf8(err).unwrap(), + } + } + + fn private_dir(tag: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!( + "kharon-run-test-{tag}-{}-{:08x}", + std::process::id(), + rand::random::() + )); + std::fs::create_dir(&d).unwrap(); + d + } + + /// An executable that exists, so browser discovery passes. It is never started in these tests. + fn fake_browser(dir: &std::path::Path) -> std::path::PathBuf { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fake-browser"); + std::fs::write(&p, "#!/bin/sh\nexit 1\n").unwrap(); + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap(); + p + } + + #[tokio::test] + async fn refusals_exit_2_before_the_notice() { + let before = temp_listing(); + let d = private_dir("refuse"); + let browser = fake_browser(&d); + let existing = d.join("taken.json"); + std::fs::write(&existing, b"x").unwrap(); + let b = browser.to_str().unwrap(); + let missing = d.join("no-such-browser"); + let cases: Vec> = vec![ + vec!["http://ex.test/", "--browser", b], + vec!["https://u:p@ex.test/", "--browser", b], + vec!["https://ex.test/", "--browser", missing.to_str().unwrap()], + vec![ + "https://ex.test/", + "--browser", + b, + "--out", + existing.to_str().unwrap(), + ], + vec![ + "https://ex.test/", + "--browser", + b, + "--out", + "/no/such/dir/a.json", + ], + vec![], + ]; + for args in cases { + let r = go(&args, "yes\n").await; + assert_eq!(r.exit, Exit::Refused, "{args:?}"); + assert!(!r.out.contains("THROWAWAY"), "{args:?}"); + assert!(r.out.is_empty(), "{args:?}: {}", r.out); + assert!(r.err.starts_with("kharon-teach: "), "{}", r.err); + assert_eq!(r.err.lines().count(), 1, "{}", r.err); + } + let after = temp_listing(); + assert!( + after.iter().all(|n| before.contains(n)), + "a refusal created a profile directory" + ); + std::fs::remove_dir_all(&d).unwrap(); + } + + #[tokio::test] + async fn anything_but_yes_records_nothing() { + let d = private_dir("consent"); + let browser = fake_browser(&d); + let b = browser.to_str().unwrap(); + for input in ["no\n", "y\n", "YES\n", "\n", ""] { + let r = go(&["https://ex.test/", "--browser", b], input).await; + assert_eq!(r.exit, Exit::Done, "{input:?}"); + assert!(r.out.contains("THROWAWAY")); + assert!(r.out.contains("Nothing was recorded."), "{input:?}"); + assert!(r.err.is_empty()); + } + std::fs::remove_dir_all(&d).unwrap(); + } + + #[tokio::test] + async fn show_prints_the_summary_and_refuses_a_bad_file() { + let d = private_dir("show"); + let bad = d.join("bad.json"); + std::fs::write(&bad, b"{}").unwrap(); + let r = go(&["show", bad.to_str().unwrap()], "").await; + assert_eq!(r.exit, Exit::Refused); + assert!(r.out.is_empty()); + assert!(r.err.starts_with("kharon-teach: ") && r.err.lines().count() == 1); + let r = go(&["show", d.join("missing").to_str().unwrap()], "").await; + assert_eq!(r.exit, Exit::Refused); + let big = d.join("big.json"); + std::fs::write(&big, vec![b' '; 16 * 1024 * 1024 + 1]).unwrap(); + let r = go(&["show", big.to_str().unwrap()], "").await; + assert_eq!(r.exit, Exit::Refused); + assert!(r.err.contains("16 MiB")); + std::fs::remove_dir_all(&d).unwrap(); + } + + #[tokio::test] + async fn show_prints_the_summary_of_a_good_file() { + let d = private_dir("show-ok"); + let good = d.join("good.json"); + std::fs::write(&good, cli::tests::recording().to_json()).unwrap(); + let r = go(&["show", good.to_str().unwrap()], "").await; + assert_eq!(r.exit, Exit::Done, "{}", r.err); + assert!(r.out.contains("Site: https://ex.test/login")); + std::fs::remove_dir_all(&d).unwrap(); + } + + #[test] + fn the_write_picks_a_numbered_name_when_the_file_appears() { + let d = private_dir("numbered"); + let first = d.join(default_file_name("ex.test", std::time::UNIX_EPOCH)); + let a = write_new(&first, Naming::Default, "a").unwrap(); + let b = write_new(&first, Naming::Default, "b").unwrap(); + assert_eq!(a, first); + assert!( + b.to_string_lossy().ends_with(".1.kharon-recording.json"), + "{b:?}" + ); + assert_eq!(std::fs::read_to_string(&a).unwrap(), "a"); + let ex = d.join("mine.json"); + write_new(&ex, Naming::Explicit, "x").unwrap(); + let y = write_new(&ex, Naming::Explicit, "y").unwrap(); + assert_eq!(y, d.join("mine.json.1")); + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(&y).unwrap().permissions().mode() & 0o777, + 0o600 + ); + std::fs::remove_dir_all(&d).unwrap(); + } +} diff --git a/crates/kharon-teach/src/main.rs b/crates/kharon-teach/src/main.rs index 5e58dc9..395e240 100644 --- a/crates/kharon-teach/src/main.rs +++ b/crates/kharon-teach/src/main.rs @@ -1,6 +1,24 @@ +use clap::Parser; +use kharon_teach::cli::Cli; use std::process::ExitCode; fn main() -> ExitCode { - eprintln!("kharon-teach: not implemented"); - ExitCode::from(1) + let cli = Cli::parse(); + let runtime = match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + { + Ok(r) => r, + Err(e) => { + eprintln!("kharon-teach: cannot start: {e}"); + return ExitCode::from(1); + } + }; + let exit = runtime.block_on(kharon_teach::run( + cli, + tokio::io::BufReader::new(tokio::io::stdin()), + std::io::stdout(), + std::io::stderr(), + )); + exit.into() } diff --git a/crates/kharon-teach/src/notice.rs b/crates/kharon-teach/src/notice.rs index 5583449..6ad5a9b 100644 --- a/crates/kharon-teach/src/notice.rs +++ b/crates/kharon-teach/src/notice.rs @@ -8,6 +8,13 @@ Use a THROWAWAY account. Everything you type in the window that\n\ opens is read by this program. Nothing you type is saved, and\n\ nothing is sent anywhere. The result is one file on this computer.\n"; +/// Asked on the terminal before anything starts. +pub const CONSENT_PROMPT: &str = "Type yes to open the browser window and start: "; +/// Printed once the browser window is showing the notice. +pub const END_PROMPT: &str = "Press Enter once you are logged in. Type q to discard."; +/// Printed when consent was not given. +pub const NOTHING_RECORDED: &str = "Nothing was recorded."; + const KEEP: &AsciiSet = &NON_ALPHANUMERIC .remove(b'-') .remove(b'.') diff --git a/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs b/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs index 5a73db1..829b92e 100644 --- a/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs +++ b/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs @@ -22,3 +22,17 @@ async fn no_typed_value_in_the_recording_for_any_fixture() { assert_holds_no_typed_value(&json, &[USER, PASS]); } } + +#[tokio::test] +async fn nothing_typed_reaches_the_file_or_the_output() { + let Some(p) = support::run_program("whole_program_no_typed_value", "/form").await else { + return; + }; + assert_eq!(p.exit, kharon_teach::Exit::Done, "{}", p.stderr); + assert_eq!(p.file_mode, 0o600); + assert!(p.file_text.contains("\"value_len\""), "the file is empty"); + assert!(p.stdout.contains(&p.path.display().to_string())); + for text in [&p.file_text, &p.stdout, &p.stderr] { + assert_holds_no_typed_value(text, &[USER, PASS]); + } +} diff --git a/crates/kharon-teach/tests/req_tch_003_file_contents.rs b/crates/kharon-teach/tests/req_tch_003_file_contents.rs new file mode 100644 index 0000000..944739c --- /dev/null +++ b/crates/kharon-teach/tests/req_tch_003_file_contents.rs @@ -0,0 +1,45 @@ +//! SPEC req_tch_003: the file holds only what the spec keeps. + +mod support; + +use kharon_recording::Recording; +use support::{COOKIE_VALUE, run_program}; + +const NEXT_VALUE: &str = "secret-next-value"; + +#[tokio::test] +async fn the_file_is_0600_and_holds_only_what_the_spec_keeps() { + let Some(p) = run_program("file_contents", &format!("/form?next=/{NEXT_VALUE}")).await else { + return; + }; + assert_eq!(p.exit, kharon_teach::Exit::Done, "{}", p.stderr); + assert_eq!(p.file_mode, 0o600); + let text = &p.file_text; + let rec = Recording::from_json(text).expect("the file reads back"); + // Query keys are kept, their values are not. + let keys_kept = std::iter::once(&rec.start_url) + .chain(rec.requests.iter().map(|r| &r.url)) + .any(|u| u.query_keys.iter().any(|k| k == "next")); + assert!(keys_kept, "the query key `next` was not kept"); + assert!(!text.contains(NEXT_VALUE), "a query value is in the file"); + // No cookie, no response body, no page title. + assert!( + !text.contains(COOKIE_VALUE), + "a cookie value is in the file" + ); + assert!(!text.contains("sid="), "a cookie is in the file"); + assert!(!text.contains("signed in"), "response text is in the file"); + for title in ["", ">form<", "\"home\"", "step one"] { + assert!( + !text.contains(title), + "a page title is in the file: {title}" + ); + } + // Headers are reduced to the content type. + for r in &rec.requests { + if let Some(ct) = &r.content_type { + assert!(!ct.is_empty()); + } + } + assert!(!text.to_ascii_lowercase().contains("set-cookie:")); +} diff --git a/crates/kharon-teach/tests/req_tch_004_no_network.rs b/crates/kharon-teach/tests/req_tch_004_no_network.rs new file mode 100644 index 0000000..55eff43 --- /dev/null +++ b/crates/kharon-teach/tests/req_tch_004_no_network.rs @@ -0,0 +1,105 @@ +//! SPEC req_tch_004: the recorder opens no socket but loopback CDP, and links no TLS or HTTP client. + +use std::path::{Path, PathBuf}; + +/// Shipped code only: the text before the first `#[cfg(test)]` of each file under `src/`. +fn shipped_sources() -> Vec<(PathBuf, String)> { + fn walk(dir: &Path, out: &mut Vec<PathBuf>) { + for e in std::fs::read_dir(dir).unwrap().flatten() { + let p = e.path(); + if p.is_dir() { + walk(&p, out); + } else if p.extension().is_some_and(|x| x == "rs") { + out.push(p); + } + } + } + let mut files = Vec::new(); + walk( + &Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), + &mut files, + ); + assert!(!files.is_empty()); + files + .into_iter() + .map(|p| { + let text = std::fs::read_to_string(&p).unwrap(); + let shipped = text.split("#[cfg(test)]").next().unwrap().to_owned(); + (p, shipped) + }) + .collect() +} + +#[test] +fn the_crate_listens_on_nothing_and_has_no_tls() { + for (path, text) in shipped_sources() { + for banned in ["TcpListener", "UdpSocket", "bind(", "accept("] { + assert!(!text.contains(banned), "{} holds {banned}", path.display()); + } + } + let out = std::process::Command::new(env!("CARGO")) + .args([ + "tree", + "-p", + "kharon-teach", + "-e", + "normal", + "--prefix", + "none", + ]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .output() + .expect("cargo tree runs"); + assert!( + out.status.success(), + "cargo tree failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + let tree = String::from_utf8_lossy(&out.stdout); + assert!(tree.lines().any(|l| l.starts_with("kharon-teach "))); + for banned in [ + "rustls", + "native-tls", + "openssl", + "openssl-sys", + "reqwest", + "hyper", + "kharon-connector", + "kharond", + ] { + let hit = tree + .lines() + .any(|l| l.split_whitespace().next() == Some(banned)); + assert!(!hit, "the dependency tree holds {banned}"); + } +} + +#[test] +fn the_only_dial_is_loopback() { + let sources = shipped_sources(); + let cdp = sources + .iter() + .find(|(p, _)| p.ends_with("cdp.rs")) + .expect("cdp.rs exists"); + assert!( + cdp.1.contains("ws://127.0.0.1:"), + "cdp.rs does not build its URL from the loopback literal" + ); + let dials: Vec<_> = sources + .iter() + .flat_map(|(p, t)| { + ["connect_async", "TcpStream::connect", "UnixStream::connect"] + .into_iter() + .filter(|d| t.contains(d)) + .map(move |d| format!("{}: {d}", p.display())) + }) + .collect(); + assert!( + !dials.is_empty(), + "cdp.rs no longer dials with connect_async" + ); + assert!( + dials.iter().all(|d| d.contains("cdp.rs")), + "a dial outside cdp.rs: {dials:?}" + ); +} diff --git a/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs index 1dfa283..d85eb0d 100644 --- a/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs +++ b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs @@ -199,3 +199,122 @@ async fn a_stale_profile_is_swept_at_the_next_start() { assert!(!stale.exists(), "the stale profile was not swept"); let _ = run.finish(End::Discard).await; } + +#[test] +fn the_shipped_main_never_asks_for_headless() { + fn walk(dir: &Path, out: &mut Vec<PathBuf>) { + for e in std::fs::read_dir(dir).unwrap().flatten() { + let p = e.path(); + if p.is_dir() { + walk(&p, out); + } else if p.extension().is_some_and(|x| x == "rs") { + out.push(p); + } + } + } + let mut files = Vec::new(); + walk( + &Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), + &mut files, + ); + for p in files { + let text = std::fs::read_to_string(&p).unwrap(); + let shipped = text.split("#[cfg(test)]").next().unwrap(); + assert!( + !shipped.contains("headless: true"), + "{} asks for headless", + p.display() + ); + // Browser discovery reads PATH; nothing else reads the environment. + if !p.ends_with("browser.rs") { + assert!( + !shipped.contains("env::var"), + "{} reads the environment", + p.display() + ); + } + } +} + +/// A `kharon-teach-*` profile directory in the temp directory whose owner file holds `pid`. +fn profile_of(pid: u32) -> Option<PathBuf> { + std::fs::read_dir(std::env::temp_dir()) + .ok()? + .flatten() + .filter(|e| e.file_name().to_string_lossy().starts_with("kharon-teach-")) + .map(|e| e.path()) + .find(|p| { + std::fs::read_to_string(p.join("kharon-teach.owner")) + .is_ok_and(|t| t.trim() == pid.to_string()) + }) +} + +/// Runs the built program against a browser that never opens a DevTools port, sends it `signal`, +/// and checks: exit code 1, the profile removed, no file written, all within 10 s. +fn interrupted_by(signal: &str) { + use std::os::unix::fs::PermissionsExt; + let dir = std::env::temp_dir().join(format!( + "kharon-sig-{}-{:08x}", + std::process::id(), + rand::random::<u32>() + )); + std::fs::create_dir(&dir).unwrap(); + let fake = dir.join("fake-browser"); + std::fs::write(&fake, "#!/bin/sh\nexec sleep 30\n").unwrap(); + std::fs::set_permissions(&fake, std::fs::Permissions::from_mode(0o755)).unwrap(); + let out = dir.join("login.kharon-recording.json"); + let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_kharon-teach")) + .args(["--yes", "https://ex.test/login", "--browser"]) + .arg(&fake) + .arg("--out") + .arg(&out) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let pid = child.id(); + let limit = std::time::Instant::now() + Duration::from_secs(10); + let profile = loop { + if let Some(p) = profile_of(pid) { + break p; + } + assert!( + std::time::Instant::now() < limit, + "the program made no profile in 10 s" + ); + std::thread::sleep(Duration::from_millis(20)); + }; + let sent = std::process::Command::new("kill") + .args([&format!("-{signal}"), &pid.to_string()]) + .status() + .unwrap(); + assert!(sent.success()); + let status = loop { + if let Some(s) = child.try_wait().unwrap() { + break s; + } + if std::time::Instant::now() >= limit + Duration::from_secs(10) { + let _ = child.kill(); + panic!("the program did not stop after {signal}"); + } + std::thread::sleep(Duration::from_millis(20)); + }; + assert_eq!(status.code(), Some(1), "exit status after {signal}"); + let mut err = String::new(); + std::io::Read::read_to_string(child.stderr.as_mut().unwrap(), &mut err).unwrap(); + assert!(err.contains("Interrupted. Nothing was saved."), "{err}"); + assert!(!profile.exists(), "the profile is still there"); + assert!(!out.exists(), "a file was written"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn sigint_exits_1_and_removes_the_profile() { + interrupted_by("INT"); +} + +#[test] +fn sigterm_exits_1_and_removes_the_profile() { + interrupted_by("TERM"); +} diff --git a/crates/kharon-teach/tests/support/mod.rs b/crates/kharon-teach/tests/support/mod.rs index aaa220c..56f6ba6 100644 --- a/crates/kharon-teach/tests/support/mod.rs +++ b/crates/kharon-teach/tests/support/mod.rs @@ -598,3 +598,116 @@ impl Person { .await } } + +// ---- the whole program, driven through `run_with` ---- + +/// A cloneable in-memory writer; the program gets one clone and the test reads the other. +#[derive(Clone, Default)] +pub struct Shared(std::sync::Arc<std::sync::Mutex<Vec<u8>>>); + +impl Shared { + pub fn text(&self) -> String { + String::from_utf8_lossy(&self.0.lock().unwrap()).into_owned() + } +} + +impl std::io::Write for Shared { + fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// What a whole run of the program left behind. +pub struct Program { + pub exit: kharon_teach::Exit, + pub path: PathBuf, + pub file_text: String, + pub file_mode: u32, + pub stdout: String, + pub stderr: String, +} + +/// Runs the program headless against the fixture's `/form` login (the path may carry a query), +/// as a person would: says `yes`, opens the login page from the notice, types the throwaway +/// values, signs in, presses Enter. `None` when there is no browser and none is required. +pub async fn run_program(test: &str, form_path_and_query: &str) -> Option<Program> { + use clap::Parser; + use std::os::unix::fs::PermissionsExt; + let bin = chromium_or_skip(test)?; + let _permit = GATE.acquire().await.unwrap(); + let fixture = Fixture::start().await; + let login = fixture.url(form_path_and_query); + let dir = std::env::temp_dir().join(format!( + "kharon-prog-{}-{:08x}", + std::process::id(), + rand::random::<u32>() + )); + std::fs::create_dir(&dir).unwrap(); + let out = dir.join("login.kharon-recording.json"); + let cli = kharon_teach::cli::Cli::try_parse_from([ + "kharon-teach", + login.as_str(), + "--browser", + bin.to_str().unwrap(), + "--out", + out.to_str().unwrap(), + ]) + .unwrap(); + let (stdout, stderr) = (Shared::default(), Shared::default()); + let (ready_tx, ready_rx) = oneshot::channel(); + let (mut keys, stdin) = tokio::io::duplex(1024); + let task = tokio::spawn({ + let (stdout, stderr) = (stdout.clone(), stderr.clone()); + async move { + kharon_teach::run_with( + cli, + tokio::io::BufReader::new(stdin), + stdout, + stderr, + kharon_teach::Hooks { + headless: true, + ready: Some(ready_tx), + }, + ) + .await + } + }); + keys.write_all(b"yes\n").await.unwrap(); + let ready = tokio::time::timeout(Duration::from_secs(40), ready_rx) + .await + .expect("the program was not ready in 40 s") + .expect("the program ended before it was ready"); + let person = Person::connect(ready.devtools_port).await; + person.click("a").await; + person.wait_for_path(path_of(form_path_and_query)).await; + person.type_into("input[name=user]", USER).await; + person.type_into("input[name=pass]", PASS).await; + person.click("#go").await; + person.wait_for_path("/home").await; + keys.write_all(b"\n").await.unwrap(); + let exit = tokio::time::timeout(Duration::from_secs(20), task) + .await + .expect("the program did not end in 20 s") + .expect("the program panicked"); + let file_text = std::fs::read_to_string(&out).unwrap_or_default(); + let file_mode = std::fs::metadata(&out) + .map(|m| m.permissions().mode() & 0o777) + .unwrap_or(0); + let _ = std::fs::remove_dir_all(&dir); + Some(Program { + exit, + path: out, + file_text, + file_mode, + stdout: stdout.text(), + stderr: stderr.text(), + }) +} + +fn path_of(path_and_query: &str) -> &str { + path_and_query.split('?').next().unwrap_or("") +} From 2a86a5e8d5664394c0d957ae6bf7554c296272aa Mon Sep 17 00:00:00 2001 From: Cycle Five <cycle.five@proton.me> Date: Wed, 18 Nov 2026 06:28:46 -0500 Subject: [PATCH 23/36] fix(teach): selectors, world probe, post data and group cleanup for the live session (#361) - listener: the typed text is kept out of the descriptor and the selectors by comparing it, at any length, with the attributes that feed them; form and element properties are read through the prototypes so a child named action, method, id, tagName or getAttribute cannot replace them. - world probe: each install carries its own nonce and only that attempt's hello counts; the isolated wait is 5 s; record_with and WorldProbe::SkipIsolated make the main-world fallback testable. Chromium accepts Runtime.addBinding without a context name after the same name was added with one; removeBinding is called first anyway. - Network.getRequestPostData is bounded by 5 s and now runs in a test (a Blob body is not sent inline). - cleanup: the browser leads its own process group; close and drop SIGKILL the group and wait (at most 3 s) until it is empty before the profile is removed. The process-tree walking is gone. - tests: mirrored attribute, raw listener reports, FormData, Blob, main-world fallback, clobbered form, popup, cross-origin frame, checkbox, browser killed from outside, browser quitting itself, profile databases with their -wal and -journal files. Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-teach/src/browser.rs | 88 ++++++- crates/kharon-teach/src/events.rs | 14 +- crates/kharon-teach/src/listener.js | 118 ++++++--- crates/kharon-teach/src/session.rs | 215 ++++++++-------- .../tests/req_tch_002_no_typed_value.rs | 103 +++++++- .../kharon-teach/tests/req_tch_005_shapes.rs | 198 +++++++++++++- .../tests/req_tch_006_profile_and_visible.rs | 104 +++++++- crates/kharon-teach/tests/support/mod.rs | 242 +++++++++++++++++- 8 files changed, 900 insertions(+), 182 deletions(-) diff --git a/crates/kharon-teach/src/browser.rs b/crates/kharon-teach/src/browser.rs index 96e4b5c..3dc8525 100644 --- a/crates/kharon-teach/src/browser.rs +++ b/crates/kharon-teach/src/browser.rs @@ -280,11 +280,19 @@ pub struct Launch<'a> { pub headless: bool, } -/// The browser process. Killed when dropped. +/// The browser process and the process group it leads. Killing it kills the whole group, and a +/// drop kills it. pub struct Browser { child: tokio::process::Child, + /// The group id (the browser's own pid: it is launched as a group leader). + #[cfg(unix)] + group: Option<rustix::process::Pid>, } +/// How long to wait for a killed group to be empty. +const GROUP_WAIT: Duration = Duration::from_secs(3); +const GROUP_POLL: Duration = Duration::from_millis(20); + pub struct DevTools { pub port: u16, pub ws_path: String, @@ -327,27 +335,37 @@ impl Browser { .stdout(Stdio::null()) .stderr(Stdio::null()) .kill_on_drop(true); - let mut child = cmd.spawn().map_err(launch_err)?; + // Its own process group, so that "kill the browser" means the helpers too. + #[cfg(unix)] + cmd.process_group(0); + let child = cmd.spawn().map_err(launch_err)?; + #[cfg(unix)] + let group = child + .id() + .and_then(|id| i32::try_from(id).ok()) + .and_then(rustix::process::Pid::from_raw); + // From here on a drop (including every early return) kills the group. + let mut browser = Browser { + child, + #[cfg(unix)] + group, + }; let file = profile.join("DevToolsActivePort"); let deadline = tokio::time::Instant::now() + START_TIMEOUT; loop { - match child.try_wait() { + match browser.child.try_wait() { Ok(Some(status)) => return Err(BrowserError::Launch(status.to_string())), Ok(None) => {} - Err(e) => { - let _ = child.kill().await; - return Err(launch_err(e)); - } + Err(e) => return Err(launch_err(e)), } if let Some(dt) = std::fs::read_to_string(&file) .ok() .and_then(|t| parse_devtools_active_port(&t)) { - return Ok((Browser { child }, dt)); + return Ok((browser, dt)); } if tokio::time::Instant::now() >= deadline { - let _ = child.kill().await; return Err(BrowserError::Launch( "no DevTools port after 20 s".to_owned(), )); @@ -364,9 +382,57 @@ impl Browser { self.child.wait().await } - /// Kill the browser and wait until it is reaped. + /// SIGKILL to every process in the browser's group. Safe to repeat. + pub fn kill_group(&self) { + #[cfg(unix)] + if let Some(g) = self.group { + let _ = rustix::process::kill_process_group(g, rustix::process::Signal::KILL); + } + } + + /// True once no process is left in the browser's group. A group that was never made counts + /// as empty. + pub fn group_is_empty(&self) -> bool { + #[cfg(unix)] + if let Some(g) = self.group { + return matches!( + rustix::process::test_kill_process_group(g), + Err(rustix::io::Errno::SRCH) + ); + } + true + } + + /// Kill the whole group, reap the browser, and wait up to 3 s for the group to be empty. pub async fn close(mut self) -> std::io::Result<()> { - self.child.kill().await + self.kill_group(); + let reaped = self.child.wait().await.map(|_| ()); + let deadline = tokio::time::Instant::now() + GROUP_WAIT; + while !self.group_is_empty() && tokio::time::Instant::now() < deadline { + tokio::time::sleep(GROUP_POLL).await; + } + reaped + } + + /// [`Browser::close`] for a place that cannot await (a `Drop`): it blocks the calling thread + /// for at most 3 s. + pub fn close_blocking(&mut self) { + self.kill_group(); + let deadline = std::time::Instant::now() + GROUP_WAIT; + loop { + // Reaps the leader, which would otherwise stay a zombie and keep the group non-empty. + let _ = self.child.try_wait(); + if self.group_is_empty() || std::time::Instant::now() >= deadline { + return; + } + std::thread::sleep(GROUP_POLL); + } + } +} + +impl Drop for Browser { + fn drop(&mut self) { + self.kill_group(); } } diff --git a/crates/kharon-teach/src/events.rs b/crates/kharon-teach/src/events.rs index 6448a43..6fe3d53 100644 --- a/crates/kharon-teach/src/events.rs +++ b/crates/kharon-teach/src/events.rs @@ -340,6 +340,14 @@ pub struct Empty {} #[derive(Deserialize)] pub struct Hello { pub hello: bool, + /// The nonce this install of the script was given: a hello from an earlier install is not + /// the current one's. + pub nonce: String, +} + +#[derive(Serialize)] +pub struct RemoveBinding<'a> { + pub name: &'a str, } const MAX_FIELD_CHARS: usize = 4096; @@ -634,7 +642,11 @@ mod tests { #[test] fn hello_is_not_a_report() { - assert!(parse_quiet::<Hello>(r#"{"hello":true}"#).is_ok_and(|h| h.hello)); + assert!( + parse_quiet::<Hello>(r#"{"hello":true,"nonce":"n1"}"#) + .is_ok_and(|h| h.hello && h.nonce == "n1") + ); + assert!(parse_quiet::<Hello>(r#"{"hello":true}"#).is_err()); assert!(parse_quiet::<Hello>(REPORT).is_err()); assert!(parse_quiet::<ListenerReport>(r#"{"hello":true}"#).is_err()); } diff --git a/crates/kharon-teach/src/listener.js b/crates/kharon-teach/src/listener.js index e0dfec5..7a7bedb 100644 --- a/crates/kharon-teach/src/listener.js +++ b/crates/kharon-teach/src/listener.js @@ -7,6 +7,8 @@ "use strict"; try { const BINDING = "__kharonTeach"; + // Replaced by the recorder with a fresh value for each install of this script. + const NONCE = "__NONCE__"; const GUARD = Symbol.for("kharon-teach.listener"); if (Object.prototype.hasOwnProperty.call(document, GUARD)) return; Object.defineProperty(document, GUARD, { value: true, enumerable: false }); @@ -28,6 +30,24 @@ documentId = String(Math.random()).slice(2) + String(Date.now()); } + // Reads go through the prototypes, never through the element: a form's child named + // "action", "method", "id", "tagName" or "getAttribute" replaces that property on the form. + const proto = Element.prototype; + const getAttr = proto.getAttribute; + const matchesFn = proto.matches; + const closestFn = proto.closest; + const getRoot = Node.prototype.getRootNode; + const getter = (obj, name) => { + const d = Object.getOwnPropertyDescriptor(obj, name); + return d && d.get ? d.get : () => undefined; + }; + const tagGetter = getter(proto, "tagName"); + const formActionGetter = getter(HTMLFormElement.prototype, "action"); + const formMethodGetter = getter(HTMLFormElement.prototype, "method"); + const docFormsGetter = getter(Document.prototype, "forms"); + const attr = (el, name) => getAttr.call(el, name); + const tagOf = (el) => String(tagGetter.call(el)).toLowerCase(); + const send = (report) => { try { const bind = window[BINDING]; @@ -60,11 +80,10 @@ const parts = []; let node = el; for (let depth = 0; node && node.nodeType === 1 && depth < 6; depth++) { - const tag = node.tagName.toLowerCase(); - let part = tag; + let part = tagOf(node); const parent = node.parentElement; if (parent) { - const same = Array.from(parent.children).filter((c) => c.tagName === node.tagName); + const same = Array.from(parent.children).filter((c) => tagOf(c) === part); if (same.length > 1) part += ":nth-of-type(" + (same.indexOf(node) + 1) + ")"; } parts.unshift(part); @@ -79,11 +98,36 @@ return null; }; + const textEntry = (el) => { + const tag = tagOf(el); + if (tag === "textarea") return true; + if (tag !== "input") return false; + return TEXTISH.includes(String(el.type || "text").toLowerCase()); + }; + + // What the person has typed into the element now: used to keep it out of the descriptor + // and the selectors, whether or not this report sends it. + const typedValue = (el) => { + try { + return textEntry(el) ? String(el.value === undefined ? "" : el.value) : ""; + } catch (_) { + return ""; + } + }; + + // An attribute that holds the typed text (a page that mirrors the field into its own id or + // name) is not reported and builds no selector. + const clean = (typed, v) => { + if (v === null || v === undefined) return null; + const t = String(v); + return typed.length > 0 && t.includes(typed) ? null : t; + }; + const selectorsFor = (el, typed) => { const out = []; let root = null; try { - root = el.getRootNode(); + root = getRoot.call(el); } catch (_) { root = document; } @@ -91,8 +135,6 @@ const scope = inShadow ? root : document; const add = (s, unique) => { if (!s || s.length > MAX_STR || out.includes(s)) return; - // A value this short is contained in an ordinary selector by chance. - if (typed && typed.length >= 3 && s.includes(typed)) return; try { if (scope.querySelector(s) !== el) return; if (unique && scope.querySelectorAll(s).length !== 1) return; @@ -101,13 +143,14 @@ /* an invalid candidate is dropped */ } }; - const tag = el.tagName.toLowerCase(); - if (el.id) add("#" + CSS.escape(el.id)); - const name = el.getAttribute("name"); + const tag = tagOf(el); + const id = clean(typed, attr(el, "id")); + if (id) add("#" + CSS.escape(id)); + const name = clean(typed, attr(el, "name")); if (name) add(tag + '[name="' + attrEscape(name) + '"]'); - const auto = el.getAttribute("autocomplete"); + const auto = clean(typed, attr(el, "autocomplete")); if (auto) add(tag + '[autocomplete="' + attrEscape(auto) + '"]'); - const type = el.getAttribute("type"); + const type = clean(typed, attr(el, "type")); if (type) add(tag + '[type="' + attrEscape(type) + '"]', true); add(structural(el, scope)); return { selectors: out.slice(0, MAX_SELECTORS), inShadow }; @@ -117,39 +160,36 @@ if (!form) return null; let index = 0; try { - index = Math.max(0, Array.prototype.indexOf.call(document.forms, form)); + index = Math.max(0, Array.prototype.indexOf.call(docFormsGetter.call(document), form)); } catch (_) { index = 0; } - return { - index, - action: cut(form.action), - method: String(form.method || "get").toLowerCase(), - }; - }; - - const textEntry = (el) => { - const tag = el.tagName.toLowerCase(); - if (tag === "textarea") return true; - if (tag !== "input") return false; - const type = String(el.type || "text").toLowerCase(); - return TEXTISH.includes(type); + let action = null; + let method = "get"; + try { + action = cut(formActionGetter.call(form)); + method = String(formMethodGetter.call(form) || "get").toLowerCase(); + } catch (_) { + /* leave the defaults */ + } + return { index, action, method }; }; - const describe = (el, typed) => { - const tag = el.tagName.toLowerCase(); + const describe = (el) => { + const tag = tagOf(el); const isInput = tag === "input"; + const typed = typedValue(el); const sel = selectorsFor(el, typed); const form = tag === "form" ? el : el.form || null; return { element_id: elementId(el), tag, input_type: isInput ? String(el.type || "text").toLowerCase() : null, - id: cut(el.id || null), - name: cut(el.getAttribute("name")), - autocomplete: cut(el.getAttribute("autocomplete")), - placeholder: cut(el.getAttribute("placeholder")), - aria_label: cut(el.getAttribute("aria-label")), + id: cut(clean(typed, attr(el, "id"))), + name: cut(clean(typed, attr(el, "name"))), + autocomplete: cut(clean(typed, attr(el, "autocomplete"))), + placeholder: cut(clean(typed, attr(el, "placeholder"))), + aria_label: cut(clean(typed, attr(el, "aria-label"))), is_password_now: isInput && String(el.type).toLowerCase() === "password", form: formInfo(form), selectors: sel.selectors, @@ -179,7 +219,7 @@ document_id: documentId, frame_url: cut(location.href) || "", is_top: isTop(), - element: describe(el, value), + element: describe(el), }; if (value !== null) body.value = value; send(body); @@ -190,25 +230,25 @@ const handlers = { input: (e) => { const el = originOf(e); - if (el && el.matches("input, textarea, select")) report("input", el, true); + if (el && matchesFn.call(el, "input, textarea, select")) report("input", el, true); }, change: (e) => { const el = originOf(e); - if (el && el.matches("input, textarea, select")) report("change", el, true); + if (el && matchesFn.call(el, "input, textarea, select")) report("change", el, true); }, click: (e) => { const el = originOf(e); - const hit = el && el.closest ? el.closest(CLICKABLE) : null; + const hit = el ? closestFn.call(el, CLICKABLE) : null; if (hit) report("click", hit, false); }, submit: (e) => { const el = originOf(e); - if (el && el.tagName.toLowerCase() === "form") report("submit", el, false); + if (el && tagOf(el) === "form") report("submit", el, false); }, keydown: (e) => { if (e.key !== "Enter") return; const el = originOf(e); - if (el && el.tagName.toLowerCase() === "input") report("enter_key", el, false); + if (el && tagOf(el) === "input") report("enter_key", el, false); }, }; @@ -227,7 +267,7 @@ ); } - if (isTop()) send({ hello: true }); + if (isTop()) send({ hello: true, nonce: NONCE }); } catch (_) { /* never throw into the page */ } diff --git a/crates/kharon-teach/src/session.rs b/crates/kharon-teach/src/session.rs index 5868931..5696d37 100644 --- a/crates/kharon-teach/src/session.rs +++ b/crates/kharon-teach/src/session.rs @@ -11,8 +11,9 @@ use crate::cdp::{Cdp, CdpError, Event, parse_quiet}; use crate::events::{ AddBinding, AddScriptToEvaluateOnNewDocument, AttachedToTarget, BindingCalled, Empty, FrameNavigated, GetRequestPostData, Hello, LayoutMetrics, ListenerReport, Navigate, PostData, - RemoveScript, ReportKind, RequestWillBeSent, ResponseReceived, ResponseReceivedExtraInfo, - ScriptId, SetAutoAttach, SetDiscoverTargets, TargetDestroyed, VersionInfo, + RemoveBinding, RemoveScript, ReportKind, RequestWillBeSent, ResponseReceived, + ResponseReceivedExtraInfo, ScriptId, SetAutoAttach, SetDiscoverTargets, TargetDestroyed, + VersionInfo, }; use crate::notice::notice_page_url; use kharon_recording::{BrowserInfo, EndedBy, ListenerWorld, Recording, ResourceKind, Viewport}; @@ -20,6 +21,7 @@ use serde::Serialize; use serde::de::IgnoredAny; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, Instant, SystemTime}; use tokio::sync::{mpsc::UnboundedReceiver, oneshot}; use zeroize::Zeroizing; @@ -29,12 +31,28 @@ pub const LISTENER_JS: &str = include_str!("listener.js"); const BINDING: &str = "__kharonTeach"; const WORLD: &str = "kharon-teach"; /// How long the listener may take to announce itself on the notice page. -const HELLO_WAIT: Duration = Duration::from_secs(3); +const HELLO_WAIT: Duration = Duration::from_secs(5); +/// The same, when a test has made the isolated world unable to answer. +const HELLO_WAIT_SKIPPED: Duration = Duration::from_millis(300); /// How long to keep taking events after the person confirms. const LATE_WAIT: Duration = Duration::from_millis(300); +/// How long `Network.getRequestPostData` may take. +const POST_DATA_WAIT: Duration = Duration::from_secs(5); +/// How many times the recorder had to ask the browser for a request body that was not inline. +#[doc(hidden)] +pub static POST_DATA_FETCHES: AtomicUsize = AtomicUsize::new(0); /// How long the browser may take to show its first page. const FIRST_PAGE_WAIT: Duration = Duration::from_secs(10); +/// Which world the listener is tried in first. `SkipIsolated` is for tests of the fallback. +#[doc(hidden)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum WorldProbe { + Normal, + /// The isolated world gets its binding but never the script, so the fallback runs. + SkipIsolated, +} + pub enum End { Confirmed, Discard, @@ -115,6 +133,19 @@ struct Rec { clock: Option<(Instant, SystemTime)>, hops: HashMap<String, Vec<Hop>>, hello: bool, + /// This attempt's value in the script; only a hello carrying it counts. + nonce: String, + probe: WorldProbe, +} + +fn fresh_nonce() -> String { + use rand::Rng; + use rand::distributions::Alphanumeric; + rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(24) + .map(char::from) + .collect() } async fn send<P: Serialize>( @@ -135,6 +166,17 @@ pub async fn record( opts: RecordOptions<'_>, ready: oneshot::Sender<Ready>, end: oneshot::Receiver<End>, +) -> Result<Outcome, SessionError> { + record_with(opts, ready, end, WorldProbe::Normal).await +} + +/// [`record`], with a way for a test to make the isolated world fail. +#[doc(hidden)] +pub async fn record_with( + opts: RecordOptions<'_>, + ready: oneshot::Sender<Ready>, + end: oneshot::Receiver<End>, + probe: WorldProbe, ) -> Result<Outcome, SessionError> { let stale = Profile::sweep_stale().unwrap_or(0); let profile = Profile::create()?; @@ -159,13 +201,16 @@ pub async fn record( &dev, &opts, ¬ice, - Ready { - devtools_port: dev.port, - profile_dir, - stale_profiles_removed: stale, - }, - ready, + ( + Ready { + devtools_port: dev.port, + profile_dir, + stale_profiles_removed: stale, + }, + ready, + ), end, + probe, ) .await; guard.finish().await; @@ -179,7 +224,7 @@ struct Cleanup { profile: Option<Profile>, } -/// A just-killed browser's helper processes can still be writing into the profile for a moment. +/// Removal retries: the group is empty by now, but a filesystem can still be busy for a moment. const REMOVE_TRIES: u32 = 60; const REMOVE_PAUSE: Duration = Duration::from_millis(50); @@ -189,85 +234,13 @@ fn try_remove(path: &std::path::Path) -> bool { !path.exists() } -/// The browser's process and everything below it. A killed browser's helper processes (the -/// network service above all) outlive it for a moment and recreate files in a profile that was -/// just removed, so the profile is removed only after they are gone. -#[cfg(target_os = "linux")] -fn family(root: u32) -> Vec<u32> { - let mut parent_of: Vec<(u32, u32)> = Vec::new(); - if let Ok(procs) = std::fs::read_dir("/proc") { - for e in procs.flatten() { - let Ok(pid) = e.file_name().to_string_lossy().parse::<u32>() else { - continue; - }; - if let Some((_, ppid)) = stat_of(pid) { - parent_of.push((pid, ppid)); - } - } - } - let mut out = vec![root]; - let mut i = 0; - while i < out.len() { - let at = out[i]; - out.extend( - parent_of - .iter() - .filter(|(_, pp)| *pp == at) - .map(|(p, _)| *p), - ); - i += 1; - } - out -} - -/// A process's state letter and parent pid, from `/proc/<pid>/stat`. -#[cfg(target_os = "linux")] -fn stat_of(pid: u32) -> Option<(char, u32)> { - let text = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; - // The command name is in parentheses and may hold spaces and parentheses itself. - let rest = &text[text.rfind(')')? + 1..]; - let mut f = rest.split_whitespace(); - let state = f.next()?.chars().next()?; - let ppid = f.next()?.parse().ok()?; - Some((state, ppid)) -} - -#[cfg(target_os = "linux")] -fn any_alive(pids: &[u32]) -> bool { - pids.iter() - .any(|p| stat_of(*p).is_some_and(|(state, _)| state != 'Z' && state != 'X')) -} - -#[cfg(not(target_os = "linux"))] -fn family(root: u32) -> Vec<u32> { - vec![root] -} - -#[cfg(not(target_os = "linux"))] -fn any_alive(_pids: &[u32]) -> bool { - false -} - impl Cleanup { - fn family(&self) -> Vec<u32> { - self.browser - .as_ref() - .map(|b| family(b.pid())) - .unwrap_or_default() - } - - /// The normal end: wait for the browser to be gone, then remove the profile. + /// The normal end: kill the browser's whole group, wait for it to be empty (at most 3 s), + /// then remove the profile. async fn finish(&mut self) { - let procs = self.family(); if let Some(b) = self.browser.take() { let _ = b.close().await; } - for _ in 0..REMOVE_TRIES { - if !any_alive(&procs) { - break; - } - tokio::time::sleep(REMOVE_PAUSE).await; - } if let Some(p) = self.profile.take() { let path = p.path().to_owned(); for _ in 0..REMOVE_TRIES { @@ -281,15 +254,11 @@ impl Cleanup { } impl Drop for Cleanup { + /// A drop cannot await, so this blocks its thread: up to 3 s for the browser's group to be + /// empty, then up to 3 s for the removal. It is the Ctrl-C path, and it cannot hang. fn drop(&mut self) { - let procs = self.family(); - // Dropping the browser kills it. - drop(self.browser.take()); - for _ in 0..REMOVE_TRIES { - if !any_alive(&procs) { - break; - } - std::thread::sleep(REMOVE_PAUSE); + if let Some(mut b) = self.browser.take() { + b.close_blocking(); } if let Some(p) = self.profile.take() { let path = p.path().to_owned(); @@ -308,9 +277,9 @@ async fn run( dev: &crate::browser::DevTools, opts: &RecordOptions<'_>, notice: &str, - info: Ready, - ready: oneshot::Sender<Ready>, + (info, ready): (Ready, oneshot::Sender<Ready>), mut end: oneshot::Receiver<End>, + probe: WorldProbe, ) -> Result<Outcome, SessionError> { let (cdp, mut rx) = Cdp::connect(dev).await?; let mut rec = Rec { @@ -325,6 +294,8 @@ async fn run( clock: None, hops: HashMap::new(), hello: false, + nonce: fresh_nonce(), + probe, }; send( @@ -441,7 +412,8 @@ impl Rec { Wait::Closed => return Ok(Choice::Closed), Wait::Timeout => {} } - // The isolated world did not deliver: install in the main world instead. + // The isolated world did not deliver: install in the main world instead, under a new + // nonce, so that a late hello from the isolated attempt cannot pass for the new one. if let Some(id) = self .sessions .get_mut(&first) @@ -456,6 +428,7 @@ impl Rec { .await?; } self.world = ListenerWorld::Main; + self.nonce = fresh_nonce(); self.install(&first).await?; send( &self.cdp, @@ -473,7 +446,13 @@ impl Rec { async fn wait_hello(&mut self, rx: &mut UnboundedReceiver<Event>) -> Wait { self.hello = false; - let deadline = tokio::time::Instant::now() + HELLO_WAIT; + let wait = + if self.world == ListenerWorld::Isolated && self.probe == WorldProbe::SkipIsolated { + HELLO_WAIT_SKIPPED + } else { + HELLO_WAIT + }; + let deadline = tokio::time::Instant::now() + wait; while !self.hello { match tokio::time::timeout_at(deadline, rx.recv()).await { Ok(Some(ev)) => { @@ -491,6 +470,16 @@ impl Rec { /// The binding and the script, in the world chosen now. async fn install(&mut self, session: &str) -> Result<(), CdpError> { let isolated = self.world == ListenerWorld::Isolated; + if !isolated { + // The isolated attempt's binding may still exist under the same name. + let _ = send( + &self.cdp, + "Runtime.removeBinding", + &RemoveBinding { name: BINDING }, + Some(session), + ) + .await; + } send( &self.cdp, "Runtime.addBinding", @@ -501,12 +490,16 @@ impl Rec { Some(session), ) .await?; + if isolated && self.probe == WorldProbe::SkipIsolated { + return Ok(()); + } + let source = LISTENER_JS.replacen("__NONCE__", &self.nonce, 1); let id: ScriptId = self .cdp .call( "Page.addScriptToEvaluateOnNewDocument", &AddScriptToEvaluateOnNewDocument { - source: LISTENER_JS, + source: &source, world_name: isolated.then_some(WORLD), }, Some(session), @@ -722,8 +715,10 @@ impl Rec { if p.name != BINDING { return; } - if parse_quiet::<Hello>(&p.payload).is_ok_and(|h| h.hello) { - self.hello = true; + if let Ok(h) = parse_quiet::<Hello>(&p.payload) { + if h.hello && h.nonce == self.nonce { + self.hello = true; + } return; } let Ok(r) = parse_quiet::<ListenerReport>(&p.payload) else { @@ -825,17 +820,17 @@ impl Rec { let body = match (&p.request.post_data, p.request.has_post_data) { (Some(d), _) => RawBody::Bytes(Zeroizing::new(d.as_bytes().to_vec())), (None, Some(true)) => { - match self - .cdp - .call::<_, PostData>( - "Network.getRequestPostData", - &GetRequestPostData { request_id: &id }, - Some(session), - ) - .await - { - Ok(d) => RawBody::Bytes(Zeroizing::new(d.post_data.as_bytes().to_vec())), - Err(_) => RawBody::Unavailable, + POST_DATA_FETCHES.fetch_add(1, Ordering::Relaxed); + // A target that dies mid-call must not stall event handling for CALL_TIMEOUT. + let params = GetRequestPostData { request_id: &id }; + let call = self.cdp.call::<_, PostData>( + "Network.getRequestPostData", + ¶ms, + Some(session), + ); + match tokio::time::timeout(POST_DATA_WAIT, call).await { + Ok(Ok(d)) => RawBody::Bytes(Zeroizing::new(d.post_data.as_bytes().to_vec())), + Ok(Err(_)) | Err(_) => RawBody::Unavailable, } } _ => RawBody::None, diff --git a/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs b/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs index 829b92e..2940b3b 100644 --- a/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs +++ b/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs @@ -2,8 +2,9 @@ mod support; +use kharon_teach::session::{End, Outcome}; use kharon_teach::testing::assert_holds_no_typed_value; -use support::{Login, PASS, USER, record_login}; +use support::{Login, PASS, Run, USER, record_login}; #[tokio::test] async fn no_typed_value_in_the_recording_for_any_fixture() { @@ -36,3 +37,103 @@ async fn nothing_typed_reaches_the_file_or_the_output() { assert_holds_no_typed_value(text, &[USER, PASS]); } } + +/// A page that copies what is typed into a field into the field's own `id` must not get it into +/// the recording through the descriptor or the selectors, not even on the Enter-key report and +/// not for a value of two characters (which no search for typed values can safely look for). +#[tokio::test] +async fn a_page_that_mirrors_the_value_into_an_attribute_leaks_nothing() { + use kharon_recording::Step; + let Some(run) = Run::start("mirror", "/mirror").await else { + return; + }; + run.begin("/mirror").await; + run.person.type_into("input[name=user]", "q7").await; + run.person.type_into("input[name=pass]", PASS).await; + run.person.press_enter().await; + run.person.wait_for_path("/home").await; + let Outcome::Recorded(r) = run.finish(End::Confirmed).await else { + panic!("expected a recording"); + }; + assert_holds_no_typed_value(&r.to_json(), &[PASS]); + let mut seen = 0; + for step in &r.steps { + let element = match step { + Step::Input { element, .. } | Step::EnterKey { element, .. } => element, + _ => continue, + }; + seen += 1; + let texts = element + .selectors + .iter() + .chain(element.id.iter()) + .chain(element.name.iter()) + .chain(element.autocomplete.iter()); + for t in texts { + assert!(!t.contains("q7"), "a selector or attribute holds the value"); + } + assert!(!element.selectors.is_empty(), "the field keeps a selector"); + } + assert!(seen >= 3, "two inputs and the Enter key were recorded"); +} + +/// The listener itself, run in the page's main world with a stand-in binding, so the raw reports +/// can be read. Whatever the recorder later strips, the script must not send the typed text in +/// a descriptor or a selector. +#[tokio::test] +async fn the_listener_never_describes_an_element_by_what_was_typed_into_it() { + use kharon_teach::events::ListenerReport; + let Some(run) = Run::start("listener_raw", "/mirror").await else { + return; + }; + run.begin("/mirror").await; + let source = kharon_teach::session::LISTENER_JS.replace("__NONCE__", "raw-test"); + let install = format!( + "window.__reports=[];window.__kharonTeach=(p)=>{{window.__reports.push(p)}};\ + document.addEventListener('submit',(e)=>e.preventDefault(),true);{source}\n;'ok'" + ); + assert_eq!(run.person.eval(&install).await.as_deref(), Some("ok")); + run.person.type_into("input[name=user]", "q7").await; + run.person.press_enter().await; + let raw = run + .person + .eval("JSON.stringify(window.__reports)") + .await + .expect("the reports"); + let reports: Vec<String> = serde_json::from_str(&raw).unwrap(); + let mut seen = (0, 0); + for r in &reports { + let Ok(rep) = serde_json::from_str::<ListenerReport>(r) else { + continue; // the hello + }; + match rep.kind { + kharon_teach::events::ReportKind::Input => seen.0 += 1, + kharon_teach::events::ReportKind::EnterKey => seen.1 += 1, + _ => {} + } + let e = &rep.element; + let texts = e + .selectors + .iter() + .chain(e.id.iter()) + .chain(e.name.iter()) + .chain(e.autocomplete.iter()) + .chain(e.placeholder.iter()) + .chain(e.aria_label.iter()); + for t in texts { + assert!( + !t.contains("q7"), + "a descriptor or selector holds typed text" + ); + } + assert!( + !e.selectors.is_empty(), + "the field keeps a selector that does not hold it" + ); + } + assert!( + seen.0 >= 1 && seen.1 >= 1, + "an input and an Enter key were reported" + ); + let _ = run.finish(End::Discard).await; +} diff --git a/crates/kharon-teach/tests/req_tch_005_shapes.rs b/crates/kharon-teach/tests/req_tch_005_shapes.rs index 2f981be..bba8e64 100644 --- a/crates/kharon-teach/tests/req_tch_005_shapes.rs +++ b/crates/kharon-teach/tests/req_tch_005_shapes.rs @@ -5,8 +5,10 @@ mod support; use kharon_recording::{ EndedBy, ListenerWorld, PasswordLeg, Recording, ResourceKind, RungCeiling, Shape, }; +use kharon_teach::session::POST_DATA_FETCHES; use kharon_teach::session::{End, Outcome}; -use support::{COOKIE_VALUE, Login, Run, record_login}; +use kharon_teach::testing::assert_holds_no_typed_value; +use support::{COOKIE_VALUE, Login, PASS, Run, USER, record_login}; #[tokio::test] async fn form_page_classifies_as_form_post() { @@ -116,3 +118,197 @@ fn typed_elements(r: &Recording) -> Vec<String> { }) .collect() } + +fn request_to<'a>(r: &'a Recording, path: &str) -> &'a kharon_recording::StoredRequest { + r.requests + .iter() + .find(|q| q.method == "POST" && q.url.path == path) + .unwrap_or_else(|| panic!("no stored POST to {path}")) +} + +fn has_password_hit(parts: &[kharon_recording::FieldShape], key: &str) -> bool { + parts.iter().any(|p| { + p.key == key + && p.value + .markers + .iter() + .any(|m| m.role == kharon_recording::Role::Password) + }) +} + +#[tokio::test] +async fn a_form_data_post_is_stored_as_multipart_with_the_password_found() { + let Some(r) = record_login("script_formdata", Login::ScriptFormData).await else { + return; + }; + assert_eq!( + r.classification.shape, + Shape::ScriptPost, + "{:?}", + r.classification.notes + ); + match &request_to(&r, "/api/login-form").body { + kharon_recording::BodyShape::Multipart { parts } => { + assert!( + has_password_hit(parts, "pass"), + "no password hit on the pass part" + ) + } + _ => panic!("the body is not multipart"), + } + assert!(!r.flags.body_unavailable); + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); +} + +#[tokio::test] +async fn a_blob_body_is_fetched_with_get_request_post_data() { + use std::sync::atomic::Ordering; + let before = POST_DATA_FETCHES.load(Ordering::Relaxed); + let Some(r) = record_login("script_blob", Login::ScriptBlob).await else { + return; + }; + // Whether Chromium sends a Blob body inline or not, the stored shape must carry the password. + assert_eq!( + r.classification.shape, + Shape::ScriptPost, + "{:?}", + r.classification.notes + ); + match &request_to(&r, "/api/login-blob").body { + kharon_recording::BodyShape::Json { .. } | kharon_recording::BodyShape::Opaque { .. } => {} + other => panic!("unexpected body shape kind {}", shape_name(other)), + } + assert!(!r.flags.body_unavailable); + assert!( + POST_DATA_FETCHES.load(Ordering::Relaxed) > before, + "the body was inline: getRequestPostData never ran" + ); + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); +} + +fn shape_name(b: &kharon_recording::BodyShape) -> &'static str { + use kharon_recording::BodyShape::*; + match b { + None => "none", + Unavailable => "unavailable", + Urlencoded { .. } => "urlencoded", + Json { .. } => "json", + Multipart { .. } => "multipart", + Opaque { .. } => "opaque", + } +} + +#[tokio::test] +async fn the_main_world_fallback_still_records_a_login() { + use kharon_teach::session::WorldProbe; + let Some(run) = Run::start_with("main_world", "/form", WorldProbe::SkipIsolated).await else { + return; + }; + run.begin("/form").await; + run.person.type_into("input[name=user]", USER).await; + run.person.type_into("input[name=pass]", PASS).await; + run.person.click("#go").await; + run.person.wait_for_path("/home").await; + let Outcome::Recorded(r) = run.finish(End::Confirmed).await else { + panic!("expected a recording"); + }; + assert_eq!(r.listener_world, ListenerWorld::Main); + assert_eq!( + r.classification.shape, + Shape::FormPost, + "{:?}", + r.classification.notes + ); + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); +} + +#[tokio::test] +async fn children_named_like_form_properties_do_not_hide_the_form() { + let Some(r) = record_login("clobber", Login::Clobber).await else { + return; + }; + assert_eq!( + r.classification.shape, + Shape::FormPost, + "{:?}", + r.classification.notes + ); + let form = r + .steps + .iter() + .find_map(|s| match s { + kharon_recording::Step::Submit { form, .. } => Some(form.clone()), + _ => None, + }) + .expect("a submit step"); + assert_eq!(form.method, "post"); + assert_eq!(form.action.map(|a| a.path).as_deref(), Some("/form-login")); + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); +} + +#[tokio::test] +async fn a_popup_sets_the_flag_and_is_not_a_login() { + let Some(run) = Run::start("popup", "/popup").await else { + return; + }; + run.begin("/popup").await; + run.person.click("#go").await; + run.person.wait_for_pages(2).await; + let Outcome::Recorded(r) = run.finish(End::Confirmed).await else { + panic!("expected a recording"); + }; + assert!(r.flags.popup); + assert_eq!( + r.classification.shape, + Shape::Unrecognized, + "{:?}", + r.classification.notes + ); +} + +#[tokio::test] +async fn a_login_in_a_cross_origin_frame_sets_framed_login() { + let Some(run) = Run::start("framed", "/frame-outer").await else { + return; + }; + run.begin("/frame-outer").await; + let frame = run.person.attach_frame("localhost").await; + run.person + .type_into_frame(&frame, "input[name=user]", USER) + .await; + run.person + .type_into_frame(&frame, "input[name=pass]", PASS) + .await; + run.person.press_enter().await; + run.person.wait_for_frame_path(&frame, "/home").await; + let Outcome::Recorded(r) = run.finish(End::Confirmed).await else { + panic!("expected a recording"); + }; + assert!(r.flags.framed_login, "{:?}", r.classification.notes); + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); +} + +#[tokio::test] +async fn a_checkbox_is_not_a_typed_value() { + let Some(run) = Run::start("checkbox", "/form").await else { + return; + }; + run.begin("/form").await; + run.person.click("#rem").await; + run.person.type_into("input[name=user]", USER).await; + run.person.type_into("input[name=pass]", PASS).await; + run.person.click("#go").await; + run.person.wait_for_path("/home").await; + let Outcome::Recorded(r) = run.finish(End::Confirmed).await else { + panic!("expected a recording"); + }; + assert_eq!(r.stripped.typed_values_removed, 2); + assert_eq!( + r.classification.shape, + Shape::FormPost, + "{:?}", + r.classification.notes + ); + assert!(r.requests.iter().any(|q| q.url.path == "/form-login")); + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); +} diff --git a/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs index d85eb0d..73719e2 100644 --- a/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs +++ b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs @@ -19,6 +19,21 @@ fn row_counts(db: &Path, tables: &[&str]) -> Option<Vec<u64>> { )); let _ = std::fs::remove_file(©); std::fs::copy(db, ©).unwrap(); + // A write-ahead log or journal beside the database holds rows the main file does not yet. + let siblings: Vec<PathBuf> = ["-wal", "-journal"] + .iter() + .map(|suffix| { + let mut from = db.as_os_str().to_owned(); + from.push(suffix); + let mut to = copy.as_os_str().to_owned(); + to.push(suffix); + let (from, to) = (PathBuf::from(from), PathBuf::from(to)); + if from.exists() { + let _ = std::fs::copy(&from, &to); + } + to + }) + .collect(); let script = [ "import sqlite3,sys", "c=sqlite3.connect(sys.argv[1])", @@ -35,6 +50,9 @@ fn row_counts(db: &Path, tables: &[&str]) -> Option<Vec<u64>> { .args(tables) .output(); let _ = std::fs::remove_file(©); + for sib in &siblings { + let _ = std::fs::remove_file(sib); + } let out = match out { Ok(o) if o.status.success() => o, Ok(o) => panic!( @@ -56,10 +74,13 @@ fn row_counts(db: &Path, tables: &[&str]) -> Option<Vec<u64>> { ) } +/// True when `path` goes away and stays away: a second look a second later, because a helper +/// process of a killed browser can recreate files in a directory that was just removed. async fn gone(path: &Path) -> bool { for _ in 0..100 { if !path.exists() { - return true; + tokio::time::sleep(Duration::from_secs(1)).await; + return !path.exists(); } tokio::time::sleep(Duration::from_millis(50)).await; } @@ -82,22 +103,41 @@ async fn nothing_is_saved_in_the_profile_during_a_login() { run.person.wait_for_path("/home").await; let profile = run.ready.profile_dir.clone(); assert!(profile.is_dir(), "the profile directory exists"); - for (file, tables) in [ - ("Login Data", &["logins"][..]), - ( - "Web Data", - &["autofill", "autofill_profiles", "credit_cards"][..], - ), - ] { - let db = profile.join("Default").join(file); + // At least one of the two databases is there to be looked at (Chromium makes them lazily). + let dbs = ["Login Data", "Web Data"].map(|f| profile.join("Default").join(f)); + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while !dbs.iter().any(|d| d.exists()) { + assert!( + tokio::time::Instant::now() < deadline, + "neither Login Data nor Web Data appeared in the profile" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + for (db, tables) in dbs.iter().zip([ + &["logins"][..], + &["autofill", "autofill_profiles", "credit_cards"][..], + ]) { if !db.exists() { - eprintln!("no {file} in the profile: nothing was saved"); continue; } - if let Some(counts) = row_counts(&db, tables) { - assert!(counts.iter().all(|n| *n == 0), "{file}: {counts:?}"); + if let Some(counts) = row_counts(db, tables) { + assert!( + counts.iter().all(|n| *n == 0), + "{}: {counts:?}", + db.display() + ); } } + // The preferences the recorder wrote are still in force after the login. + #[derive(serde::Deserialize)] + struct Prefs { + credentials_enable_service: bool, + } + let prefs: Prefs = serde_json::from_str( + &std::fs::read_to_string(profile.join("Default").join("Preferences")).unwrap(), + ) + .unwrap(); + assert!(!prefs.credentials_enable_service); let _ = run.finish(End::Discard).await; } @@ -318,3 +358,43 @@ fn sigint_exits_1_and_removes_the_profile() { fn sigterm_exits_1_and_removes_the_profile() { interrupted_by("TERM"); } + +async fn outcome_is_unconfirmed_or_never_started(outcome: Outcome) { + match outcome { + Outcome::Recorded(r) => { + assert_eq!(r.ended_by, EndedBy::WindowClosed); + assert!(!r.success_confirmed); + } + Outcome::NeverStarted => {} + Outcome::Discarded => panic!("a dead browser is not a discard"), + } +} + +#[tokio::test] +async fn a_browser_killed_from_outside_ends_the_session_and_the_profile_stays_gone() { + let Some(mut run) = Run::start("killed_outside", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + type_a_login(&run).await; + let pid = support::Person::browser_pid(&profile).expect("the browser process"); + let status = std::process::Command::new("kill") + .args(["-KILL", &pid.to_string()]) + .status() + .unwrap(); + assert!(status.success()); + outcome_is_unconfirmed_or_never_started(run.wait().await).await; + assert!(gone(&profile).await, "the profile is still there"); +} + +#[tokio::test] +async fn a_browser_that_quits_by_itself_ends_the_session_and_the_profile_stays_gone() { + let Some(mut run) = Run::start("quits_itself", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + type_a_login(&run).await; + run.person.close_browser().await; + outcome_is_unconfirmed_or_never_started(run.wait().await).await; + assert!(gone(&profile).await, "the profile is still there"); +} diff --git a/crates/kharon-teach/tests/support/mod.rs b/crates/kharon-teach/tests/support/mod.rs index 56f6ba6..b9bc986 100644 --- a/crates/kharon-teach/tests/support/mod.rs +++ b/crates/kharon-teach/tests/support/mod.rs @@ -36,7 +36,9 @@ pub fn chromium_or_skip(test: &str) -> Option<PathBuf> { use kharon_teach::browser::DevTools; use kharon_teach::cdp::Cdp; -use kharon_teach::session::{End, Outcome, Ready, RecordOptions, SessionError, record}; +use kharon_teach::session::{ + End, Outcome, Ready, RecordOptions, SessionError, WorldProbe, record_with, +}; use serde::{Deserialize, Serialize}; use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -52,7 +54,8 @@ pub const COOKIE_VALUE: &str = "COOKIEVALUE123"; static GATE: Semaphore = Semaphore::const_new(2); const FORM: &str = "<!doctype html><title>form
\ -
"; +\ +"; const SCRIPT: &str = "script\ "; + +const SCRIPT_BLOB: &str = "blob\ +"; + +/// A form that copies what is typed into the user field into the field's own `id`. +const MIRROR: &str = "mirror
\ +
\ +"; + +/// A form whose children replace `action`, `method`, `id` and `tagName` on the form element. +const CLOBBER: &str = "clobber
\ +\ +
"; + +const POPUP: &str = "popup\ +"; + +/// `{HOST}` is the same server under another host name, so it is another origin. +const FRAME_OUTER: &str = "outer\ +"; + const HOME: &str = "home

signed in

"; fn response(status: &str, extra: &str, body: &str, ctype: &str) -> String { @@ -89,7 +131,7 @@ fn response(status: &str, extra: &str, body: &str, ctype: &str) -> String { ) } -fn route(method: &str, path: &str) -> String { +fn route(method: &str, path: &str, host: &str) -> String { let html = "text/html; charset=utf-8"; match (method, path) { ("GET", "/form") => response("200 OK", "", FORM, html), @@ -103,6 +145,18 @@ fn route(method: &str, path: &str) -> String { ("POST", "/api/login") => response("200 OK", "", "{\"ok\":true}", "application/json"), ("GET", "/two-step") => response("200 OK", "", TWO_STEP, html), ("POST", "/two-step/password") => response("200 OK", "", TWO_STEP_PASSWORD, html), + ("GET", "/script-formdata") => response("200 OK", "", SCRIPT_FORMDATA, html), + ("POST", "/api/login-form") => response("200 OK", "", "{\"ok\":true}", "application/json"), + ("GET", "/script-blob") => response("200 OK", "", SCRIPT_BLOB, html), + ("POST", "/api/login-blob") => response("200 OK", "", "{\"ok\":true}", "application/json"), + ("GET", "/mirror") => response("200 OK", "", MIRROR, html), + ("GET", "/clobber") => response("200 OK", "", CLOBBER, html), + ("GET", "/popup") => response("200 OK", "", POPUP, html), + ("GET", "/frame-outer") => { + // Same server, another host name: a different origin. + let other = host.replace("127.0.0.1", "localhost"); + response("200 OK", "", &FRAME_OUTER.replace("{HOST}", &other), html) + } ("GET", "/hashed") => response("200 OK", "", HASHED, html), ("GET", "/home") => response("200 OK", "", HOME, html), _ => response("404 Not Found", "", "not found", "text/plain"), @@ -139,7 +193,15 @@ async fn serve_one(mut sock: tokio::net::TcpStream) { } } let path = path.split('?').next().unwrap_or(""); - let _ = sock.write_all(route(method, path).as_bytes()).await; + let host = head + .lines() + .find_map(|l| { + l.to_ascii_lowercase() + .starts_with("host:") + .then(|| l[5..].trim().to_owned()) + }) + .unwrap_or_default(); + let _ = sock.write_all(route(method, path, &host).as_bytes()).await; let _ = sock.shutdown().await; } @@ -191,6 +253,8 @@ struct TargetRow { id: String, #[serde(rename = "type")] kind: String, + #[serde(default)] + url: String, } #[derive(Serialize)] @@ -331,7 +395,7 @@ impl Person { t.infos.into_iter().find(|t| t.kind == "page").map(|t| t.id) } - async fn eval(&self, expression: &str) -> Option { + pub async fn eval(&self, expression: &str) -> Option { let r: EvalResult = self .cdp .call( @@ -469,6 +533,11 @@ impl Run { /// Starts `record` headless on the fixture's `login_path` and waits until the notice page is /// showing. `None` when there is no browser and the run does not require one. pub async fn start(test: &str, login_path: &str) -> Option { + Self::start_with(test, login_path, WorldProbe::Normal).await + } + + /// [`Run::start`] with a way to make the isolated world fail. + pub async fn start_with(test: &str, login_path: &str, probe: WorldProbe) -> Option { let bin = chromium_or_skip(test)?; let permit = GATE.acquire().await.unwrap(); let fixture = Fixture::start().await; @@ -476,7 +545,7 @@ impl Run { let (ready_tx, ready_rx) = oneshot::channel(); let (end_tx, end_rx) = oneshot::channel(); let task = tokio::spawn(async move { - record( + record_with( RecordOptions { login_url: &login, browser: &bin, @@ -485,6 +554,7 @@ impl Run { }, ready_tx, end_rx, + probe, ) .await }); @@ -547,6 +617,10 @@ pub enum Login { Script, TwoStep, Hashed, + ScriptFormData, + ScriptBlob, + Mirror, + Clobber, } impl Login { @@ -556,6 +630,10 @@ impl Login { Login::Script => "/script", Login::TwoStep => "/two-step", Login::Hashed => "/hashed", + Login::ScriptFormData => "/script-formdata", + Login::ScriptBlob => "/script-blob", + Login::Mirror => "/mirror", + Login::Clobber => "/clobber", } } } @@ -567,7 +645,17 @@ pub async fn record_login(test: &str, which: Login) -> Option { + Login::Mirror => { + p.type_into("input[name=user]", USER).await; + p.type_into("input[name=pass]", PASS).await; + p.press_enter().await; + } + Login::Form + | Login::Script + | Login::Hashed + | Login::ScriptFormData + | Login::ScriptBlob + | Login::Clobber => { p.type_into("input[name=user]", USER).await; p.type_into("input[name=pass]", PASS).await; p.click("#go").await; @@ -711,3 +799,143 @@ pub async fn run_program(test: &str, form_path_and_query: &str) -> Option &str { path_and_query.split('?').next().unwrap_or("") } + +#[derive(Serialize)] +struct FlattenAttach<'a> { + #[serde(rename = "targetId")] + target_id: &'a str, + flatten: bool, +} + +impl Person { + /// How many page targets the browser has. + pub async fn page_count(&self) -> usize { + let t: Targets = self + .cdp + .call("Target.getTargets", &Ignored2 {}, None) + .await + .unwrap(); + t.infos.iter().filter(|t| t.kind == "page").count() + } + + pub async fn wait_for_pages(&self, n: usize) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while self.page_count().await < n { + assert!(tokio::time::Instant::now() < deadline, "no page number {n}"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + /// The browser closes itself, as when someone quits it from its menu. + pub async fn close_browser(&self) { + let _: Result = self.cdp.call("Browser.close", &Ignored2 {}, None).await; + } + + /// A session on the page's embedded frame whose URL is on `host`, waiting for it to exist. + /// The frame is in another process, so the page's own session cannot reach into it. + pub async fn attach_frame(&self, host: &str) -> String { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + let t: Targets = self + .cdp + .call("Target.getTargets", &Ignored2 {}, None) + .await + .unwrap(); + if let Some(f) = t + .infos + .iter() + .find(|t| t.kind == "iframe" && t.url.contains(host)) + { + let a: Attached = self + .cdp + .call( + "Target.attachToTarget", + &FlattenAttach { + target_id: &f.id, + flatten: true, + }, + None, + ) + .await + .unwrap(); + return a.session_id; + } + assert!( + tokio::time::Instant::now() < deadline, + "no frame target for {host}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + async fn eval_in(&self, session: &str, expression: &str) -> Option { + let r: EvalResult = self + .cdp + .call( + "Runtime.evaluate", + &EvalParam { + expression, + return_by_value: true, + }, + Some(session), + ) + .await + .ok()?; + r.result.value + } + + /// Focus `css` inside the frame, then type into it. + pub async fn type_into_frame(&self, frame: &str, css: &str, text: &str) { + let sel = serde_json::to_string(css).unwrap(); + let focus = format!( + "(()=>{{const e=document.querySelector({sel});if(!e)return 'no';e.focus();return 'ok'}})()" + ); + // The frame's document may not be there yet when its target first appears. + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while self.eval_in(frame, &focus).await.as_deref() != Some("ok") { + assert!( + tokio::time::Instant::now() < deadline, + "the field {css} never appeared in the frame" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + let _: Ignored = self + .cdp + .call("Input.insertText", &TextParam { text }, Some(frame)) + .await + .unwrap(); + } + + pub async fn wait_for_frame_path(&self, frame: &str, path: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + if self.eval_in(frame, "location.pathname").await.as_deref() == Some(path) { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "the frame never reached {path}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + /// The pid of the main browser process of the profile at `profile`, found through `/proc`. + pub fn browser_pid(profile: &std::path::Path) -> Option { + let want = format!("--user-data-dir={}", profile.display()); + for e in std::fs::read_dir("/proc").ok()?.flatten() { + let Ok(pid) = e.file_name().to_string_lossy().parse::() else { + continue; + }; + let Ok(cmd) = std::fs::read(e.path().join("cmdline")) else { + continue; + }; + // Chrome rewrites its own argv into one space-separated string; no path here has a space. + let args: Vec<&[u8]> = cmd.split(|b| *b == 0 || *b == b' ').collect(); + if args.contains(&want.as_bytes()) && !args.iter().any(|a| a.starts_with(b"--type=")) { + return Some(pid); + } + } + None + } +} From 776b3d0c0e796346c308555092487444ddccaa04 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 06:32:18 -0500 Subject: [PATCH 24/36] fix(teach): exit without waiting on stdin; tighter scans and file handling (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-teach/src/browser.rs | 4 +- crates/kharon-teach/src/cli.rs | 46 +++- crates/kharon-teach/src/lib.rs | 73 ++++-- crates/kharon-teach/src/main.rs | 10 + .../tests/req_tch_003_file_contents.rs | 30 ++- .../tests/req_tch_004_no_network.rs | 30 +-- .../tests/req_tch_006_profile_and_visible.rs | 234 ++++++++++++++++-- crates/kharon-teach/tests/support/mod.rs | 36 +++ 8 files changed, 371 insertions(+), 92 deletions(-) diff --git a/crates/kharon-teach/src/browser.rs b/crates/kharon-teach/src/browser.rs index 3dc8525..08d20ff 100644 --- a/crates/kharon-teach/src/browser.rs +++ b/crates/kharon-teach/src/browser.rs @@ -632,7 +632,9 @@ mod tests { /// A non-dumpable process's `/proc/self` belongs to root, so a sweep that learned its uid /// from there would compare against 0 and never remove anything. The check runs in a child - /// process (this test binary again) so the parent stays dumpable. + /// process (this test binary again) so the parent stays dumpable. On newer kernels + /// `/proc/self` stays owned by the user after that call, so the old code passes this test + /// too: it exercises the sweep under `PR_SET_DUMPABLE 0` and is not proof of the fix. #[test] fn sweep_still_removes_a_leftover_when_the_process_is_non_dumpable() { const MARK: &str = "KHARON_TEACH_SWEEP_NONDUMPABLE_DIR"; diff --git a/crates/kharon-teach/src/cli.rs b/crates/kharon-teach/src/cli.rs index d4f55e4..5403e56 100644 --- a/crates/kharon-teach/src/cli.rs +++ b/crates/kharon-teach/src/cli.rs @@ -88,11 +88,8 @@ pub fn validate_login_url(s: &str) -> Result { } } -/// `-.kharon-recording.json`. Every character of the host outside -/// `[a-z0-9.-]` becomes `_`, so an IPv6 host puts no `[`, `]` or `:` in a file name. -pub fn default_file_name(host: &str, utc: SystemTime) -> String { - let host: String = host - .to_ascii_lowercase() +fn sanitize_host(host: &str) -> String { + host.to_ascii_lowercase() .chars() .map(|c| { if c.is_ascii_lowercase() || c.is_ascii_digit() || c == '.' || c == '-' { @@ -101,10 +98,16 @@ pub fn default_file_name(host: &str, utc: SystemTime) -> String { '_' } }) - .collect(); + .collect() +} + +/// `-.kharon-recording.json`. Every character of the host outside +/// `[a-z0-9.-]` becomes `_`, so an IPv6 host puts no `[`, `]` or `:` in a file name. +pub fn default_file_name(host: &str, utc: SystemTime) -> String { let t = time::OffsetDateTime::from(utc); format!( - "{host}-{:04}{:02}{:02}T{:02}{:02}{:02}Z{FILE_SUFFIX}", + "{}-{:04}{:02}{:02}T{:02}{:02}{:02}Z{FILE_SUFFIX}", + sanitize_host(host), t.year(), u8::from(t.month()), t.day(), @@ -114,6 +117,26 @@ pub fn default_file_name(host: &str, utc: SystemTime) -> String { ) } +/// [`default_file_name`] for the instant a recording says it ended (`ended_at`, UTC RFC 3339 +/// as `session::utc_rfc3339` writes it). `None` when the text is not in that shape. +pub fn default_file_name_at(host: &str, ended_at: &str) -> Option { + let digits: Vec = ended_at + .chars() + .take(19) + .filter(char::is_ascii_digit) + .collect(); + if digits.len() != 14 || !ended_at.is_ascii() { + return None; + } + let d: String = digits.into_iter().collect(); + Some(format!( + "{}-{}T{}Z{FILE_SUFFIX}", + sanitize_host(host), + &d[..8], + &d[8..] + )) +} + /// `dir` is an existing directory this user can create files in. Tried, not read from /// permission bits: a temporary file is created and removed. pub fn check_dir_writable(dir: &Path) -> Result<(), Refusal> { @@ -314,6 +337,15 @@ pub(crate) mod tests { ); } + #[test] + fn file_name_follows_the_recordings_end_time() { + assert_eq!( + default_file_name_at("Ex.test", "2026-11-18T04:04:00.123Z").unwrap(), + "ex.test-20261118T040400Z.kharon-recording.json" + ); + assert!(default_file_name_at("ex.test", "yesterday").is_none()); + } + #[test] fn file_name_sanitizes_the_host() { let n = default_file_name("[::1]", SystemTime::UNIX_EPOCH); diff --git a/crates/kharon-teach/src/lib.rs b/crates/kharon-teach/src/lib.rs index 4544a3d..0cabaa6 100644 --- a/crates/kharon-teach/src/lib.rs +++ b/crates/kharon-teach/src/lib.rs @@ -15,8 +15,8 @@ pub mod strip; pub mod testing; use cli::{ - Cli, Command, Refusal, check_dir_writable, check_out_path, default_file_name, summary, - validate_login_url, + Cli, Command, Refusal, check_dir_writable, check_out_path, default_file_name, + default_file_name_at, summary, validate_login_url, }; use kharon_recording::Recording; use session::{End, Outcome, Ready, RecordOptions, record}; @@ -96,7 +96,11 @@ pub enum Naming { /// to `.9` added (before the `.kharon-recording.json` ending for a generated name, at the end /// for an `--out` name) is tried, so a demonstration is not lost to a name clash. #[doc(hidden)] -pub fn write_new(first: &Path, naming: Naming, text: &str) -> std::io::Result { +pub fn write_new( + first: &Path, + naming: Naming, + text: &str, +) -> Result { use std::os::unix::fs::OpenOptionsExt; let name = first .file_name() @@ -120,17 +124,26 @@ pub fn write_new(first: &Path, naming: Naming, text: &str) -> std::io::Result { - f.write_all(text.as_bytes())?; - f.flush()?; - return Ok(path); + return match f.write_all(text.as_bytes()).and_then(|()| f.flush()) { + Ok(()) => Ok(path), + Err(e) => { + // Never leave a truncated file where the next run would refuse to write. + drop(f); + let _ = std::fs::remove_file(&path); + Err((path, e)) + } + }; } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} - Err(e) => return Err(e), + Err(e) => return Err((path, e)), } } - Err(std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - "ten files of that name already exist", + Err(( + first.to_owned(), + std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "ten files of that name already exist", + ), )) } @@ -334,13 +347,17 @@ pub async fn run_with( drop(rec); match outcome { Ok(Outcome::Recorded(r)) => { - let first = match &prep.target { - Target::Explicit(p) => p.clone(), - Target::Here => PathBuf::from(default_file_name( - prep.url.host_str().unwrap_or("login"), - std::time::SystemTime::now(), - )), - }; + let first = + match &prep.target { + Target::Explicit(p) => p.clone(), + Target::Here => { + let host = prep.url.host_str().unwrap_or("login"); + // The name carries the instant the recording ended, not the time of the write. + PathBuf::from(default_file_name_at(host, &r.ended_at).unwrap_or_else( + || default_file_name(host, std::time::SystemTime::now()), + )) + } + }; let naming = match prep.target { Target::Explicit(_) => Naming::Explicit, Target::Here => Naming::Default, @@ -351,9 +368,9 @@ pub async fn run_with( let _ = writeln!(stdout, "Saved: {}", path.display()); Exit::Done } - Err(e) => fail( + Err((at, e)) => fail( &mut stderr, - &format_args!("cannot write {}: {e}", first.display()), + &format_args!("cannot write {}: {e}", at.display()), Exit::Failed, ), } @@ -382,12 +399,18 @@ mod tests { Cli::try_parse_from(std::iter::once("kharon-teach").chain(args.iter().copied())).unwrap() } - fn temp_listing() -> Vec { + /// Profile directories in the temp directory that name THIS process as their owner. Other + /// tests and other processes make profiles too; only ours can come from `run_with` here. + fn my_profiles() -> Vec { + let me = std::process::id().to_string(); std::fs::read_dir(std::env::temp_dir()) .unwrap() .flatten() - .map(|e| e.file_name()) - .filter(|n| n.to_string_lossy().starts_with("kharon-teach-")) + .filter(|e| e.file_name().to_string_lossy().starts_with("kharon-teach-")) + .map(|e| e.path()) + .filter(|p| { + std::fs::read_to_string(p.join(browser::OWNER_FILE)).is_ok_and(|t| t.trim() == me) + }) .collect() } @@ -428,7 +451,7 @@ mod tests { #[tokio::test] async fn refusals_exit_2_before_the_notice() { - let before = temp_listing(); + let before = my_profiles(); let d = private_dir("refuse"); let browser = fake_browser(&d); let existing = d.join("taken.json"); @@ -463,9 +486,9 @@ mod tests { assert!(r.err.starts_with("kharon-teach: "), "{}", r.err); assert_eq!(r.err.lines().count(), 1, "{}", r.err); } - let after = temp_listing(); + let after = my_profiles(); assert!( - after.iter().all(|n| before.contains(n)), + after.iter().all(|p| before.contains(p)), "a refusal created a profile directory" ); std::fs::remove_dir_all(&d).unwrap(); diff --git a/crates/kharon-teach/src/main.rs b/crates/kharon-teach/src/main.rs index 395e240..e34352b 100644 --- a/crates/kharon-teach/src/main.rs +++ b/crates/kharon-teach/src/main.rs @@ -20,5 +20,15 @@ fn main() -> ExitCode { std::io::stdout(), std::io::stderr(), )); + // `run` has already closed the browser and removed its profile, whatever ended the session + // (the window, a confirm, Ctrl-C or SIGTERM). What is left is the blocking thread of the + // end-prompt read, which cannot be cancelled and would hold the process until Enter, so + // the runtime is shut down without waiting for it. + { + use std::io::Write; + let _ = std::io::stdout().flush(); + let _ = std::io::stderr().flush(); + } + runtime.shutdown_background(); exit.into() } diff --git a/crates/kharon-teach/tests/req_tch_003_file_contents.rs b/crates/kharon-teach/tests/req_tch_003_file_contents.rs index 944739c..ca61704 100644 --- a/crates/kharon-teach/tests/req_tch_003_file_contents.rs +++ b/crates/kharon-teach/tests/req_tch_003_file_contents.rs @@ -35,11 +35,29 @@ async fn the_file_is_0600_and_holds_only_what_the_spec_keeps() { "a page title is in the file: {title}" ); } - // Headers are reduced to the content type. - for r in &rec.requests { - if let Some(ct) = &r.content_type { - assert!(!ct.is_empty()); - } + // Headers are reduced to the content type: no other header the fixture and the browser + // exchanged is in the file, by name or by value. + let lower = text.to_ascii_lowercase(); + for header in [ + "set-cookie", + "user-agent", + "referer", + "accept-language", + "accept-encoding", + "content-length", + "location", + "\"cookie\"", + ] { + assert!(!lower.contains(header), "header {header} is in the file"); } - assert!(!text.to_ascii_lowercase().contains("set-cookie:")); + // The cookie was set by the response, and the file says only that it was. + let login = rec + .requests + .iter() + .find(|r| r.url.path == "/form-login") + .expect("the sign-in request was stored"); + assert!( + login.response.as_ref().is_some_and(|r| r.set_cookie), + "the response's Set-Cookie was not noted as a flag" + ); } diff --git a/crates/kharon-teach/tests/req_tch_004_no_network.rs b/crates/kharon-teach/tests/req_tch_004_no_network.rs index 55eff43..7e20fae 100644 --- a/crates/kharon-teach/tests/req_tch_004_no_network.rs +++ b/crates/kharon-teach/tests/req_tch_004_no_network.rs @@ -1,34 +1,8 @@ //! SPEC req_tch_004: the recorder opens no socket but loopback CDP, and links no TLS or HTTP client. -use std::path::{Path, PathBuf}; +mod support; -/// Shipped code only: the text before the first `#[cfg(test)]` of each file under `src/`. -fn shipped_sources() -> Vec<(PathBuf, String)> { - fn walk(dir: &Path, out: &mut Vec) { - for e in std::fs::read_dir(dir).unwrap().flatten() { - let p = e.path(); - if p.is_dir() { - walk(&p, out); - } else if p.extension().is_some_and(|x| x == "rs") { - out.push(p); - } - } - } - let mut files = Vec::new(); - walk( - &Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), - &mut files, - ); - assert!(!files.is_empty()); - files - .into_iter() - .map(|p| { - let text = std::fs::read_to_string(&p).unwrap(); - let shipped = text.split("#[cfg(test)]").next().unwrap().to_owned(); - (p, shipped) - }) - .collect() -} +use support::shipped_sources; #[test] fn the_crate_listens_on_nothing_and_has_no_tls() { diff --git a/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs index 73719e2..0cbaade 100644 --- a/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs +++ b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs @@ -242,38 +242,50 @@ async fn a_stale_profile_is_swept_at_the_next_start() { #[test] fn the_shipped_main_never_asks_for_headless() { - fn walk(dir: &Path, out: &mut Vec) { - for e in std::fs::read_dir(dir).unwrap().flatten() { - let p = e.path(); - if p.is_dir() { - walk(&p, out); - } else if p.extension().is_some_and(|x| x == "rs") { - out.push(p); - } - } - } - let mut files = Vec::new(); - walk( - &Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), - &mut files, - ); - for p in files { - let text = std::fs::read_to_string(&p).unwrap(); - let shipped = text.split("#[cfg(test)]").next().unwrap(); + let sources = support::shipped_sources(); + let mut env_reads = Vec::new(); + for (p, shipped) in &sources { + let name = p.file_name().unwrap().to_string_lossy().into_owned(); assert!( !shipped.contains("headless: true"), "{} asks for headless", p.display() ); - // Browser discovery reads PATH; nothing else reads the environment. - if !p.ends_with("browser.rs") { - assert!( - !shipped.contains("env::var"), - "{} reads the environment", - p.display() - ); + // Every `headless:` initializer outside the session and the browser launcher is the + // literal `false` or a pass-through of the hook inside `run_with`. + if name != "session.rs" && name != "browser.rs" { + for (n, line) in shipped.lines().enumerate() { + let t = line.trim(); + if t.starts_with("//") || !t.contains("headless:") { + continue; + } + let ok = t == "headless: false," || t == "headless: hooks.headless,"; + let decl = t.starts_with("pub headless: bool"); + assert!(ok || decl, "{}:{}: {t}", p.display(), n + 1); + } + } + for pat in [ + "env::var(", + "env::var_os(", + "env::vars(", + "env::vars_os(", + "use std::env::{", + "use std::env::var", + "use std::env;", + ] { + for (n, line) in shipped.lines().enumerate() { + if !line.trim_start().starts_with("//") && line.contains(pat) { + env_reads.push(format!("{name}:{}: {}", n + 1, line.trim())); + } + } } } + // Browser discovery reads PATH, and that is the only read of the environment. + assert_eq!(env_reads.len(), 1, "{env_reads:?}"); + assert!( + env_reads[0].starts_with("browser.rs:") && env_reads[0].contains("var_os(\"PATH\")"), + "{env_reads:?}" + ); } /// A `kharon-teach-*` profile directory in the temp directory whose owner file holds `pid`. @@ -398,3 +410,175 @@ async fn a_browser_that_quits_by_itself_ends_the_session_and_the_profile_stays_g outcome_is_unconfirmed_or_never_started(run.wait().await).await; assert!(gone(&profile).await, "the profile is still there"); } + +/// The shipped binary, past `Ready`, in a browser without a display. +struct Live { + child: std::process::Child, + /// Held open and never written to: end of input must not end the recording. + _stdin: std::process::ChildStdin, + stdout: std::sync::Arc>, + dir: PathBuf, + out: PathBuf, + profile: PathBuf, + _fixture: support::Fixture, + _permit: tokio::sync::SemaphorePermit<'static>, +} + +impl Live { + /// `None` when there is no browser and none is required. + async fn start(test: &str) -> Option { + use std::io::Read; + use std::os::unix::fs::PermissionsExt; + let chromium = support::chromium_or_skip(test)?; + let permit = support::browser_permit().await; + let fixture = support::Fixture::start().await; + let dir = std::env::temp_dir().join(format!( + "kharon-live-{}-{:08x}", + std::process::id(), + rand::random::() + )); + std::fs::create_dir(&dir).unwrap(); + // Only this test wrapper adds the switch; the program believes it launched a visible browser. + let wrapper = dir.join("browser"); + std::fs::write( + &wrapper, + format!( + "#!/bin/sh\nexec {} --headless=new \"$@\"\n", + chromium.display() + ), + ) + .unwrap(); + std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o755)).unwrap(); + let out = dir.join("login.kharon-recording.json"); + let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_kharon-teach")) + .args(["--yes", fixture.url("/form").as_str(), "--browser"]) + .arg(&wrapper) + .arg("--out") + .arg(&out) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap(); + let stdin = child.stdin.take().unwrap(); + let stdout = std::sync::Arc::new(std::sync::Mutex::new(String::new())); + let mut pipe = child.stdout.take().unwrap(); + let sink = stdout.clone(); + std::thread::spawn(move || { + let mut buf = [0u8; 1024]; + while let Ok(n) = pipe.read(&mut buf) { + if n == 0 { + break; + } + sink.lock() + .unwrap() + .push_str(&String::from_utf8_lossy(&buf[..n])); + } + }); + let live = Live { + profile: PathBuf::new(), + child, + _stdin: stdin, + stdout, + dir, + out, + _fixture: fixture, + _permit: permit, + }; + let mut live = live; + let limit = std::time::Instant::now() + Duration::from_secs(40); + while !live + .printed() + .contains("Press Enter once you are logged in") + { + assert!( + std::time::Instant::now() < limit, + "the end prompt never showed" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + live.profile = profile_of(live.child.id()).expect("the profile exists"); + Some(live) + } + + fn printed(&self) -> String { + self.stdout.lock().unwrap().clone() + } + + /// The exit code, or `None` when the program is still running after `secs`. + async fn exit_within(&mut self, secs: u64) -> Option> { + let limit = std::time::Instant::now() + Duration::from_secs(secs); + loop { + if let Some(s) = self.child.try_wait().unwrap() { + return Some(s.code()); + } + if std::time::Instant::now() >= limit { + return None; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } +} + +impl Drop for Live { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +#[tokio::test] +async fn sigint_after_the_prompt_exits_1_without_enter() { + let Some(mut live) = Live::start("sigint_after_prompt").await else { + return; + }; + let pid = live.child.id().to_string(); + assert!( + std::process::Command::new("kill") + .args(["-INT", &pid]) + .status() + .unwrap() + .success() + ); + assert_eq!( + live.exit_within(10).await, + Some(Some(1)), + "the program did not exit on SIGINT without Enter" + ); + assert!(!live.out.exists(), "a file was written"); + assert!(gone(&live.profile).await, "the profile is still there"); +} + +#[tokio::test] +async fn closing_the_window_after_the_prompt_saves_and_exits_without_enter() { + use std::os::unix::fs::PermissionsExt; + let Some(mut live) = Live::start("close_after_prompt").await else { + return; + }; + let port: u16 = std::fs::read_to_string(live.profile.join("DevToolsActivePort")) + .unwrap() + .lines() + .next() + .unwrap() + .parse() + .unwrap(); + let person = support::Person::connect(port).await; + person.click("a").await; + person.wait_for_path("/form").await; + person.type_into("input[name=user]", USER).await; + person.type_into("input[name=pass]", PASS).await; + person.close_all_pages().await; + assert_eq!( + live.exit_within(10).await, + Some(Some(0)), + "the program did not exit after the window closed" + ); + assert!(live.printed().contains("Saved:"), "{}", live.printed()); + let mode = std::fs::metadata(&live.out).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + let rec = kharon_recording::Recording::from_json(&std::fs::read_to_string(&live.out).unwrap()) + .unwrap(); + assert_eq!(rec.ended_by, EndedBy::WindowClosed); + assert!(!rec.success_confirmed); +} diff --git a/crates/kharon-teach/tests/support/mod.rs b/crates/kharon-teach/tests/support/mod.rs index b9bc986..4601d29 100644 --- a/crates/kharon-teach/tests/support/mod.rs +++ b/crates/kharon-teach/tests/support/mod.rs @@ -939,3 +939,39 @@ impl Person { None } } + +// ---- shared by tests that drive or scan the shipped program ---- + +/// Waits for one of the browser slots (see `GATE`); hold the permit while a browser runs. +pub async fn browser_permit() -> tokio::sync::SemaphorePermit<'static> { + GATE.acquire().await.unwrap() +} + +/// Every `.rs` file under `dir`. +pub fn rust_files(dir: &std::path::Path) -> Vec { + let mut out = Vec::new(); + for e in std::fs::read_dir(dir).unwrap().flatten() { + let p = e.path(); + if p.is_dir() { + out.extend(rust_files(&p)); + } else if p.extension().is_some_and(|x| x == "rs") { + out.push(p); + } + } + out +} + +/// Shipped code only: for each file under `crates/kharon-teach/src`, the text before its first +/// `#[cfg(test)]`. +pub fn shipped_sources() -> Vec<(PathBuf, String)> { + let files = rust_files(&std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src")); + assert!(!files.is_empty()); + files + .into_iter() + .map(|p| { + let text = std::fs::read_to_string(&p).unwrap(); + let shipped = text.split("#[cfg(test)]").next().unwrap().to_owned(); + (p, shipped) + }) + .collect() +} From 671a0ffd49f3f8529b3043b9c2426d20ef386198 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 06:41:03 -0500 Subject: [PATCH 25/36] docs(spec): REQ-TCH rows and the teach-mode user page (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- README.md | 1 + SPEC.md | 13 ++++ docs/teach-mode.md | 159 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 docs/teach-mode.md diff --git a/README.md b/README.md index fc7cfd4..72a54fd 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Vaultwarden fill broker: lets AI agents fill credentials without ever seeing the - [`docs/broker-lifecycle.md`](docs/broker-lifecycle.md): host install of kharond, backup and restore of `/data`, upgrades, and the secrets inventory. - [`docs/vault-onboarding.md`](docs/vault-onboarding.md): the customer's Bitwarden organization, collection, dedicated account, and API key. - [`docs/agent-runtime-setup.md`](docs/agent-runtime-setup.md): the agent side on a hosted or root runtime, and handing over credentials. +- [`docs/teach-mode.md`](docs/teach-mode.md): `kharon-teach`, a program that records how a login works while a person logs in by hand with a throwaway account, and writes a file with no typed value in it (#361). - [`docs/hermes-end-to-end.md`](docs/hermes-end-to-end.md): Hermes after the connector is installed: the MCP client, what the agent and the owner do in a fill, and what each failure looks like. A record of one live run. - [`docs/acceptance-checks.md`](docs/acceptance-checks.md): checks a new managed deploy must pass before it carries anyone but a pilot (edge, one real fill, REQ-GOAL-002 canaries). - [`deploy/compose/README.md`](deploy/compose/README.md): running kharond with Docker Compose. diff --git a/SPEC.md b/SPEC.md index c89e288..ebbac59 100644 --- a/SPEC.md +++ b/SPEC.md @@ -209,6 +209,19 @@ The agent keeps its own CDP session (Playwright/puppeteer `connectOverCDP`). kha | REQ-ARCH-005 | Target ownership. The broker creates one browser context per agent (`Target.createBrowserContext`) and hands the agent its `browserContextId` via `kharon_context` / `POST /v1/context` (#65): the call connects if needed (the broker stays lazy, no connect at startup), creates or returns the agent's cached context, and replies `browser_context_id`. It is idempotent while the CDP connection lives; a reconnect or a lock invalidates the id (the next call returns a new one, and fills into tabs of the old context get `target_not_owned`). It is lock-gated (`423` while locked), runs under the `fill_deadline`, and is audited like a fill (`context_open` before any CDP write, then exactly one `context` outcome; REQ-AUD-003†). `kharon_status` reports the cached id or `null` and never connects. `fill` is accepted only if `cdp_target_id` is in the requesting agent's context. Targets in the default context are refused unless `allow_default_context=true`, for single-agent setups. | E2E: agent A fills into a tab in agent B's context → `target_not_owned`. Default-context tab with the flag off → refused. Daemon + CDP stub: no connection before the first `kharon_context`; two calls → same id, one `createBrowserContext`; after a reconnect or lock+unlock → new id; `423` while locked; `status.browser_context_id` is `null` before the first call and after invalidation. | | REQ-ARCH-004 | Chromium launched with password manager and autofill save disabled. | Profile `Login Data` contains 0 rows after 10 fills. | +### 3.4 Teach mode: recording a demonstrated login + +`kharon-teach` is a separate program. A person runs it on their own computer, with a throwaway account, to show kharon how one site's login works. It opens its own visible browser, reads what the page does while the person logs in by hand, and writes one file that describes the structure of the login. The file holds no typed value. It shares nothing with a fill: it has no broker credential, no vault access and no connection to a broker. Its CDP client subscribes to `Runtime`, `Log`, `Network` and `Page` events. The broker's never does (REQ-ARCH-003), so the recorder is its own client and does not use the broker's. Turning a recording into a login method, and replaying one, is later work (#361). The design is in [`docs/superpowers/specs/2026-11-18-teach-recorder-design.md`](docs/superpowers/specs/2026-11-18-teach-recorder-design.md). The user page is [`docs/teach-mode.md`](docs/teach-mode.md). + +| ID | Requirement | AC | +|---|---|---| +| REQ-TCH-001 | The recorder launches the browser itself, in its own process group, on a profile directory it created (mode `0700`, with an owner file). It has no option to attach to another browser or to name a profile. The browser's debugging port is loopback only. | `crates/kharon-teach/tests/req_tch_001_own_browser.rs`: a launched browser runs with `--user-data-dir` set to the profile the recorder made, and the debugging port listens only on loopback. A scan of the shipped source finds no `ws://` or `http://` address that is not the loopback CDP one, and the fields of `Launch` are `browser`, `first_url`, `headless` and `profile`. | +| REQ-TCH-002 | No typed value is written to the recording file, to any other file of the recorder's own, or to its output. This covers every value typed in the window, not only the user name and the password. A name or path that contains a typed value is replaced (`{typed}` for a key, host label or attribute, `{id}` for a path segment). | `crates/kharon-teach/tests/req_tch_002_no_typed_value.rs`: for each of the four fixture logins, a byte search of the recording for the user name and the password, in every encoding the recorder knows, finds nothing. The same search over the whole program's file, standard output and standard error finds nothing. A page that copies a typed value into an attribute, and the listener script's own reports, carry none of it. | +| REQ-TCH-003 | The file holds no cookie, no header value other than the content type and the `Authorization` scheme, no response body, no query value and no request-body value. A URL scheme and an `Authorization` scheme are kept only from a known list. The content of a `data:` URL is never kept. | `crates/kharon-teach/tests/req_tch_003_file_contents.rs`: the file is mode `0600`, reads back as a recording, keeps a query key and not its value, and holds no cookie, no response text, no page title and none of eight named headers; the response's `Set-Cookie` shows only as a flag. The scheme lists and the `data:` rule are checked by unit tests in `crates/kharon-teach/src/strip.rs`. | +| REQ-TCH-004 | The recorder holds no broker credential, listens on no socket, and opens no network connection except loopback CDP to the browser it launched. | `crates/kharon-teach/tests/req_tch_004_no_network.rs`: the shipped source has no listener or bind call. The dependency tree has no TLS or HTTP client crate, no `kharond` and no `kharon-connector`. The only dial is in `cdp.rs`, to a `ws://127.0.0.1:` address. | +| REQ-TCH-005 | The four fixture logins classify as `form_post`, `script_post`, `two_step` and `transformed`. | `crates/kharon-teach/tests/req_tch_005_shapes.rs`: a real Chromium logs in to each of the four fixture pages, and each recording carries the shape named for it, with the expected rung ceiling. Further cases cover a multipart body, a blob body, a pop-up and a login in a frame. | +| REQ-TCH-006 | The browser profile is deleted on every exit the recorder can see (confirm, discard, window closed, browser killed, SIGINT, SIGTERM). A leftover profile is deleted at the next start unless its owner is still running. The browser is always visible: no flag and no environment variable changes that. | `crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs`: the profile is gone after each of those exits, and a leftover is removed at the next start. After a login the browser's password and autofill tables are empty. A scan of the shipped source finds no way to ask for headless, and `PATH` is the only environment variable read. The owner-still-running rule is checked by unit tests in `crates/kharon-teach/src/browser.rs`. | + ## 4. Agent-facing API Exposed as MCP (tools below) and HTTP `/v1/*`, with identical semantics. Auth: `Authorization: Bearer kh_agent__`. Only the Argon2id hash of the token is stored. One token per agent identity. The owner mints the token out of band with `kharon token mint ` (§7); the daemon does not. A dashboard instead connects with MCP OAuth and calls `POST /mcp` with a short-lived access token, `kh_at_…`, issued by the deploy's own authorization server (§4.1). diff --git a/docs/teach-mode.md b/docs/teach-mode.md new file mode 100644 index 0000000..2637f03 --- /dev/null +++ b/docs/teach-mode.md @@ -0,0 +1,159 @@ +# Teach mode: recording a login + +## What this is for + +Some sites log you in with one form. Others ask for the user name on one page and the password on the next, or run a script that changes the password before it sends it. kharon needs to know which kind a site is before it can handle that login safely. Teach mode is how it learns. You log in by hand once, with a throwaway account, and a small program called `kharon-teach` writes down how the login worked. + +This is the first piece of teach mode. It records. kharon cannot yet turn a recording into a login method or replay one; that is later work (#361). For now the recording is a file that a person or a model can read. + +## Before you start + +Use a throwaway account. Never use a real one. The program reads everything you type in the window it opens, because that is how it learns which field is which. It does not save what you type, but it does read it, and the browser keeps your session until the window closes. If you type a real password by mistake, change that password. + +Linux only for now. A Chromium-family browser must be installed: Chromium, Google Chrome, Microsoft Edge or Brave. The program looks for them on your `PATH` in that order, or you can name one with `--browser`. + +The program always opens a visible window. There is no way to turn that off. + +## Recording a login + +Give the program the address of the login page: + +```text +kharon-teach https://login.example.com/ +``` + +The address must start with `https://`. Plain `http://` is accepted only for `localhost` and other addresses on your own computer. An address that holds a user name or password is refused. + +The terminal prints this notice and asks you to agree: + +```text +kharon-teach records how a login works so kharon can learn it. +Use a THROWAWAY account. Everything you type in the window that +opens is read by this program. Nothing you type is saved, and +nothing is sent anywhere. The result is one file on this computer. +Type yes to open the browser window and start: +``` + +Type `yes` and press Enter. Anything else ends the program with "Nothing was recorded." and opens nothing. + +A browser window opens on a new, empty profile. Its first page repeats the notice and has one link, "Start recording and open" followed by the site's name. Recording starts when you click it. Nothing before that click is recorded. + +The terminal then prints: + +```text +Press Enter once you are logged in. Type q to discard. +``` + +Log in by hand in the window. When you are signed in, go back to the terminal. A recording can end in three ways: + +- Press Enter. The recording is saved and marked as a login that worked. +- Type `q` and press Enter. The recording is thrown away and nothing is saved. +- Close the browser window. The recording is saved and marked as a login that was not confirmed. If you close the window before you click the link, nothing is saved. + +Ctrl-C, or a stop signal, ends the program with "Interrupted. Nothing was saved." and a failure exit code. + +On every one of these, the program closes the browser and deletes the profile it made. If an earlier run was killed and left a profile behind, the next run deletes it and says "Removed 1 leftover browser profile(s) from an earlier run." before the Enter prompt. + +When a recording is saved, the program prints a summary of it and then "Saved:" and the path. The file is named after the site and the time, for example `login.example.com-20261118T061500Z.kharon-recording.json`, and is written in the current directory. Only you can read it (mode `0600`). If a file of that name already exists, the program adds `.1`, then `.2`, up to `.9`, and does not overwrite anything. + +## Options + +```text +kharon-teach [--out ] [--browser ] [--yes] +kharon-teach show +``` + +| Option | What it does | +|---|---| +| `--out ` | Where to write the recording. The file must not exist yet, and its folder must exist and be one you can write in. Without it, the file goes in the current directory under the generated name. | +| `--browser ` | The browser program to use, instead of searching `PATH`. It must be a Chromium-family browser. | +| `--yes` | Skips the question "Type yes to open the browser window and start". The notice is still printed. | +| `show ` | Prints what a recording file says, in plain words. It opens no browser and changes nothing. A file larger than 16 MiB, or one that is not a recording this version can read, is refused. | + +There is no option to use a browser that is already running, or to pick the profile folder. The program starts its own browser every time. + +## What is in the file and what is not + +The file describes the structure of the login. It is one JSON document, and it holds nothing you typed. Before the program writes it, everything you typed is reduced to a role (user name or password) and a length. What is kept and what is dropped: + +| What the browser showed | What the file keeps | +|---|---| +| What you typed | The role (user name or password) and the length. Every field you typed in counts, not only those two. | +| Request bodies | The shape: the names of the fields, and for each value its kind, its length and, where a typed value was found, its role and how it was encoded. Never the value. A body the program cannot read, or one over 1 MiB, is kept as a length only. | +| Query strings (`?a=b`) | The names only. | +| Paths (`/a/b`) | The path, with a part replaced by `{id}` when it is all digits, looks like a UUID, is 16 or more hex characters, or contains something you typed. | +| Names that contained something you typed | Replaced by `{typed}`. This covers the names of fields and parts, the labels of a host name and the content type. | +| Web addresses | The scheme is kept only if it is a common one (http, https, ws, wss, data, blob, about, file, chrome, chrome-extension, devtools). Any other is written `{other}`. The content of a `data:` address is never kept. | +| Headers | The content type only. For `Authorization`, only the scheme (for example Basic or Bearer), and only from a known list; any other scheme is written `{other}`. Whether a typed value was in it is recorded. | +| Cookies | Whether a response set one. Nothing else. | +| Response bodies | Never read. | +| Page text, titles and HTML | Never read. Only the attributes of the fields you used are kept: tag, type, `id`, `name`, `autocomplete`, placeholder, aria-label, the form it is in, and a list of ways to find it again. An attribute or way of finding a field that held something you typed is left out. | + +Only three kinds of request are stored: pages the main window moved to, requests in which a typed value was found, and any request other than GET to the login site after your first keystroke. All other requests are counted and dropped. + +The file also records the program version, the browser's name and user agent, the window size, the start and end times, how the recording ended, and the steps you took: pages visited, fields typed in (by length), clicks, form submissions and Enter presses. + +At the end of the file, a `stripped` block counts what was removed: how many requests the browser made and how many were kept, how many typed values, query values and path parts were dropped, and how many bodies were reduced to a shape. The summary prints the same counts. + +## Reading the result + +The program ends with a summary, and `show` prints the same one later. Each line starts with a label. + +Shape of the login ("How the login works"): + +| Shape | Meaning | +|---|---| +| `form_post` | One page. The form sends the password to the site as typed. | +| `script_post` | One page. The page's script sends the password to the site as typed. | +| `two_step` | The user name goes first, then the password on a later step. The summary says how the password was sent: by a form, by a script, or changed by the page first. | +| `transformed` | The page changes the password before sending it, so no request holds it as typed. If the change was a known digest (md5, sha1, sha256 or sha512), the summary names it. | +| `unrecognized` | None of the above. This also covers a recording in which no password field was used, one in which the password was sent in a way the program does not know, one in which request bodies were unavailable and the password was found nowhere, and one with a flag below that needs a person to look. | + +Strongest protection ("Strongest protection"), the best that kharon could do for this login: + +| Value | Meaning | +|---|---| +| `milliseconds` | kharon could fill and submit in one step, so the password is in the page for milliseconds. | +| `agent_continues` | The agent has to continue after the fill. This is the case for two-step logins, logins with a captcha or a one-time-code step, and one-page logins where the recording shows no way to submit. | +| `needs_person` | A person has to look at the recording. | + +"Network swap" says whether the password was found, as typed or encoded, in exactly one stored request. Such a login might let kharon put the password in as the request leaves the browser, so the page never holds it. kharon cannot do this yet. The line records which logins might allow it. + +Flags, each printed as "Flag:" with a sentence: + +| Flag | Meaning | +|---|---| +| `captcha` | A captcha was on the page. The program knows five captcha hosts, plus Google's. | +| `otp_step` | There was a field for a one-time code after the password. | +| `cross_site` | The login went through another site, as single sign-on does. | +| `popup` | The login opened another window. | +| `framed_login` | A login field was inside a frame from another web address. | +| `shadow_dom` | A login field was inside a shadow root, a part of the page that hides its fields from ordinary lookups. | +| `multiple_passwords` | The page had two password fields with different values, as a sign-up or change-password form does. | +| `body_unavailable` | The browser would not hand over the body of some request, so the shape may be incomplete. | + +`cross_site`, `popup`, `framed_login` and `multiple_passwords` make the shape `unrecognized`. The summary may also print "Note:" lines that say why in plain words. + +A "Caution" line appears when the page's own scripts could see and fake what the program reports. Trust such a recording less. + +## Sending a recording to someone + +A recording is an ordinary file. The program never sends it anywhere. If you want someone to read it, you copy it to them yourself. + +Before you do, run `kharon-teach show ` and read what it says. The `stripped` block at the end of the file lets the reader check what was removed: the number of typed values, query values and path parts dropped, and how many requests were kept out of how many seen. The file still names the site you recorded and the structure of its login, so send it only to someone who may know which site that is. + +## What it cannot do + +- It records in a visible desktop browser. The agent's browser is usually headless, and a site may lay itself out differently there. The file records the window size and the user agent so a mismatch can be seen later. +- A site with bot detection may treat a fresh profile with a debugging port differently from your everyday browser. +- If you type a real password despite the notice, the program still writes it nowhere, but it read it, and the browser held it until the profile was deleted. +- If the program is killed or the computer loses power, the profile stays on disk until the next start deletes it. +- It does not detect passkey logins. +- It does not follow web workers or service workers. +- It finds a typed value in a request only in forms it knows: as typed, percent-encoded, escaped for JSON, base64, base64url and hex, and for the password the md5, sha1, sha256 and sha512 digests in hex or base64. A site that lowercases or re-encodes a typed value before sending it may not be recognised as the user name. +- A typed value that a site sends as a JSON number is looked for as its text. If the site writes the number differently from how it was typed, it is not found. +- A typed value shorter than six characters is searched for, but a match is marked low confidence, because it may be chance. +- Once 32 MiB of request bodies are held, later bodies are recorded as unavailable. +- A `transformed` login with no named digest is mostly a site that encrypts the password in the page with a key the site supplies, as banks often do. It is an expected result and is marked `needs_person`. +- A recording made with the listener in the page's own script world, not the isolated one, is marked as such and is less trustworthy. +- Turning a recording into a login method is not built yet. From 56d2cc2343c7a78b662971d175b341d1788aea44 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 06:44:24 -0500 Subject: [PATCH 26/36] docs(spec): make the REQ-TCH rows and the user page say what is built (#361) Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- SPEC.md | 8 ++++---- .../kharon-teach/tests/req_tch_005_shapes.rs | 7 +++++++ docs/teach-mode.md | 19 +++++++++++-------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/SPEC.md b/SPEC.md index ebbac59..d333a53 100644 --- a/SPEC.md +++ b/SPEC.md @@ -211,15 +211,15 @@ The agent keeps its own CDP session (Playwright/puppeteer `connectOverCDP`). kha ### 3.4 Teach mode: recording a demonstrated login -`kharon-teach` is a separate program. A person runs it on their own computer, with a throwaway account, to show kharon how one site's login works. It opens its own visible browser, reads what the page does while the person logs in by hand, and writes one file that describes the structure of the login. The file holds no typed value. It shares nothing with a fill: it has no broker credential, no vault access and no connection to a broker. Its CDP client subscribes to `Runtime`, `Log`, `Network` and `Page` events. The broker's never does (REQ-ARCH-003), so the recorder is its own client and does not use the broker's. Turning a recording into a login method, and replaying one, is later work (#361). The design is in [`docs/superpowers/specs/2026-11-18-teach-recorder-design.md`](docs/superpowers/specs/2026-11-18-teach-recorder-design.md). The user page is [`docs/teach-mode.md`](docs/teach-mode.md). +`kharon-teach` is a separate program. A person runs it on their own computer, with a throwaway account, to show kharon how one site's login works. It opens its own visible browser, reads what the page does while the person logs in by hand, and writes one file that describes the structure of the login. The file holds no typed value. It shares nothing with a fill: it has no broker credential, no vault access and no connection to a broker. It needs to hear what the page does, so its CDP client enables the `Runtime`, `Network` and `Page` domains. The broker's client enables none of them (REQ-ARCH-003). The recorder therefore has its own client and never uses the broker's. Turning a recording into a login method, and replaying one, is later work (#361). The design is in [`docs/superpowers/specs/2026-11-18-teach-recorder-design.md`](docs/superpowers/specs/2026-11-18-teach-recorder-design.md). The user page is [`docs/teach-mode.md`](docs/teach-mode.md). | ID | Requirement | AC | |---|---|---| | REQ-TCH-001 | The recorder launches the browser itself, in its own process group, on a profile directory it created (mode `0700`, with an owner file). It has no option to attach to another browser or to name a profile. The browser's debugging port is loopback only. | `crates/kharon-teach/tests/req_tch_001_own_browser.rs`: a launched browser runs with `--user-data-dir` set to the profile the recorder made, and the debugging port listens only on loopback. A scan of the shipped source finds no `ws://` or `http://` address that is not the loopback CDP one, and the fields of `Launch` are `browser`, `first_url`, `headless` and `profile`. | -| REQ-TCH-002 | No typed value is written to the recording file, to any other file of the recorder's own, or to its output. This covers every value typed in the window, not only the user name and the password. A name or path that contains a typed value is replaced (`{typed}` for a key, host label or attribute, `{id}` for a path segment). | `crates/kharon-teach/tests/req_tch_002_no_typed_value.rs`: for each of the four fixture logins, a byte search of the recording for the user name and the password, in every encoding the recorder knows, finds nothing. The same search over the whole program's file, standard output and standard error finds nothing. A page that copies a typed value into an attribute, and the listener script's own reports, carry none of it. | -| REQ-TCH-003 | The file holds no cookie, no header value other than the content type and the `Authorization` scheme, no response body, no query value and no request-body value. A URL scheme and an `Authorization` scheme are kept only from a known list. The content of a `data:` URL is never kept. | `crates/kharon-teach/tests/req_tch_003_file_contents.rs`: the file is mode `0600`, reads back as a recording, keeps a query key and not its value, and holds no cookie, no response text, no page title and none of eight named headers; the response's `Set-Cookie` shows only as a flag. The scheme lists and the `data:` rule are checked by unit tests in `crates/kharon-teach/src/strip.rs`. | +| REQ-TCH-002 | No typed value is written to the recording file, to any other file of the recorder's own, or to its output. This covers every value typed in the window, not only the user name and the password. A name that contains a typed value is replaced with `{typed}` (a field or part name, a host label, a content type, a scheme). A path segment that contains one is replaced with `{id}`. An element attribute or selector that holds one is left out. | `crates/kharon-teach/tests/req_tch_002_no_typed_value.rs`: for each of the four fixture logins, a byte search of the recording for the user name and the password, in every encoding the recorder knows, finds nothing. The same search over the whole program's file, standard output and standard error finds nothing. A page that copies a typed value into an attribute, and the listener script's own reports, carry none of it. | +| REQ-TCH-003 | The file holds no cookie, no request or response header value other than the content type and the `Authorization` scheme (the browser's own name and user-agent string are recorded, as they are not header values the page sent), no response body, no query value and no request-body value. A URL scheme and an `Authorization` scheme are kept only from a known list. The content of a `data:` URL is never kept. | `crates/kharon-teach/tests/req_tch_003_file_contents.rs`: the file is mode `0600`, reads back as a recording, keeps a query key and not its value, and holds no cookie, no response text, no page title and none of eight named headers; the response's `Set-Cookie` shows only as a flag. The scheme lists and the `data:` rule are checked by unit tests in `crates/kharon-teach/src/strip.rs`. | | REQ-TCH-004 | The recorder holds no broker credential, listens on no socket, and opens no network connection except loopback CDP to the browser it launched. | `crates/kharon-teach/tests/req_tch_004_no_network.rs`: the shipped source has no listener or bind call. The dependency tree has no TLS or HTTP client crate, no `kharond` and no `kharon-connector`. The only dial is in `cdp.rs`, to a `ws://127.0.0.1:` address. | -| REQ-TCH-005 | The four fixture logins classify as `form_post`, `script_post`, `two_step` and `transformed`. | `crates/kharon-teach/tests/req_tch_005_shapes.rs`: a real Chromium logs in to each of the four fixture pages, and each recording carries the shape named for it, with the expected rung ceiling. Further cases cover a multipart body, a blob body, a pop-up and a login in a frame. | +| REQ-TCH-005 | The four fixture logins classify as `form_post`, `script_post`, `two_step` and `transformed`. | `crates/kharon-teach/tests/req_tch_005_shapes.rs`: a real Chromium logs in to each of the four fixture pages, and each recording carries the shape named for it and the rung ceiling `milliseconds` (the form, script and hashed pages) or `agent_continues` (the two-step page). Further cases cover a multipart body, a blob body, a pop-up and a login in a frame. | | REQ-TCH-006 | The browser profile is deleted on every exit the recorder can see (confirm, discard, window closed, browser killed, SIGINT, SIGTERM). A leftover profile is deleted at the next start unless its owner is still running. The browser is always visible: no flag and no environment variable changes that. | `crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs`: the profile is gone after each of those exits, and a leftover is removed at the next start. After a login the browser's password and autofill tables are empty. A scan of the shipped source finds no way to ask for headless, and `PATH` is the only environment variable read. The owner-still-running rule is checked by unit tests in `crates/kharon-teach/src/browser.rs`. | ## 4. Agent-facing API diff --git a/crates/kharon-teach/tests/req_tch_005_shapes.rs b/crates/kharon-teach/tests/req_tch_005_shapes.rs index bba8e64..bb6d310 100644 --- a/crates/kharon-teach/tests/req_tch_005_shapes.rs +++ b/crates/kharon-teach/tests/req_tch_005_shapes.rs @@ -49,6 +49,12 @@ async fn script_page_classifies_as_script_post() { "{:?}", r.classification.notes ); + assert_eq!( + r.classification.rung_ceiling, + RungCeiling::Milliseconds, + "{:?}", + r.classification.notes + ); } #[tokio::test] @@ -82,6 +88,7 @@ async fn hashed_page_classifies_as_transformed() { "{:?}", c.notes ); + assert_eq!(c.rung_ceiling, RungCeiling::Milliseconds, "{:?}", c.notes); assert!(!c.network_swap_candidate); } diff --git a/docs/teach-mode.md b/docs/teach-mode.md index 2637f03..4c3b973 100644 --- a/docs/teach-mode.md +++ b/docs/teach-mode.md @@ -50,11 +50,13 @@ Log in by hand in the window. When you are signed in, go back to the terminal. A - Type `q` and press Enter. The recording is thrown away and nothing is saved. - Close the browser window. The recording is saved and marked as a login that was not confirmed. If you close the window before you click the link, nothing is saved. -Ctrl-C, or a stop signal, ends the program with "Interrupted. Nothing was saved." and a failure exit code. +If you press Enter before you click the link, nothing is saved, and the program prints "The window was closed before the login page opened. Nothing was saved." even though the window is still open. + +Ctrl-C, or a stop signal, ends the program with "Interrupted. Nothing was saved." and a failure exit code. This line and every refusal go to the error output, each starting with "kharon-teach: ". On every one of these, the program closes the browser and deletes the profile it made. If an earlier run was killed and left a profile behind, the next run deletes it and says "Removed 1 leftover browser profile(s) from an earlier run." before the Enter prompt. -When a recording is saved, the program prints a summary of it and then "Saved:" and the path. The file is named after the site and the time, for example `login.example.com-20261118T061500Z.kharon-recording.json`, and is written in the current directory. Only you can read it (mode `0600`). If a file of that name already exists, the program adds `.1`, then `.2`, up to `.9`, and does not overwrite anything. +When a recording is saved, the program prints a summary of it and then "Saved:" and the path. The file is named after the site and the time, for example `login.example.com-20261118T061500Z.kharon-recording.json`, and is written in the current directory. Only you can read it (mode `0600`). If a file of that name already exists, the program adds `.1`, then `.2`, up to `.9`, and does not overwrite anything. For a generated name the number goes before `.kharon-recording.json`. For a name you gave with `--out` it goes at the end. ## Options @@ -66,7 +68,7 @@ kharon-teach show | Option | What it does | |---|---| | `--out ` | Where to write the recording. The file must not exist yet, and its folder must exist and be one you can write in. Without it, the file goes in the current directory under the generated name. | -| `--browser ` | The browser program to use, instead of searching `PATH`. It must be a Chromium-family browser. | +| `--browser ` | The browser program to use, instead of searching `PATH`. The program expects a Chromium-family browser and does not check. | | `--yes` | Skips the question "Type yes to open the browser window and start". The notice is still printed. | | `show ` | Prints what a recording file says, in plain words. It opens no browser and changes nothing. A file larger than 16 MiB, or one that is not a recording this version can read, is refused. | @@ -74,7 +76,7 @@ There is no option to use a browser that is already running, or to pick the prof ## What is in the file and what is not -The file describes the structure of the login. It is one JSON document, and it holds nothing you typed. Before the program writes it, everything you typed is reduced to a role (user name or password) and a length. What is kept and what is dropped: +The file describes the structure of the login. It is one JSON document (a plain-text format that programs can read), and it holds nothing you typed. Before the program writes it, everything you typed is reduced to a role (user name or password) and a length. What is kept and what is dropped: | What the browser showed | What the file keeps | |---|---| @@ -86,12 +88,13 @@ The file describes the structure of the login. It is one JSON document, and it h | Web addresses | The scheme is kept only if it is a common one (http, https, ws, wss, data, blob, about, file, chrome, chrome-extension, devtools). Any other is written `{other}`. The content of a `data:` address is never kept. | | Headers | The content type only. For `Authorization`, only the scheme (for example Basic or Bearer), and only from a known list; any other scheme is written `{other}`. Whether a typed value was in it is recorded. | | Cookies | Whether a response set one. Nothing else. | +| Responses | For each stored request, the status number (for example 200) and the content type of the reply. | | Response bodies | Never read. | | Page text, titles and HTML | Never read. Only the attributes of the fields you used are kept: tag, type, `id`, `name`, `autocomplete`, placeholder, aria-label, the form it is in, and a list of ways to find it again. An attribute or way of finding a field that held something you typed is left out. | Only three kinds of request are stored: pages the main window moved to, requests in which a typed value was found, and any request other than GET to the login site after your first keystroke. All other requests are counted and dropped. -The file also records the program version, the browser's name and user agent, the window size, the start and end times, how the recording ended, and the steps you took: pages visited, fields typed in (by length), clicks, form submissions and Enter presses. +The file also records the program version, the browser's name and user agent (the line a browser sends to say what it is), the window size, the start and end times, how the recording ended, and the steps you took: pages visited, fields typed in (by length), clicks, form submissions and Enter presses. At the end of the file, a `stripped` block counts what was removed: how many requests the browser made and how many were kept, how many typed values, query values and path parts were dropped, and how many bodies were reduced to a shape. The summary prints the same counts. @@ -106,7 +109,7 @@ Shape of the login ("How the login works"): | `form_post` | One page. The form sends the password to the site as typed. | | `script_post` | One page. The page's script sends the password to the site as typed. | | `two_step` | The user name goes first, then the password on a later step. The summary says how the password was sent: by a form, by a script, or changed by the page first. | -| `transformed` | The page changes the password before sending it, so no request holds it as typed. If the change was a known digest (md5, sha1, sha256 or sha512), the summary names it. | +| `transformed` | The page changes the password before sending it, so no request holds it as typed. If the change was a known digest, the summary names it. A digest is a fixed-length scramble of the password that cannot be turned back (md5, sha1, sha256 or sha512). | | `unrecognized` | None of the above. This also covers a recording in which no password field was used, one in which the password was sent in a way the program does not know, one in which request bodies were unavailable and the password was found nowhere, and one with a flag below that needs a person to look. | Strongest protection ("Strongest protection"), the best that kharon could do for this login: @@ -114,10 +117,10 @@ Strongest protection ("Strongest protection"), the best that kharon could do for | Value | Meaning | |---|---| | `milliseconds` | kharon could fill and submit in one step, so the password is in the page for milliseconds. | -| `agent_continues` | The agent has to continue after the fill. This is the case for two-step logins, logins with a captcha or a one-time-code step, and one-page logins where the recording shows no way to submit. | +| `agent_continues` | The agent has to continue after the fill. This is the case for two-step logins, logins with a captcha or a one-time-code step, and one-page logins where the password field was not in a form and no Enter key and no click on a button followed it. | | `needs_person` | A person has to look at the recording. | -"Network swap" says whether the password was found, as typed or encoded, in exactly one stored request. Such a login might let kharon put the password in as the request leaves the browser, so the page never holds it. kharon cannot do this yet. The line records which logins might allow it. +"Network swap" says whether the password was found, as typed or encoded but not hashed, in exactly one stored request. Such a login might let kharon put the password in as the request leaves the browser, so the page never holds it. kharon cannot do this yet. The line records which logins might allow it. Flags, each printed as "Flag:" with a sentence: From 950e091ea5c8b3b2df472a9bd4c12207b049ffff Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 07:04:37 -0500 Subject: [PATCH 27/36] fix(teach): sweep only profiles we made; clean up on hangup and quit (#361) The sweep now removes a directory only when its name is exactly what Profile::create makes, it has an owner file naming a dead pid, and no live browser holds it. SIGHUP and SIGQUIT end a recording like SIGINT and SIGTERM. A Browser forgets its group id once the group is empty. Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-teach/src/browser.rs | 182 ++++++++++++++---- crates/kharon-teach/src/lib.rs | 9 +- .../tests/req_tch_006_profile_and_visible.rs | 111 ++++++++++- 3 files changed, 253 insertions(+), 49 deletions(-) diff --git a/crates/kharon-teach/src/browser.rs b/crates/kharon-teach/src/browser.rs index 08d20ff..653de21 100644 --- a/crates/kharon-teach/src/browser.rs +++ b/crates/kharon-teach/src/browser.rs @@ -17,6 +17,8 @@ pub enum BrowserError { } pub const PROFILE_PREFIX: &str = "kharon-teach-"; +/// The random part of a profile's name: this many ASCII alphanumerics. +const SUFFIX_LEN: usize = 20; /// Written into every profile: the pid of the recorder that owns it. pub const OWNER_FILE: &str = "kharon-teach.owner"; @@ -120,7 +122,7 @@ impl Profile { for _ in 0..16 { let suffix: String = rand::thread_rng() .sample_iter(&Alphanumeric) - .take(20) + .take(SUFFIX_LEN) .map(char::from) .collect(); let staging = temp.join(format!(".{PROFILE_PREFIX}new-{suffix}")); @@ -175,7 +177,11 @@ impl Profile { remove_tree(&self.path) } - /// Removes leftover PROFILE_PREFIX directories in the temp location owned by this user. Returns how many. + /// Removes profiles an earlier run of this program left in the temp location. A directory is + /// removed only when ALL hold: its name is exactly what [`Profile::create`] makes, it is a + /// real directory owned by this user, its owner file names a pid, that pid is gone, and its + /// browser is gone. Anything else, such as a folder someone made by hand, is left alone. + /// Returns how many were removed. pub fn sweep_stale() -> std::io::Result { Self::sweep_stale_in(&std::env::temp_dir()) } @@ -188,22 +194,21 @@ impl Profile { let mut n = 0; for entry in std::fs::read_dir(dir)? { let Ok(entry) = entry else { continue }; - if !entry - .file_name() - .to_string_lossy() - .starts_with(PROFILE_PREFIX) - { + if !is_profile_name(&entry.file_name().to_string_lossy()) { continue; } let path = entry.path(); let Ok(meta) = std::fs::symlink_metadata(&path) else { continue; }; - if !meta.file_type().is_dir() - || meta.uid() != me - || owner_is_alive(&path) - || browser_is_alive(&path) - { + if !meta.file_type().is_dir() || meta.uid() != me { + continue; + } + // No owner file, or one that does not parse, means this program did not make it. + let Some(owner) = owner_pid(&path) else { + continue; + }; + if pid_is_alive(owner) || browser_is_alive(&path) { continue; } if remove_tree(&path).is_ok() { @@ -243,20 +248,29 @@ fn remove_tree(path: &Path) -> std::io::Result<()> { last } -/// The owner file names a live pid. A missing, symlinked, non-regular, -/// over-32-byte or unparseable owner file counts as no live owner. -fn owner_is_alive(dir: &Path) -> bool { +/// `kharon-teach-` and exactly the random suffix `Profile::create` makes. +fn is_profile_name(name: &str) -> bool { + name.strip_prefix(PROFILE_PREFIX) + .is_some_and(|s| s.len() == SUFFIX_LEN && s.bytes().all(|b| b.is_ascii_alphanumeric())) +} + +/// The pid in the owner file. A missing, symlinked, non-regular, over-32-byte or unparseable +/// owner file gives `None`. +fn owner_pid(dir: &Path) -> Option { let file = dir.join(OWNER_FILE); - let Ok(meta) = std::fs::symlink_metadata(&file) else { - return false; - }; + let meta = std::fs::symlink_metadata(&file).ok()?; if !meta.file_type().is_file() || meta.len() > 32 { - return false; + return None; } std::fs::read_to_string(&file) + .ok()? + .trim() + .parse::() .ok() - .and_then(|t| t.trim().parse::().ok()) - .is_some_and(|pid| Path::new(&format!("/proc/{pid}")).exists()) +} + +fn pid_is_alive(pid: u32) -> bool { + Path::new(&format!("/proc/{pid}")).exists() } /// Chromium's `SingletonLock` is a symlink whose target text is @@ -270,7 +284,7 @@ fn browser_is_alive(dir: &Path) -> bool { .rsplit('-') .next() .and_then(|p| p.parse::().ok()) - .is_some_and(|pid| Path::new(&format!("/proc/{pid}")).exists()) + .is_some_and(pid_is_alive) } pub struct Launch<'a> { @@ -391,20 +405,30 @@ impl Browser { } /// True once no process is left in the browser's group. A group that was never made counts - /// as empty. - pub fn group_is_empty(&self) -> bool { + /// as empty. Once the group has been seen empty the id is forgotten, so that no later signal + /// can reach a group id the system has given to something else. + pub fn group_is_empty(&mut self) -> bool { #[cfg(unix)] if let Some(g) = self.group { - return matches!( + let empty = matches!( rustix::process::test_kill_process_group(g), Err(rustix::io::Errno::SRCH) ); + if empty { + self.group = None; + } + return empty; } true } /// Kill the whole group, reap the browser, and wait up to 3 s for the group to be empty. pub async fn close(mut self) -> std::io::Result<()> { + self.shut().await + } + + /// The body of [`Browser::close`], on a borrow so a test can look at the browser afterwards. + async fn shut(&mut self) -> std::io::Result<()> { self.kill_group(); let reaped = self.child.wait().await.map(|_| ()); let deadline = tokio::time::Instant::now() + GROUP_WAIT; @@ -535,15 +559,56 @@ mod tests { assert!(!path.exists()); } + /// A name of the shape `Profile::create` makes. + const A: &str = "kharon-teach-aaaaaaaaaaaaaaaaaaaa"; + const B: &str = "kharon-teach-bbbbbbbbbbbbbbbbbbbb"; + const DEAD: &str = "4294967295"; + #[test] fn sweep_removes_a_leftover_and_counts_it() { let sb = Sandbox::new("stale"); - let d = sb.mkdir("kharon-teach-stale"); + let d = dir_with_owner(&sb, A, DEAD); std::fs::write(d.join("f"), b"x").unwrap(); assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 1); assert!(!d.exists()); } + #[test] + fn sweep_leaves_a_directory_this_program_did_not_make() { + let sb = Sandbox::new("foreign"); + // Owner-less, whatever the name. + let foo = sb.mkdir("kharon-teach-foo"); + std::fs::write(foo.join("f"), b"x").unwrap(); + let versioned = sb.mkdir("kharon-teach-0.1.0-x86_64-linux"); + // The right name shape, but no owner file. + let shaped = sb.mkdir(A); + // The right shape and a junk owner file. + let junk = dir_with_owner(&sb, B, "hello"); + // An owner file with a dead pid but a name this program never makes. + let odd = dir_with_owner(&sb, "kharon-teach-scratch", DEAD); + let short = dir_with_owner(&sb, "kharon-teach-aaaaaaaaaaaaaaaaaaa", DEAD); + let punct = dir_with_owner(&sb, "kharon-teach-aaaaaaaaaaaaaaaaaa_-", DEAD); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); + for d in [foo, versioned, shaped, junk, odd, short, punct] { + assert!(d.exists(), "{} was removed", d.display()); + } + } + + #[test] + fn profile_name_shape() { + assert!(is_profile_name(A)); + assert!(!is_profile_name("kharon-teach-")); + assert!(!is_profile_name("kharon-teach-aaaaaaaaaaaaaaaaaaaaa")); + assert!(!is_profile_name(".kharon-teach-new-aaaaaaaaaaaaaaaaaaaa")); + assert!(!is_profile_name("other-aaaaaaaaaaaaaaaaaaaa")); + // Whatever `create` makes passes. + let sb = Sandbox::new("shape"); + let p = Profile::create_in(sb.path()).unwrap(); + assert!(is_profile_name( + &p.path().file_name().unwrap().to_string_lossy() + )); + } + #[test] fn sweep_leaves_other_directories() { let sb = Sandbox::new("other"); @@ -558,7 +623,7 @@ mod tests { let sb = Sandbox::new("link"); let target = sb.mkdir("other-tgt"); std::fs::write(target.join("f"), b"x").unwrap(); - let link = sb.path().join("kharon-teach-link"); + let link = sb.path().join(A); std::os::unix::fs::symlink(&target, &link).unwrap(); assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); assert!(target.join("f").exists()); @@ -570,7 +635,7 @@ mod tests { #[test] fn sweep_skips_a_profile_whose_browser_is_alive() { let sb = Sandbox::new("live"); - let d = sb.mkdir("kharon-teach-live"); + let d = dir_with_owner(&sb, A, DEAD); std::os::unix::fs::symlink( format!("host-{}", std::process::id()), d.join("SingletonLock"), @@ -596,7 +661,7 @@ mod tests { #[test] fn sweep_skips_a_profile_owned_by_this_process() { let sb = Sandbox::new("mine"); - let d = dir_with_owner(&sb, "kharon-teach-mine", &std::process::id().to_string()); + let d = dir_with_owner(&sb, A, &std::process::id().to_string()); assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); assert!(d.exists()); } @@ -604,7 +669,7 @@ mod tests { #[test] fn sweep_removes_a_profile_owned_by_a_dead_process() { let sb = Sandbox::new("dead"); - let d = dir_with_owner(&sb, "kharon-teach-dead", &dead_pid().to_string()); + let d = dir_with_owner(&sb, A, &dead_pid().to_string()); assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 1); assert!(!d.exists()); } @@ -612,22 +677,23 @@ mod tests { #[test] fn sweep_treats_an_owner_symlink_as_no_owner() { let sb = Sandbox::new("ownlink"); - let d = sb.mkdir("kharon-teach-ownlink"); + let d = sb.mkdir(A); let real = d.join("real"); - std::fs::write(&real, std::process::id().to_string()).unwrap(); + std::fs::write(&real, DEAD).unwrap(); std::os::unix::fs::symlink(&real, d.join(OWNER_FILE)).unwrap(); - assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 1); - assert!(!d.exists()); + // No owner is not a stale owner: it is a directory this program cannot vouch for. + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); + assert!(d.exists()); } #[test] - fn sweep_treats_a_junk_or_oversized_owner_as_no_owner() { + fn sweep_leaves_a_junk_or_oversized_owner_alone() { let sb = Sandbox::new("junk"); - let a = dir_with_owner(&sb, "kharon-teach-junk", "not a pid"); - let big = format!("{}{}", std::process::id(), " ".repeat(40)); - let b = dir_with_owner(&sb, "kharon-teach-big", &big); - assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 2); - assert!(!a.exists() && !b.exists()); + let a = dir_with_owner(&sb, A, "not a pid"); + let big = format!("{DEAD}{}", " ".repeat(40)); + let b = dir_with_owner(&sb, B, &big); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); + assert!(a.exists() && b.exists()); } /// A non-dumpable process's `/proc/self` belongs to root, so a sweep that learned its uid @@ -646,7 +712,7 @@ mod tests { return; } let sb = Sandbox::new("nondump"); - let d = sb.mkdir("kharon-teach-leftover"); + let d = dir_with_owner(&sb, A, DEAD); std::fs::write(d.join("f"), b"x").unwrap(); let out = std::process::Command::new(std::env::current_exe().unwrap()) .args([ @@ -665,6 +731,40 @@ mod tests { assert!(!d.exists()); } + /// A browser stand-in: a `sleep` that leads its own process group. + fn fake_browser() -> Browser { + let mut cmd = tokio::process::Command::new("sleep"); + cmd.arg("30").kill_on_drop(true); + cmd.process_group(0); + let child = cmd.spawn().unwrap(); + let group = child + .id() + .and_then(|id| i32::try_from(id).ok()) + .and_then(rustix::process::Pid::from_raw); + assert!(group.is_some()); + Browser { child, group } + } + + #[tokio::test] + async fn the_group_id_is_forgotten_once_the_group_is_closed_and_empty() { + let mut b = fake_browser(); + assert!(b.group.is_some()); + assert!(!b.group_is_empty(), "a live group is not empty"); + assert!(b.group.is_some(), "a live group's id is kept"); + b.shut().await.unwrap(); + assert!(b.group.is_none(), "the id of an empty group is dropped"); + // Nothing is signalled any more: kill_group has no id to signal. + b.kill_group(); + assert!(b.group_is_empty()); + } + + #[tokio::test] + async fn close_blocking_also_forgets_the_group_id() { + let mut b = fake_browser(); + b.close_blocking(); + assert!(b.group.is_none()); + } + #[test] fn preferences_turn_off_the_password_manager_and_autofill() { let sb = Sandbox::new("prefs"); diff --git a/crates/kharon-teach/src/lib.rs b/crates/kharon-teach/src/lib.rs index 0cabaa6..7e01951 100644 --- a/crates/kharon-teach/src/lib.rs +++ b/crates/kharon-teach/src/lib.rs @@ -262,9 +262,14 @@ pub async fn run_with( Exit::Failed, ); } - let (Ok(mut sigint), Ok(mut sigterm)) = ( + // Ctrl-C, a stop request, the terminal going away, and Ctrl-\: every way the program can be + // told to stop. The browser is in its own process group, so none of them reaches it by + // itself. + let (Ok(mut sigint), Ok(mut sigterm), Ok(mut sighup), Ok(mut sigquit)) = ( signal(SignalKind::interrupt()), signal(SignalKind::terminate()), + signal(SignalKind::hangup()), + signal(SignalKind::quit()), ) else { return fail(&mut stderr, &"cannot watch for Ctrl-C", Exit::Failed); }; @@ -330,6 +335,8 @@ pub async fn run_with( } _ = sigint.recv() => break Ended::Interrupted, _ = sigterm.recv() => break Ended::Interrupted, + _ = sighup.recv() => break Ended::Interrupted, + _ = sigquit.recv() => break Ended::Interrupted, } }; let outcome = match ended { diff --git a/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs index 0cbaade..8ca1035 100644 --- a/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs +++ b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs @@ -227,10 +227,13 @@ async fn dropping_the_future_removes_the_profile() { #[tokio::test] async fn a_stale_profile_is_swept_at_the_next_start() { + // What an earlier, killed run leaves: the name this program makes, and an owner file naming + // a pid that is gone. let stale: PathBuf = - std::env::temp_dir().join(format!("kharon-teach-stale-test-{}", std::process::id())); + std::env::temp_dir().join(format!("kharon-teach-{:020}", std::process::id())); std::fs::create_dir_all(stale.join("Default")).unwrap(); std::fs::write(stale.join("Default").join("Preferences"), "{}").unwrap(); + std::fs::write(stale.join("kharon-teach.owner"), "4294967295").unwrap(); let Some(run) = Run::start("stale_swept", "/form").await else { let _ = std::fs::remove_dir_all(&stale); return; @@ -240,6 +243,42 @@ async fn a_stale_profile_is_swept_at_the_next_start() { let _ = run.finish(End::Discard).await; } +#[tokio::test] +async fn a_directory_the_program_did_not_make_survives_the_sweep() { + // Same prefix, but no owner file, a junk owner file, or a name of another shape. + let id = std::process::id(); + let base = std::env::temp_dir(); + let ownerless = base.join(format!("kharon-teach-foo-{id}")); + let shaped = base.join(format!( + "kharon-teach-{:020}", + u64::from(id) + 1_000_000_000_000 + )); + let junk = base.join(format!( + "kharon-teach-{:020}", + u64::from(id) + 2_000_000_000_000 + )); + for d in [&ownerless, &shaped, &junk] { + std::fs::create_dir_all(d).unwrap(); + std::fs::write(d.join("keep"), "x").unwrap(); + } + std::fs::write(junk.join("kharon-teach.owner"), "not a pid").unwrap(); + let kept = |d: &Path| d.join("keep").exists(); + let ran = match Run::start("sweep_keeps_foreign", "/form").await { + Some(run) => { + let _ = run.finish(End::Discard).await; + true + } + None => false, + }; + let all_kept = [&ownerless, &shaped, &junk].iter().all(|d| kept(d)); + for d in [&ownerless, &shaped, &junk] { + let _ = std::fs::remove_dir_all(d); + } + if ran { + assert!(all_kept, "a directory the program did not make was swept"); + } +} + #[test] fn the_shipped_main_never_asks_for_headless() { let sources = support::shipped_sources(); @@ -312,7 +351,12 @@ fn interrupted_by(signal: &str) { )); std::fs::create_dir(&dir).unwrap(); let fake = dir.join("fake-browser"); - std::fs::write(&fake, "#!/bin/sh\nexec sleep 30\n").unwrap(); + // The shell stays, with the switches on its command line, as a real browser's would be. + std::fs::write( + &fake, + "#!/bin/sh\nexec sh -c 'sleep 30; true' fake-browser \"$@\"\n", + ) + .unwrap(); std::fs::set_permissions(&fake, std::fs::Permissions::from_mode(0o755)).unwrap(); let out = dir.join("login.kharon-recording.json"); let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_kharon-teach")) @@ -358,9 +402,32 @@ fn interrupted_by(signal: &str) { assert!(err.contains("Interrupted. Nothing was saved."), "{err}"); assert!(!profile.exists(), "the profile is still there"); assert!(!out.exists(), "a file was written"); + assert!( + no_process_on(&profile), + "a process on the profile is still running after {signal}" + ); let _ = std::fs::remove_dir_all(&dir); } +/// True when no process has `--user-data-dir=` on its command line. Polls up to 5 s. +fn no_process_on(profile: &Path) -> bool { + let needle = format!("--user-data-dir={}", profile.display()); + let limit = std::time::Instant::now() + Duration::from_secs(5); + loop { + let any = std::fs::read_dir("/proc").unwrap().flatten().any(|e| { + std::fs::read(e.path().join("cmdline")) + .is_ok_and(|c| c.split(|b| *b == 0).any(|arg| arg == needle.as_bytes())) + }); + if !any { + return true; + } + if std::time::Instant::now() >= limit { + return false; + } + std::thread::sleep(Duration::from_millis(50)); + } +} + #[test] fn sigint_exits_1_and_removes_the_profile() { interrupted_by("INT"); @@ -371,6 +438,16 @@ fn sigterm_exits_1_and_removes_the_profile() { interrupted_by("TERM"); } +#[test] +fn sighup_exits_1_and_removes_the_profile() { + interrupted_by("HUP"); +} + +#[test] +fn sigquit_exits_1_and_removes_the_profile() { + interrupted_by("QUIT"); +} + async fn outcome_is_unconfirmed_or_never_started(outcome: Outcome) { match outcome { Outcome::Recorded(r) => { @@ -528,15 +605,16 @@ impl Drop for Live { } } -#[tokio::test] -async fn sigint_after_the_prompt_exits_1_without_enter() { - let Some(mut live) = Live::start("sigint_after_prompt").await else { +/// After the prompt: send `signal`, expect exit 1 without Enter, no file, no profile and no +/// browser process left. +async fn signal_after_the_prompt(signal: &str, test: &str) { + let Some(mut live) = Live::start(test).await else { return; }; let pid = live.child.id().to_string(); assert!( std::process::Command::new("kill") - .args(["-INT", &pid]) + .args([&format!("-{signal}"), &pid]) .status() .unwrap() .success() @@ -544,10 +622,29 @@ async fn sigint_after_the_prompt_exits_1_without_enter() { assert_eq!( live.exit_within(10).await, Some(Some(1)), - "the program did not exit on SIGINT without Enter" + "the program did not exit on SIG{signal} without Enter" ); assert!(!live.out.exists(), "a file was written"); assert!(gone(&live.profile).await, "the profile is still there"); + assert!( + no_process_on(&live.profile), + "a browser process is still running after SIG{signal}" + ); +} + +#[tokio::test] +async fn sigint_after_the_prompt_exits_1_without_enter() { + signal_after_the_prompt("INT", "sigint_after_prompt").await; +} + +#[tokio::test] +async fn sighup_after_the_prompt_exits_1_and_leaves_no_browser() { + signal_after_the_prompt("HUP", "sighup_after_prompt").await; +} + +#[tokio::test] +async fn sigquit_after_the_prompt_exits_1_and_leaves_no_browser() { + signal_after_the_prompt("QUIT", "sigquit_after_prompt").await; } #[tokio::test] From ab5866160413fb75d318ef4676f7d121f50b79ea Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 07:07:47 -0500 Subject: [PATCH 28/36] fix(teach): keep a name only when it looks like one; never lose a hit in a name (#361) A urlencoded key, JSON key, multipart part name or query key is kept only when it looks like a name; otherwise it is {other}. A typed value in a name still gives {typed}, and its hits now go to the field. A urlencoded body with a piece that has no = or mostly odd keys is Opaque, so a jQuery JSON login keeps its password hit. Short markers are checked in their encoded forms too, as whole tokens. One percent-decoder serves marker.rs and strip.rs. Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-teach/src/body.rs | 260 +++++++++++++++++++++---- crates/kharon-teach/src/capture/mod.rs | 32 +++ crates/kharon-teach/src/marker.rs | 84 ++++++-- crates/kharon-teach/src/strip.rs | 162 +++++++++++---- 4 files changed, 455 insertions(+), 83 deletions(-) diff --git a/crates/kharon-teach/src/body.rs b/crates/kharon-teach/src/body.rs index bfc4df1..8b14d28 100644 --- a/crates/kharon-teach/src/body.rs +++ b/crates/kharon-teach/src/body.rs @@ -4,12 +4,15 @@ //! be, while its shape is extracted. Known limit: the parsed `Value` holds typed values in //! ordinary `String`s that serde_json does not zeroize. -use kharon_recording::{BodyShape, FieldShape, JsonField, JsonShape, ValueKind, ValueShape}; +use kharon_recording::{ + BodyShape, FieldShape, JsonField, JsonShape, MarkerHit, ValueKind, ValueShape, +}; use memchr::memmem; use zeroize::Zeroizing; use crate::marker::Marker; -use crate::strip::{TYPED, decode_percent, hits_in, holds_typed}; +use crate::marker::percent_decode as decode_percent; +use crate::strip::{OTHER, TYPED, dedup_hits, hits_in, holds_typed, looks_like_a_name}; /// Bodies larger than this are not parsed: they are `Opaque`. pub const MAX_BODY_BYTES: usize = 1 << 20; @@ -20,7 +23,10 @@ pub const MAX_BODY_BYTES: usize = 1 << 20; /// /// Values are searched after decoding (percent-decoded urlencoded value with `+` as space, a JSON /// string's content, a JSON number's text, a multipart text part's bytes). An `Opaque` body is -/// searched as raw bytes. A name (key, part name) holding a typed value is written as `{typed}`. +/// searched as raw bytes. A name (key, part name) holding a typed value is written as `{typed}` +/// and its hits are added to its field; a name that does not look like a name (see +/// `strip::looks_like_a_name`) is written as `{other}`. A urlencoded body with a piece that has +/// no `=`, or whose names mostly do not look like names, is not urlencoded data and is `Opaque`. /// Never returns `None` or `Unavailable`. pub fn shape_of_body( content_type: Option<&str>, @@ -30,9 +36,8 @@ pub fn shape_of_body( ) -> BodyShape { let parsed = match content_type.map(ContentType::parse) { Some(ct) if body.len() <= MAX_BODY_BYTES => match ct.essence.as_str() { - "application/x-www-form-urlencoded" => Some(BodyShape::Urlencoded { - fields: urlencoded_fields(body, markers, pair), - }), + "application/x-www-form-urlencoded" => urlencoded_fields(body, markers, pair) + .map(|fields| BodyShape::Urlencoded { fields }), "multipart/form-data" => ct .boundary .as_deref() @@ -145,36 +150,88 @@ fn text_value(bytes: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)> value_shape(ValueKind::String, char_len(bytes), bytes, markers, pair) } -/// A name that is kept only when it holds no typed value (any encoding, or the text in any -/// letter case, raw or percent-decoded; see `strip::holds_typed`). -fn name(bytes: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> String { +/// A name as it is written to the file, and the marker hits found in it. A name that holds a +/// typed value (any encoding, or the text in any letter case, raw or percent-decoded; see +/// `strip::holds_typed`) is `{typed}`, and its hits are returned so the caller can attach them +/// to the field. A name that is not a typed value is kept only when it looks like a name, else +/// it is `{other}`. +fn name( + bytes: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> (String, Vec) { if holds_typed(bytes, markers, pair) { - TYPED.to_owned() + return (TYPED.to_owned(), hits_in(bytes, markers, pair)); + } + let text = String::from_utf8_lossy(bytes); + if looks_like_a_name(&text) { + (text.into_owned(), Vec::new()) } else { - String::from_utf8_lossy(bytes).into_owned() + (OTHER.to_owned(), Vec::new()) + } +} + +/// Adds `extra` to `value.markers` without repeating a (role, encoding). +fn add_hits(value: &mut ValueShape, extra: Vec) { + if extra.is_empty() { + return; } + let mut all = std::mem::take(&mut value.markers); + all.extend(extra); + value.markers = dedup_hits(all); } +/// The urlencoded fields, or `None` when the body is not urlencoded data: some piece has no `=`, +/// or more than half of the keys do not look like names. (jQuery sends JSON text under a +/// urlencoded content type; read as fields, the whole JSON would be one key.) fn urlencoded_fields( body: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>, -) -> Vec { - body.split(|b| *b == b'&') - .filter(|piece| !piece.is_empty()) - .map(|piece| { - let (k, v) = match piece.iter().position(|b| *b == b'=') { - Some(i) => (&piece[..i], &piece[i + 1..]), - None => (piece, &piece[..0]), - }; - let key = decode_percent(k, true); - let value = decode_percent(v, true); - FieldShape { - key: name(&key, markers, pair), - value: text_value(&value, markers, pair), - } - }) - .collect() +) -> Option> { + let mut pieces = Vec::new(); + for piece in body.split(|b| *b == b'&').filter(|p| !p.is_empty()) { + let i = piece.iter().position(|b| *b == b'=')?; + pieces.push((decode_percent(&piece[..i], true), &piece[i + 1..])); + } + let odd_keys = pieces + .iter() + .filter(|(k, _)| !looks_like_a_name(&String::from_utf8_lossy(k))) + .count(); + if odd_keys * 2 > pieces.len() { + return None; + } + Some( + pieces + .into_iter() + .map(|(key, v)| { + let value = decode_percent(v, true); + let (key, key_hits) = name(&key, markers, pair); + let mut value = text_value(&value, markers, pair); + add_hits(&mut value, key_hits); + FieldShape { key, value } + }) + .collect(), + ) +} + +/// Adds `extra` to the first scalar value under `shape`, depth first. The format gives a JSON +/// object key no marker list of its own, so the hits of a typed value found in a key go to the +/// value under it, or to the first value inside it when that is an object or an array. An empty +/// object or array has nowhere to hold them; those hits are not recorded. +fn add_hits_to_first_value(shape: &mut JsonShape, extra: Vec) -> bool { + match shape { + JsonShape::Value { value } => { + add_hits(value, extra); + true + } + JsonShape::Array { items } => items + .iter_mut() + .any(|i| add_hits_to_first_value(i, extra.clone())), + JsonShape::Object { fields } => fields + .iter_mut() + .any(|f| add_hits_to_first_value(&mut f.value, extra.clone())), + } } fn json_shape( @@ -188,9 +245,11 @@ fn json_shape( Value::Object(map) => JsonShape::Object { fields: map .iter() - .map(|(k, v)| JsonField { - key: name(k.as_bytes(), markers, pair), - value: json_shape(v, markers, pair), + .map(|(k, v)| { + let (key, key_hits) = name(k.as_bytes(), markers, pair); + let mut value = json_shape(v, markers, pair); + add_hits_to_first_value(&mut value, key_hits); + JsonField { key, value } }) .collect(), }, @@ -310,10 +369,10 @@ fn part_shape( } else { text_value(content, markers, pair) }; - FieldShape { - key: name(&part_name, markers, pair), - value, - } + let (key, key_hits) = name(&part_name, markers, pair); + let mut value = value; + add_hits(&mut value, key_hits); + FieldShape { key, value } } #[cfg(test)] @@ -710,16 +769,29 @@ mod tests { #[test] fn a_site_cased_key_and_name_is_not_kept() { let m = [user("Bob.Smith@Ex.Test")]; + // Two keys hold the value; two are ordinary, so the body is still urlencoded data. let s = shape_of_body( Some(URLENC), - b"bob.smith%40ex.test=1&BOB.SMITH@EX.TEST=2", + b"a=1&b=2&bob.smith%40ex.test=1&BOB.SMITH@EX.TEST=2", &m, None, ); let BodyShape::Urlencoded { fields } = s else { panic!() }; - assert!(fields.iter().all(|f| f.key == "{typed}")); + assert_eq!( + fields.iter().map(|f| f.key.as_str()).collect::>(), + ["a", "b", "{typed}", "{typed}"] + ); + // Decision change: a body whose keys all fail to look like names is Opaque. (The match + // is case-sensitive, so a site-cased value gives no hit here: a known limit.) + let s = shape_of_body( + Some(URLENC), + b"bob.smith%40ex.test=1&BOB.SMITH@EX.TEST=2", + &m, + None, + ); + assert!(matches!(s, BodyShape::Opaque { .. }), "not opaque"); let s = shape_of_body( Some("application/json"), br#"{"bob.smith@ex.test":1}"#, @@ -883,4 +955,120 @@ mod tests { assert_holds_no_typed_value(&text, &values); } } + + const TOKEN: &str = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk"; + + fn json_hits(s: &BodyShape) -> Vec { + fn walk(j: &JsonShape, out: &mut Vec) { + match j { + JsonShape::Object { fields } => fields.iter().for_each(|f| walk(&f.value, out)), + JsonShape::Array { items } => items.iter().for_each(|i| walk(i, out)), + JsonShape::Value { value } => out.extend(value.markers.clone()), + } + } + let mut out = Vec::new(); + if let BodyShape::Json { root } = s { + walk(root, &mut out); + } + out + } + + fn serialized(s: &BodyShape) -> String { + serde_json::to_string(s).unwrap() + } + + #[test] + fn a_urlencoded_body_that_is_one_token_is_opaque_and_not_kept() { + let s = shape_of_body(Some(URLENC), TOKEN.as_bytes(), &[], None); + assert!(matches!(s, BodyShape::Opaque { .. }), "{s:?}"); + assert!(!serialized(&s).contains("eyJhbGci")); + // One piece without `=` among ordinary ones does the same. + let body = format!("a=1&{TOKEN}&b=2"); + let s = shape_of_body(Some(URLENC), body.as_bytes(), &[], None); + assert!(matches!(s, BodyShape::Opaque { .. })); + assert!(!serialized(&s).contains("eyJhbGci")); + } + + #[test] + fn a_json_object_keyed_by_a_token_does_not_keep_it() { + let body = format!(r#"{{"{TOKEN}":1,"ok":2}}"#); + let s = shape_of_body(Some("application/json"), body.as_bytes(), &[], None); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = &s + else { + panic!("{s:?}") + }; + let mut keys: Vec<&str> = fields.iter().map(|f| f.key.as_str()).collect(); + keys.sort_unstable(); + assert_eq!(keys, ["ok", "{other}"]); + assert!(!serialized(&s).contains("eyJhbGci")); + } + + #[test] + fn a_multipart_part_named_like_a_token_does_not_keep_it() { + let body = + format!("--b\r\nContent-Disposition: form-data; name=\"{TOKEN}\"\r\n\r\nx\r\n--b--"); + let s = shape_of_body( + Some("multipart/form-data; boundary=b"), + body.as_bytes(), + &[], + None, + ); + let BodyShape::Multipart { parts } = &s else { + panic!("{s:?}") + }; + assert_eq!(parts[0].key, "{other}"); + assert!(!serialized(&s).contains("eyJhbGci")); + } + + #[test] + fn a_typed_value_found_in_a_name_keeps_its_hit() { + let m = [pw("Tr0ub4dor9x")]; + let plain = Some(hit(Role::Password, Encoding::AsTyped)); + // Urlencoded: the key is the password. + let s = shape_of_body(Some(URLENC), b"a=1&Tr0ub4dor9x=1", &m, None); + let BodyShape::Urlencoded { fields } = s else { + panic!() + }; + assert_eq!(fields[1].key, "{typed}"); + assert_eq!( + fields[1].value.markers, + plain.clone().into_iter().collect::>() + ); + assert!(fields[0].value.markers.is_empty()); + // JSON: the hit lands on the value under the key, or the first value inside it. + for body in [ + br#"{"Tr0ub4dor9x":"v"}"#.as_slice(), + br#"{"Tr0ub4dor9x":{"x":[1,2]}}"#.as_slice(), + ] { + let s = shape_of_body(Some("application/json"), body, &m, None); + assert_eq!(json_hits(&s), plain.clone().into_iter().collect::>()); + } + // Multipart. + let body = b"--b\r\nContent-Disposition: form-data; name=\"Tr0ub4dor9x\"\r\n\r\nx\r\n--b--"; + let s = shape_of_body(Some("multipart/form-data; boundary=b"), body, &m, None); + let BodyShape::Multipart { parts } = s else { + panic!() + }; + assert_eq!(parts[0].key, "{typed}"); + assert_eq!( + parts[0].value.markers, + plain.into_iter().collect::>() + ); + } + + #[test] + fn a_json_login_under_a_urlencoded_header_keeps_its_password_hit() { + // jQuery: $.post(url, JSON.stringify({user, pass})) with the default content type. + let m = markers(); + let body = br#"{"user":"kh.throwaway7","pass":"Tr0ub4dor-9x"}"#; + let s = shape_of_body(Some(URLENC), body, &m, None); + let BodyShape::Opaque { len, markers } = s else { + panic!("not opaque") + }; + assert_eq!(usize::try_from(len).unwrap(), body.len()); + assert!(markers.contains(&hit(Role::Password, Encoding::AsTyped))); + assert!(markers.contains(&hit(Role::Username, Encoding::AsTyped))); + } } diff --git a/crates/kharon-teach/src/capture/mod.rs b/crates/kharon-teach/src/capture/mod.rs index aad2514..0914d2a 100644 --- a/crates/kharon-teach/src/capture/mod.rs +++ b/crates/kharon-teach/src/capture/mod.rs @@ -1206,6 +1206,38 @@ mod tests { assert_eq!(rec.classification.shape, Shape::ScriptPost); } + #[test] + fn a_json_login_under_a_urlencoded_header_is_a_script_post() { + // jQuery: $.post(url, JSON.stringify({user, pass})) sends JSON text under the default + // urlencoded content type. It must not read as a form whose one key is the JSON. + let rec = Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .click(&button()) + .request( + "POST", + "https://ex.test/api/login", + ResourceKind::Xhr, + false, + Some(FORM), + RawBody::Bytes(Zeroizing::new( + format!(r#"{{"user":"{USER}","pass":"{PASS}"}}"#).into_bytes(), + )), + ) + .nav("https://ex.test/home") + .finish(); + assert_eq!(rec.classification.shape, Shape::ScriptPost); + assert_eq!(rec.requests.len(), 1); + let login = &rec.requests[0]; + assert!(matches!(&login.body, BodyShape::Opaque { .. })); + assert!( + login + .hits() + .iter() + .any(|h| h.role == Role::Password && !h.low_confidence) + ); + } + #[test] fn only_the_three_kinds_of_request_are_stored() { let rec = scenario("which_requests"); diff --git a/crates/kharon-teach/src/marker.rs b/crates/kharon-teach/src/marker.rs index 26971b5..54501f3 100644 --- a/crates/kharon-teach/src/marker.rs +++ b/crates/kharon-teach/src/marker.rs @@ -118,7 +118,9 @@ fn hex_value(b: u8) -> Option { } /// Decodes `%XX` (either hex case); a `%` not followed by two hex digits is copied unchanged. -fn percent_decode(raw: &[u8], plus_is_space: bool) -> Needle { +/// With `plus_is_space`, `+` becomes a space first. The one percent-decoder of this crate; the +/// copy is zeroized. +pub(crate) fn percent_decode(raw: &[u8], plus_is_space: bool) -> Needle { let mut out = Zeroizing::new(Vec::with_capacity(raw.len())); let mut i = 0; while i < raw.len() { @@ -184,18 +186,8 @@ pub fn search_pair(haystack: &[u8], username: &Marker, password: &Marker) -> Vec if username.is_empty() || password.is_empty() { return Vec::new(); } - let mut joined = Zeroizing::new(String::with_capacity( - username.value.len() + password.value.len() + 1, - )); - joined.push_str(&username.value); - joined.push(':'); - joined.push_str(&password.value); let low_confidence = username.len().saturating_add(password.len()) < LOW_CONFIDENCE_BELOW; - let (standard, url_safe) = base64_forms(joined.as_bytes()); - let url_safe = { - let seen: Vec<&[u8]> = standard.iter().map(|n| n.as_slice()).collect(); - dedup(url_safe, &seen) - }; + let (standard, url_safe) = pair_forms(username, password); let haystacks = Haystacks::new(haystack); let mut hits = Vec::new(); for (encoding, forms) in [ @@ -215,6 +207,74 @@ pub fn search_pair(haystack: &[u8], username: &Marker, password: &Marker) -> Vec hits } +/// Whether any of `forms` occurs in the raw bytes or a decoded copy as a whole token: the byte +/// before it and the byte after it are not ASCII alphanumeric, or the text ends there. +fn contains_whole_token(haystacks: &Haystacks, forms: &[Needle]) -> bool { + let bounded = |hay: &[u8], needle: &[u8]| { + let mut from = 0; + while let Some(i) = memmem::find(&hay[from..], needle) { + let (start, end) = (from + i, from + i + needle.len()); + let before = start.checked_sub(1).map(|j| hay[j]); + let after = hay.get(end).copied(); + if !before.is_some_and(|b| b.is_ascii_alphanumeric()) + && !after.is_some_and(|b| b.is_ascii_alphanumeric()) + { + return true; + } + from = start + 1; + } + false + }; + forms + .iter() + .filter(|n| !n.is_empty()) + .any(|n| bounded(haystacks.raw, n) || haystacks.decoded.iter().any(|d| bounded(d, n))) +} + +/// Whether `marker` occurs in `haystack` in ANY of its forms (as typed, escaped, base64, hex, and +/// for a password the digests) as a whole token. For kept text, where a short marker's forms +/// would otherwise match inside ordinary words. +pub(crate) fn found_as_whole_token(haystack: &[u8], marker: &Marker) -> bool { + if marker.is_empty() { + return false; + } + let haystacks = Haystacks::new(haystack); + needles(marker) + .iter() + .any(|(_, forms)| contains_whole_token(&haystacks, forms)) +} + +/// [`found_as_whole_token`] for the base64 forms of `username:password`. +pub(crate) fn pair_found_as_whole_token( + haystack: &[u8], + username: &Marker, + password: &Marker, +) -> bool { + if username.is_empty() || password.is_empty() { + return false; + } + let (standard, url_safe) = pair_forms(username, password); + let haystacks = Haystacks::new(haystack); + contains_whole_token(&haystacks, &standard) || contains_whole_token(&haystacks, &url_safe) +} + +/// The base64 forms of `username:password`: (standard, URL-safe), each padded then unpadded, +/// the URL-safe ones without any that equal a standard one. +fn pair_forms(username: &Marker, password: &Marker) -> (Vec, Vec) { + let mut joined = Zeroizing::new(String::with_capacity( + username.value.len() + password.value.len() + 1, + )); + joined.push_str(&username.value); + joined.push(':'); + joined.push_str(&password.value); + let (standard, url_safe) = base64_forms(joined.as_bytes()); + let url_safe = { + let seen: Vec<&[u8]> = standard.iter().map(|n| n.as_slice()).collect(); + dedup(url_safe, &seen) + }; + (standard, url_safe) +} + fn contains(haystack: &[u8], needle: &[u8]) -> bool { !needle.is_empty() && memmem::find(haystack, needle).is_some() } diff --git a/crates/kharon-teach/src/strip.rs b/crates/kharon-teach/src/strip.rs index 1dffec9..2263091 100644 --- a/crates/kharon-teach/src/strip.rs +++ b/crates/kharon-teach/src/strip.rs @@ -8,17 +8,36 @@ use std::ops::Range; use base64ct::{Base64, Base64Unpadded, Base64Url, Base64UrlUnpadded, Encoding as B64}; use kharon_recording::{AuthShape, Encoding, MarkerHit, StrippedUrl}; -use percent_encoding::percent_decode; use url::{Position, Url}; use zeroize::Zeroizing; -use crate::marker::{LOW_CONFIDENCE_BELOW, Marker, search, search_pair}; +use crate::marker::{ + LOW_CONFIDENCE_BELOW, Marker, found_as_whole_token, pair_found_as_whole_token, percent_decode, + search, search_pair, +}; /// Written in place of a name (key, part name, host labels) that held a typed value. pub const TYPED: &str = "{typed}"; /// Written in place of a path segment that is an identifier or held a typed value. pub const ID: &str = "{id}"; -const OTHER: &str = "{other}"; +/// Written in place of a name (key, part name, query key) that is not a typed value but does not +/// look like a name either: it is long, or holds characters a name never holds, or is an +/// identifier. Such text is more likely a value, so it is not kept. +pub const OTHER: &str = "{other}"; + +/// The longest key a kept name may have. +const MAX_NAME_CHARS: usize = 64; +/// The longest query piece with no `=` that is kept as a key. +const MAX_BARE_KEY_CHARS: usize = 32; + +/// Whether `s` looks like the name of a field: at most 64 characters, each one of +/// `A-Za-z0-9_.:-[]`, and not an identifier (all digits, a UUID, or 16 or more hex characters). +pub(crate) fn looks_like_a_name(s: &str) -> bool { + s.chars().count() <= MAX_NAME_CHARS + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | ':' | '-' | '[' | ']')) + && !segment_is_id(s) +} /// URL schemes that are kept; any other is written `{other}`. const URL_SCHEMES: [&str; 11] = [ @@ -98,19 +117,6 @@ pub(crate) fn any_hit( !hits_in(haystack, markers, pair).is_empty() } -/// Percent-decodes `raw`, with `+` as a space first when `plus_is_space`. A lone `%` stays. -pub(crate) fn decode_percent(raw: &[u8], plus_is_space: bool) -> Zeroizing> { - if plus_is_space { - let spaced: Zeroizing> = Zeroizing::new( - raw.iter() - .map(|b| if *b == b'+' { b' ' } else { *b }) - .collect(), - ); - return Zeroizing::new(percent_decode(&spaced).collect()); - } - Zeroizing::new(percent_decode(raw).collect()) -} - /// `bytes` as text of the SAME byte length: every invalid UTF-8 byte becomes `?`, so byte /// offsets in the text are offsets in `bytes`. fn text_same_len(bytes: &[u8]) -> Zeroizing { @@ -179,19 +185,24 @@ fn rebase(ranges: &[Range], start: usize, end: usize) -> Vec .collect() } -/// `search` over kept text, for the rule that applies to KEPT text only: a marker shorter than -/// `LOW_CONFIDENCE_BELOW` characters (and a pair of that combined length) is left out, because -/// its encoded forms would match inside ordinary words. Short markers are matched instead by -/// `occurrences`, as the typed text with token boundaries. Rule chosen: the simple one. +/// `search` over kept text. A marker shorter than `LOW_CONFIDENCE_BELOW` characters (and a pair +/// of that combined length) is checked in ALL its forms too, but only as a whole token: the form +/// must be bounded by characters that are not ASCII alphanumeric, or by the text's ends, because +/// its encoded forms would otherwise match inside ordinary words. fn kept_hit(haystack: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> bool { - markers - .iter() - .filter(|m| !is_short(m)) - .any(|m| !search(haystack, m).is_empty()) - || pair.is_some_and(|(u, p)| { - u.len().saturating_add(p.len()) >= LOW_CONFIDENCE_BELOW - && !search_pair(haystack, u, p).is_empty() - }) + markers.iter().any(|m| { + if is_short(m) { + found_as_whole_token(haystack, m) + } else { + !search(haystack, m).is_empty() + } + }) || pair.is_some_and(|(u, p)| { + if u.len().saturating_add(p.len()) >= LOW_CONFIDENCE_BELOW { + !search_pair(haystack, u, p).is_empty() + } else { + pair_found_as_whole_token(haystack, u, p) + } + }) } /// Whether kept text `bytes` holds a typed value: `search` finds any hit (any encoding, the pair @@ -206,7 +217,7 @@ pub(crate) fn holds_typed( return true; } [false, true].into_iter().any(|plus| { - let decoded = decode_percent(bytes, plus); + let decoded = percent_decode(bytes, plus); decoded.as_slice() != bytes && !occurrences(&text_same_len(&decoded), markers).is_empty() }) } @@ -266,7 +277,7 @@ pub fn strip_url( // them, so offsets of the host, path and query inside it are known. Its capacity covers the // text, so it is never reallocated (the copy is zeroized on drop). let host = &ser[host_start..host_end]; - let decode = |s: &str| text_same_len(&decode_percent(s.as_bytes(), false)); + let decode = |s: &str| text_same_len(&percent_decode(s.as_bytes(), false)); let mut whole = Zeroizing::new(String::with_capacity(ser.len() + 2)); whole.push_str(&decode(&ser[..host_start])); let d_host = whole.len()..whole.len() + host.len(); @@ -395,7 +406,7 @@ impl<'a> PathParts<'a> { let segments: Vec<&str> = path.split('/').collect(); let decoded: Vec>> = segments .iter() - .map(|s| decode_percent(s.as_bytes(), false)) + .map(|s| percent_decode(s.as_bytes(), false)) .collect(); let texts: Vec> = decoded.iter().map(|d| text_same_len(d)).collect(); let mut decoded_path = Zeroizing::new(String::with_capacity(path.len())); @@ -489,6 +500,8 @@ fn is_uuid(seg: &str) -> bool { struct QueryPiece<'a> { raw_key: &'a str, + /// The piece had an `=`. + has_value: bool, raw_span: Range, key: Zeroizing>, decoded_span: Range, @@ -517,7 +530,7 @@ impl<'a> QueryParts<'a> { Some((k, v)) => (k, Some(v)), None => (piece, None), }; - let key = decode_percent(raw_key.as_bytes(), true); + let key = percent_decode(raw_key.as_bytes(), true); if !decoded_query.is_empty() { decoded_query.push('&'); } @@ -526,10 +539,11 @@ impl<'a> QueryParts<'a> { let decoded_span = decoded_start..decoded_query.len(); if let Some(v) = value { decoded_query.push('='); - decoded_query.push_str(&text_same_len(&decode_percent(v.as_bytes(), true))); + decoded_query.push_str(&text_same_len(&percent_decode(v.as_bytes(), true))); } pieces.push(QueryPiece { raw_key, + has_value: value.is_some(), raw_span: start..start + raw_key.len(), key, decoded_span, @@ -543,7 +557,8 @@ impl<'a> QueryParts<'a> { } } -/// Query keys, in order, with a key a match touches written `{typed}`. The raw query string and a +/// Query keys, in order, with a key a match touches written `{typed}` and a key that does not +/// look like a name written `{other}` (a piece with no `=` must also be at most 32 characters). The raw query string and a /// decoded copy (`+` as space, `%XX`) are each searched whole, as are `raw_extra` and `dec_extra` /// (matches found over the whole URL, in raw and decoded query coordinates); a match touching a /// key's bytes (separators included) marks the key. A match inside a value alone costs nothing. @@ -578,7 +593,13 @@ fn strip_query( { TYPED.to_owned() } else { - String::from_utf8_lossy(&p.key).into_owned() + let key = String::from_utf8_lossy(&p.key); + let bare_too_long = !p.has_value && key.chars().count() > MAX_BARE_KEY_CHARS; + if looks_like_a_name(&key) && !bare_too_long { + key.into_owned() + } else { + OTHER.to_owned() + } } }) .collect(); @@ -1018,4 +1039,75 @@ mod tests { let s = strip_url("https://on.ex.test/", &on, None).unwrap(); assert_eq!(s.value.origin, "https://{typed}.ex.test"); } + + #[test] + fn the_base64_of_a_short_password_is_found_as_a_whole_path_segment_only() { + // base64 of "abcde". + let m = [pw("abcde")]; + for seg in ["YWJjZGU=", "YWJjZGU"] { + let s = strip_url(&format!("https://ex.test/x/{seg}/y"), &m, None).unwrap(); + assert_eq!(s.value.path, "/x/{id}/y", "{seg}"); + } + // The same letters inside a longer word are not a match. + let s = strip_url("https://ex.test/x/helloYWJjZGUworld/y", &m, None).unwrap(); + assert_eq!(s.value.path, "/x/helloYWJjZGUworld/y"); + // Nor in a query key. + let s = strip_url("https://ex.test/x?YWJjZGU=1&aYWJjZGUa=2", &m, None).unwrap(); + assert_eq!(s.value.query_keys, ["{typed}", "aYWJjZGUa"]); + } + + #[test] + fn a_name_is_kept_only_when_it_looks_like_one() { + for ok in [ + "user", + "user_name", + "a.b:c-d", + "items[0]", + "x", + "", + &"g".repeat(64), + ] { + assert!(looks_like_a_name(ok), "{ok:?}"); + } + for bad in [ + "12345", + "3f2b8c1e-9a4d-4c1b-8e2f-0a1b2c3d4e5f", + "0123456789abcdef", + "has space", + "a@b.c", + "a=b", + "naïve", + &"g".repeat(65), + ] { + assert!(!looks_like_a_name(bad), "{bad:?}"); + } + } + + #[test] + fn a_query_key_that_is_not_a_name_is_other() { + let token = + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.abcdefghijklmnopqrstuvwxyz0123456789"; + let s = strip_url( + &format!("https://ex.test/x?{token}&ok=1&a%20b=2&{token}=3"), + &[], + None, + ) + .unwrap(); + assert_eq!(s.value.query_keys, ["{other}", "ok", "{other}", "{other}"]); + // A bare piece of 33 characters is other; of 32 it is a key. + let s = strip_url(&format!("https://ex.test/x?{}", "k".repeat(33)), &[], None).unwrap(); + assert_eq!(s.value.query_keys, ["{other}"]); + let s = strip_url(&format!("https://ex.test/x?{}", "k".repeat(32)), &[], None).unwrap(); + assert_eq!(s.value.query_keys, ["k".repeat(32)]); + // With a value, 33 is still a name (up to 64). + let s = strip_url( + &format!("https://ex.test/x?{}=1", "k".repeat(33)), + &[], + None, + ) + .unwrap(); + assert_eq!(s.value.query_keys, ["k".repeat(33)]); + let s = strip_url("https://ex.test/x?12345=1", &[], None).unwrap(); + assert_eq!(s.value.query_keys, ["{other}"]); + } } From 8072467ea0c1a9e9391d59fd4be8bce5b75594ae Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 07:10:39 -0500 Subject: [PATCH 29/36] fix(teach): cross-site only before the password is sent; confident hits decide (#361) A move to another site counts as cross_site only when it happens before the password was sent (or, with no request carrying it, before the last typing); a later one is a plain-words note. Only http and https pages count. A weak hit of a role decides nothing when a confident one exists: password leg, first-step detection, swap candidate and which requests are kept. A two-step login whose password leg is transformed needs a person unless a digest is named. Step is tagged with kind. Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-recording/src/classify.rs | 150 ++++++++++++++++++-- crates/kharon-recording/src/format.rs | 4 +- crates/kharon-recording/src/lib.rs | 4 +- crates/kharon-recording/src/request.rs | 64 +++++++++ crates/kharon-teach/src/capture/build.rs | 33 ++--- crates/kharon-teach/src/capture/mod.rs | 169 ++++++++++++++++++++--- 6 files changed, 376 insertions(+), 48 deletions(-) diff --git a/crates/kharon-recording/src/classify.rs b/crates/kharon-recording/src/classify.rs index 3d3bff5..4c7be35 100644 --- a/crates/kharon-recording/src/classify.rs +++ b/crates/kharon-recording/src/classify.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use crate::format::{ElementDescriptor, Step}; -use crate::request::{ResourceKind, Role, StoredRequest}; +use crate::request::{Deciding, ResourceKind, Role, StoredRequest}; #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct Flags { @@ -51,14 +51,16 @@ pub struct Classification { pub notes: Vec, } -fn plain_hit(r: &StoredRequest) -> bool { +/// A plain (not digest) password hit that decides anything: a weak one counts only when no +/// confident password hit exists anywhere (see `Deciding`). +fn plain_hit(r: &StoredRequest, d: &Deciding) -> bool { r.hits() .iter() - .any(|h| h.role == Role::Password && !h.encoding.is_digest()) + .any(|h| h.role == Role::Password && !h.encoding.is_digest() && d.counts(h)) } -fn has_hit(r: &StoredRequest, role: Role) -> bool { - r.hits().iter().any(|h| h.role == role) +fn has_hit(r: &StoredRequest, role: Role, d: &Deciding) -> bool { + r.hits().iter().any(|h| h.role == role && d.counts(h)) } /// Restates the three TOTP tests in `resolveField` in @@ -94,7 +96,8 @@ fn unrecognized(note: &str) -> Classification { /// Names the shape of a recorded login and the strongest rung it can take. pub fn classify(steps: &[Step], requests: &[StoredRequest], flags: &Flags) -> Classification { - let swap_candidate = requests.iter().filter(|r| plain_hit(r)).count() == 1; + let deciding = Deciding::from_hits(requests.iter().flat_map(|r| r.hits())); + let swap_candidate = requests.iter().filter(|r| plain_hit(r, &deciding)).count() == 1; // Rule 1. let blocking = [ @@ -157,16 +160,18 @@ pub fn classify(steps: &[Step], requests: &[StoredRequest], flags: &Flags) -> Cl matches!(s, Step::Navigation { frame, .. } if frame.main) && t_ms(s) > u && t_ms(s) < p }) }); - let username_only_request = requests - .iter() - .any(|r| r.t_ms < p && has_hit(r, Role::Username) && !has_hit(r, Role::Password)); + let username_only_request = requests.iter().any(|r| { + r.t_ms < p + && has_hit(r, Role::Username, &deciding) + && !has_hit(r, Role::Password, &deciding) + }); let two_step = navigated || username_only_request; // Rule 4. let mut notes = Vec::new(); let earliest_plain = requests .iter() - .filter(|r| plain_hit(r)) + .filter(|r| plain_hit(r, &deciding)) .min_by_key(|r| r.t_ms); let leg = match earliest_plain { Some(r) => match r.resource { @@ -194,7 +199,7 @@ pub fn classify(steps: &[Step], requests: &[StoredRequest], flags: &Flags) -> Cl let digest = requests .iter() .flat_map(|r| r.hits()) - .find(|h| h.role == Role::Password && h.encoding.is_digest()) + .find(|h| h.role == Role::Password && h.encoding.is_digest() && deciding.counts(h)) .and_then(|h| h.encoding.digest_name()) .map(str::to_string); @@ -205,7 +210,9 @@ pub fn classify(steps: &[Step], requests: &[StoredRequest], flags: &Flags) -> Cl match leg { PasswordLeg::FormPost => Shape::FormPost, PasswordLeg::ScriptPost => Shape::ScriptPost, - PasswordLeg::Transformed => Shape::Transformed { digest }, + PasswordLeg::Transformed => Shape::Transformed { + digest: digest.clone(), + }, } }; @@ -223,6 +230,27 @@ pub fn classify(steps: &[Step], requests: &[StoredRequest], flags: &Flags) -> Cl "The username and password are entered on separate steps, so the agent has to carry on after the first.", ); } + if matches!( + shape, + Shape::TwoStep { + password_leg: PasswordLeg::Transformed + } + ) { + // The leg is the same uncertainty as a one-page transformed login: the password was + // found nowhere in plain form. Only a named digest lets the agent carry on. + match &digest { + Some(name) => lower( + RungCeiling::AgentContinues, + &format!( + "The password is sent as a {name} digest, so the agent has to hash it the same way." + ), + ), + None => lower( + RungCeiling::NeedsPerson, + "The password is changed before it is sent in a way this tool cannot name, so a person has to look at it.", + ), + } + } if flags.captcha { lower( RungCeiling::AgentContinues, @@ -493,6 +521,104 @@ mod tests { } ); assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); + assert!( + c.notes.iter().any(|n| n.contains("sha256 digest")), + "{:?}", + c.notes + ); + } + + #[test] + fn two_step_with_a_transformed_leg_and_no_named_digest_needs_a_person() { + let steps = vec![ + nav(0), + input(10, Role::Username, true), + nav(50), + input(60, Role::Password, true), + ]; + let c = classify( + &steps, + &[req(100, ResourceKind::Fetch, vec![])], + &Flags::default(), + ); + assert_eq!( + c.shape, + Shape::TwoStep { + password_leg: PasswordLeg::Transformed + } + ); + assert_eq!(c.rung_ceiling, RungCeiling::NeedsPerson); + assert!( + c.notes.iter().any(|n| n.contains("cannot name")), + "{:?}", + c.notes + ); + } + + fn weak(role: Role, encoding: Encoding) -> MarkerHit { + MarkerHit { + role, + encoding, + low_confidence: true, + } + } + + #[test] + fn a_weak_password_hit_does_not_pick_the_leg_when_a_confident_one_exists() { + // A short password matched by chance in an earlier request of a kind no leg is made + // from; the request that really carried it is a form post. + let reqs = [ + req( + 30, + ResourceKind::Other, + vec![weak(Role::Password, Encoding::AsTyped)], + ), + req(100, ResourceKind::Document, vec![plain_pw()]), + ]; + let c = classify(&one_page_steps(), &reqs, &Flags::default()); + assert_eq!(c.shape, Shape::FormPost); + assert!( + c.network_swap_candidate, + "only the confident request counts" + ); + // With no confident hit anywhere, the weak ones decide, as before. + let only_weak = [req( + 100, + ResourceKind::Document, + vec![weak(Role::Password, Encoding::AsTyped)], + )]; + let c = classify(&one_page_steps(), &only_weak, &Flags::default()); + assert_eq!(c.shape, Shape::FormPost); + assert!(c.network_swap_candidate); + } + + #[test] + fn a_weak_username_hit_does_not_make_a_two_step_when_a_confident_one_exists() { + let reqs = [ + req( + 15, + ResourceKind::Xhr, + vec![weak(Role::Username, Encoding::AsTyped)], + ), + req( + 100, + ResourceKind::Document, + vec![hit(Role::Username, Encoding::AsTyped), plain_pw()], + ), + ]; + let c = classify(&one_page_steps(), &reqs, &Flags::default()); + assert_eq!(c.shape, Shape::FormPost); + // Alone, a weak username-only request still reads as a first step. + let alone = [ + req( + 15, + ResourceKind::Xhr, + vec![weak(Role::Username, Encoding::AsTyped)], + ), + req(100, ResourceKind::Document, vec![plain_pw()]), + ]; + let c = classify(&one_page_steps(), &alone, &Flags::default()); + assert!(matches!(c.shape, Shape::TwoStep { .. })); } #[test] diff --git a/crates/kharon-recording/src/format.rs b/crates/kharon-recording/src/format.rs index cc7e74b..18ccd06 100644 --- a/crates/kharon-recording/src/format.rs +++ b/crates/kharon-recording/src/format.rs @@ -53,7 +53,7 @@ pub struct ElementDescriptor { /// its `value_len` is updated in place as the value changes. `change` events /// add no step. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] +#[serde(tag = "kind", rename_all = "snake_case")] pub enum Step { Navigation { t_ms: u64, @@ -390,7 +390,7 @@ mod tests { fn wire_names_are_snake_case_and_tagged() { let j = sample().to_json(); for needle in [ - "\"type\": \"navigation\"", + "\"kind\": \"navigation\"", "\"role\": \"password\"", "\"kind\": \"urlencoded\"", "\"encoding\": \"as_typed\"", diff --git a/crates/kharon-recording/src/lib.rs b/crates/kharon-recording/src/lib.rs index 87646db..5e8a110 100644 --- a/crates/kharon-recording/src/lib.rs +++ b/crates/kharon-recording/src/lib.rs @@ -12,6 +12,6 @@ pub use format::{ ListenerWorld, Recording, Step, StrippedCounts, StrippedUrl, Viewport, }; pub use request::{ - AuthShape, BodyShape, Encoding, FieldShape, JsonField, JsonShape, MarkerHit, ResourceKind, - ResponseShape, Role, StoredRequest, ValueKind, ValueShape, + AuthShape, BodyShape, Deciding, Encoding, FieldShape, JsonField, JsonShape, MarkerHit, + ResourceKind, ResponseShape, Role, StoredRequest, ValueKind, ValueShape, }; diff --git a/crates/kharon-recording/src/request.rs b/crates/kharon-recording/src/request.rs index d0f850a..422de50 100644 --- a/crates/kharon-recording/src/request.rs +++ b/crates/kharon-recording/src/request.rs @@ -60,6 +60,40 @@ pub struct MarkerHit { pub low_confidence: bool, } +/// Which weak hits still decide anything. A weak (`low_confidence`) hit of a role counts only +/// when no confident hit of that role was found anywhere in the recording: a short value matches +/// by chance in unrelated requests, and must not outvote the request that really carried it. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct Deciding { + confident_username: bool, + confident_password: bool, +} + +impl Deciding { + /// Looks at every hit given. + pub fn from_hits<'a>(hits: impl IntoIterator) -> Self { + let mut d = Self::default(); + for h in hits.into_iter().filter(|h| !h.low_confidence) { + match h.role { + Role::Username => d.confident_username = true, + Role::Password => d.confident_password = true, + Role::None => {} + } + } + d + } + + /// Whether `hit` decides anything. A hit with no role counts only when confident. + pub fn counts(&self, hit: &MarkerHit) -> bool { + let confident_elsewhere = match hit.role { + Role::Username => self.confident_username, + Role::Password => self.confident_password, + Role::None => true, + }; + !hit.low_confidence || !confident_elsewhere + } +} + #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ValueKind { @@ -184,3 +218,33 @@ impl StoredRequest { out } } + +#[cfg(test)] +mod tests { + use super::*; + + fn h(role: Role, low_confidence: bool) -> MarkerHit { + MarkerHit { + role, + encoding: Encoding::AsTyped, + low_confidence, + } + } + + #[test] + fn a_weak_hit_counts_only_when_no_confident_hit_of_its_role_exists() { + let all = [ + h(Role::Password, true), + h(Role::Password, false), + h(Role::Username, true), + ]; + let d = Deciding::from_hits(all.iter()); + assert!(!d.counts(&h(Role::Password, true))); + assert!(d.counts(&h(Role::Password, false))); + // No confident username hit anywhere: the weak one decides. + assert!(d.counts(&h(Role::Username, true))); + // A hit with no role counts only when confident. + assert!(d.counts(&h(Role::None, false))); + assert!(!d.counts(&h(Role::None, true))); + } +} diff --git a/crates/kharon-teach/src/capture/build.rs b/crates/kharon-teach/src/capture/build.rs index 6a76f83..101b5a0 100644 --- a/crates/kharon-teach/src/capture/build.rs +++ b/crates/kharon-teach/src/capture/build.rs @@ -1,14 +1,14 @@ //! Turning buffered events into the value-free parts of a `Recording`. use kharon_recording::{ - BodyShape, ElementDescriptor, FormRef, FrameRef, ResponseShape, Role, StoredRequest, + BodyShape, ElementDescriptor, FormRef, FrameRef, MarkerHit, ResponseShape, StoredRequest, StrippedUrl, }; use super::{Buffered, FrameInfo, RawBody, RawElement, RawForm}; use crate::body::shape_of_body; use crate::marker::Marker; -use crate::strip::{TYPED, hits_in, holds_typed, reduce_authorization, strip_url}; +use crate::strip::{TYPED, dedup_hits, hits_in, holds_typed, reduce_authorization, strip_url}; /// What is known while stripping: every typed value, and the credential pair once chosen. #[derive(Clone, Copy)] @@ -214,21 +214,23 @@ pub(super) fn response_shape(r: &super::BufferedResponse, ctx: Ctx<'_>) -> Respo } } -/// A hit that makes a request worth keeping: a credential, or another typed value that is not a -/// weak (very short) match. -fn is_telling(hit: &kharon_recording::MarkerHit) -> bool { - match hit.role { - Role::Username | Role::Password => true, - Role::None => !hit.low_confidence, +/// Every marker hit in the request's raw bytes: its URL, its Authorization header and its whole +/// body, with no shape built. What decides whether the request is kept. +pub(super) fn raw_hits(b: &Buffered, ctx: Ctx<'_>) -> Vec { + let mut out = hits_in(b.url.as_bytes(), ctx.markers, ctx.pair); + if let Some(a) = &b.authorization { + out.extend(reduce_authorization(a, ctx.markers, ctx.pair).markers); } + if let RawBody::Bytes(bytes) = &b.body { + out.extend(hits_in(bytes, ctx.markers, ctx.pair)); + } + dedup_hits(out) } -/// The reduced request, with the counts of what stripping its URL removed. `None` when the URL -/// does not parse. Whether to keep it is the caller's call. -pub(super) fn reduce_request( - b: &Buffered, - ctx: Ctx<'_>, -) -> Option<(StoredRequest, Tally, bool, bool)> { +/// The reduced request, with the counts of what stripping its URL removed and whether its body +/// was reduced to a shape. `None` when the URL does not parse. Whether to keep it is decided by +/// the caller BEFORE this is called, so only kept requests are shaped. +pub(super) fn reduce_request(b: &Buffered, ctx: Ctx<'_>) -> Option<(StoredRequest, Tally, bool)> { let (url, tally) = strip(&b.url, ctx)?; let reduced_body = matches!(b.body, RawBody::Bytes(_)); let body = match &b.body { @@ -253,6 +255,5 @@ pub(super) fn reduce_request( body, response: b.response.as_ref().map(|r| response_shape(r, ctx)), }; - let telling = stored.hits().into_iter().any(is_telling); - Some((stored, tally, reduced_body, telling)) + Some((stored, tally, reduced_body)) } diff --git a/crates/kharon-teach/src/capture/mod.rs b/crates/kharon-teach/src/capture/mod.rs index 0914d2a..013232a 100644 --- a/crates/kharon-teach/src/capture/mod.rs +++ b/crates/kharon-teach/src/capture/mod.rs @@ -9,8 +9,8 @@ mod resolve; mod types; use kharon_recording::{ - BrowserInfo, Flags, ListenerWorld, Recording, Role, Step, StoredRequest, StrippedCounts, - Viewport, classify, is_otp_descriptor, + BrowserInfo, Deciding, Flags, ListenerWorld, Recording, Role, Step, StoredRequest, + StrippedCounts, Viewport, classify, is_otp_descriptor, }; pub use types::{CaptureEvent, EndInfo, PageEventKind, RawBody, RawElement, RawForm, RawFrame}; use url::Url; @@ -18,7 +18,7 @@ use zeroize::Zeroizing; use crate::body::MAX_BODY_BYTES; use crate::site::{is_captcha_host, registrable_domain}; -use build::{Ctx, Tally, describe, form_ref, frame_ref, reduce_request, stripped_url}; +use build::{Ctx, Tally, describe, form_ref, frame_ref, raw_hits, reduce_request, stripped_url}; use resolve::{Elem, resolve}; /// Once this many body bytes are buffered, every further body is buffered as `Unavailable`. @@ -84,7 +84,8 @@ pub struct Capture { requests_seen: u64, popup: bool, captcha: bool, - cross_site: bool, + /// When the first main-frame navigation to another registrable domain happened. + first_cross_site_ms: Option, framed_login: bool, body_unavailable: bool, body_budget: usize, @@ -140,7 +141,7 @@ impl Capture { requests_seen: 0, popup: false, captcha: false, - cross_site: false, + first_cross_site_ms: None, framed_login: false, body_unavailable: false, body_budget: MAX_BUFFERED_BODY_BYTES, @@ -245,8 +246,14 @@ impl Capture { if frame.main { self.main_origin = raw_origin(&frame.url); self.final_url = Some(frame.url.clone()); - if host_of(&frame.url).is_some_and(|h| registrable_domain(&h) != self.login_site) { - self.cross_site = true; + // Only web pages count: an error page (`chrome-error://`) is not another site. + let other_site = Url::parse(&frame.url).is_ok_and(|u| { + matches!(u.scheme(), "http" | "https") + && u.host_str() + .is_some_and(|h| registrable_domain(h) != self.login_site) + }); + if other_site { + self.first_cross_site_ms.get_or_insert(t_ms); } } let info = self.frame_info(frame); @@ -370,24 +377,57 @@ impl Capture { .map(|u| stripped_url(u, ctx, &mut tally)); let steps = self.build_steps(&resolved, ctx, &mut tally); + // Which requests are kept is decided on the raw bytes, before any body is shaped. A weak + // (short-value) hit counts only when no confident hit of its role exists anywhere. + let hits: Vec> = + self.reqs.iter().map(|b| raw_hits(b, ctx)).collect(); + let deciding = Deciding::from_hits(hits.iter().flatten()); let mut requests: Vec = Vec::new(); let mut bodies_reduced = 0; - for b in &self.reqs { - let Some((stored, t, reduced, telling)) = reduce_request(b, ctx) else { + for (b, hs) in self.reqs.iter().zip(&hits) { + let telling = hs.iter().any(|h| deciding.counts(h)); + if !(b.main_frame_navigation || telling || b.login_site_write) { continue; - }; - if b.main_frame_navigation || telling || b.login_site_write { - tally.add(t); - bodies_reduced += u64::from(reduced); - requests.push(stored); } + let Some((stored, t, reduced)) = reduce_request(b, ctx) else { + continue; + }; + tally.add(t); + bodies_reduced += u64::from(reduced); + requests.push(stored); } + // The login is over once the password has been sent: the last request carrying it, or + // when none does, the last typing. A move to another site after that is not part of it. + let credentials_sent = self + .reqs + .iter() + .zip(&hits) + .filter(|(_, hs)| { + hs.iter() + .any(|h| h.role == Role::Password && deciding.counts(h)) + }) + .map(|(b, _)| b.t_ms) + .max() + .or_else(|| { + steps + .iter() + .filter_map(|s| match s { + Step::Input { t_ms, .. } => Some(*t_ms), + _ => None, + }) + .max() + }); + let cross_site = self + .first_cross_site_ms + .is_some_and(|t| credentials_sent.is_none_or(|c| t < c)); + let went_on_after_login = self.first_cross_site_ms.is_some() && !cross_site; + let flags = Flags { captcha: self.captcha, otp_step: self.otp_step(), - cross_site: self.cross_site, + cross_site, popup: self.popup, framed_login: self.framed_login, shadow_dom: self.elems.iter().enumerate().any(|(i, e)| { @@ -396,7 +436,12 @@ impl Capture { multiple_passwords: resolved.multiple_passwords, body_unavailable: self.body_unavailable, }; - let classification = classify(&steps, &requests, &flags); + let mut classification = classify(&steps, &requests, &flags); + if went_on_after_login { + classification + .notes + .push("After the login the browser went on to another site.".to_owned()); + } Recording { format_version: kharon_recording::FORMAT_VERSION, tool_version: self.tool_version.clone(), @@ -1171,6 +1216,69 @@ mod tests { assert!(scenario("cross_site").flags.cross_site); } + const WENT_ON: &str = "After the login the browser went on to another site."; + + #[test] + fn a_move_to_another_site_after_the_login_is_a_note_and_no_flag() { + let rec = Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .click(&button()) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .nav("https://app.other.test/home") + .finish(); + assert!(!rec.flags.cross_site); + assert_eq!(rec.classification.shape, Shape::FormPost); + assert!( + rec.classification.notes.iter().any(|n| n == WENT_ON), + "{:?}", + rec.classification.notes + ); + } + + #[test] + fn a_move_to_another_site_before_the_password_is_sent_is_cross_site() { + let rec = scenario("cross_site"); + assert!(rec.flags.cross_site); + assert_eq!(rec.classification.shape, Shape::Unrecognized); + assert!(!rec.classification.notes.iter().any(|n| n == WENT_ON)); + } + + #[test] + fn with_no_request_carrying_the_password_the_last_typing_is_the_mark() { + // Typing, then a move to another site, then nothing sent that holds the password. + let after = Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .nav("https://app.other.test/home") + .finish(); + assert!(!after.flags.cross_site); + let before = Session::new(LOGIN) + .type_into(&user_el(), USER) + .nav("https://app.other.test/home") + .type_into(&pw_el(), PASS) + .finish(); + assert!(before.flags.cross_site); + } + + #[test] + fn a_browser_error_page_is_not_another_site() { + let rec = Session::new(LOGIN) + .type_into(&user_el(), USER) + .nav("chrome-error://chromewebdata/") + .type_into(&pw_el(), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish(); + assert!(!rec.flags.cross_site); + assert!(!rec.classification.notes.iter().any(|n| n == WENT_ON)); + } + #[test] fn same_site_subdomain_is_not_cross_site() { assert!(!scenario("subdomain").flags.cross_site); @@ -1238,6 +1346,35 @@ mod tests { ); } + #[test] + fn a_weak_hit_does_not_store_a_request_or_pick_the_leg_when_a_confident_one_exists() { + // The password is five characters, so on its own it matches by chance. The combined + // credential is long enough to be a confident hit for both roles. + let (user, pass) = ("alice1", "abcde"); + let combined = Base64::encode_string(format!("{user}:{pass}").as_bytes()); + let rec = Session::new(LOGIN) + .type_into(&user_el(), user) + .type_into(&pw_el(), pass) + .post_json("https://ads.other.test/collect", r#"{"note":"abcde"}"#) + .post_json( + "https://ex.test/api/login", + &format!(r#"{{"auth":"{combined}"}}"#), + ) + .nav("https://ex.test/home") + .finish(); + let paths: Vec<&str> = rec.requests.iter().map(|r| r.url.path.as_str()).collect(); + assert_eq!(paths, ["/api/login"], "the chance match was stored"); + assert_eq!(rec.classification.shape, Shape::ScriptPost); + assert!(rec.classification.network_swap_candidate); + // With nothing confident anywhere, a weak hit still keeps its request. + let rec = Session::new(LOGIN) + .type_into(&user_el(), "bob") + .type_into(&pw_el(), "abcde") + .post_json("https://ads.other.test/collect", r#"{"note":"abcde"}"#) + .finish(); + assert_eq!(rec.requests.len(), 1); + } + #[test] fn only_the_three_kinds_of_request_are_stored() { let rec = scenario("which_requests"); From 9e05de02424490b9a2af96ea1f71ec20543e3373 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 07:14:06 -0500 Subject: [PATCH 30/36] fix(teach): bound the work and the output a page can cause (#361) Each marker builds its encoded forms once. A body whose shape would hold more than 2,000 values is Opaque, and building stops at the cap. Which requests are kept is decided on raw bytes before any shape is built (done with the confident-hit rule in the previous commit). Steps are capped at 10,000 with a steps_dropped count shown by summary, at most 5,000 requests are buffered, and an element keeps a fixed amount of bookkeeping however much a page types. Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-recording/src/format.rs | 4 + crates/kharon-teach/src/body.rs | 175 ++++++++++++++++++--- crates/kharon-teach/src/capture/mod.rs | 123 ++++++++++++++- crates/kharon-teach/src/capture/resolve.rs | 10 +- crates/kharon-teach/src/cli.rs | 11 ++ crates/kharon-teach/src/marker.rs | 53 ++++++- 6 files changed, 337 insertions(+), 39 deletions(-) diff --git a/crates/kharon-recording/src/format.rs b/crates/kharon-recording/src/format.rs index 18ccd06..1ef4943 100644 --- a/crates/kharon-recording/src/format.rs +++ b/crates/kharon-recording/src/format.rs @@ -126,6 +126,9 @@ pub struct StrippedCounts { pub path_segments_replaced: u64, /// Typed input values discarded (only lengths are kept). pub typed_values_removed: u64, + /// Steps past the recorder's limit that were not written to `steps`. + #[serde(default)] + pub steps_dropped: u64, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -314,6 +317,7 @@ mod tests { query_values_removed: 0, path_segments_replaced: 0, typed_values_removed: 2, + steps_dropped: 0, }, } } diff --git a/crates/kharon-teach/src/body.rs b/crates/kharon-teach/src/body.rs index 8b14d28..c750e5d 100644 --- a/crates/kharon-teach/src/body.rs +++ b/crates/kharon-teach/src/body.rs @@ -17,6 +17,26 @@ use crate::strip::{OTHER, TYPED, dedup_hits, hits_in, holds_typed, looks_like_a_ /// Bodies larger than this are not parsed: they are `Opaque`. pub const MAX_BODY_BYTES: usize = 1 << 20; +/// A body whose shape would hold more than this many values (JSON nodes, urlencoded fields, +/// multipart parts) is `Opaque`: a page can send telemetry batches of any size, and a shape +/// as large as the body helps nobody. +pub const MAX_SHAPE_VALUES: usize = 2_000; + +/// What is left of the value budget for one body. Building stops as soon as it runs out. +struct Budget(usize); + +impl Budget { + fn new() -> Self { + Self(MAX_SHAPE_VALUES) + } + + /// Takes one value, or `None` when none is left. + fn take(&mut self) -> Option<()> { + self.0 = self.0.checked_sub(1)?; + Some(()) + } +} + /// `content_type` picks the reader: application/x-www-form-urlencoded, application/json (and any +json), /// multipart/form-data; anything else, or a body that fails to parse as its declared type, is Opaque. /// `pair` is (username, password) once both are known; see below. @@ -27,6 +47,7 @@ pub const MAX_BODY_BYTES: usize = 1 << 20; /// and its hits are added to its field; a name that does not look like a name (see /// `strip::looks_like_a_name`) is written as `{other}`. A urlencoded body with a piece that has /// no `=`, or whose names mostly do not look like names, is not urlencoded data and is `Opaque`. +/// So is a body whose shape would hold more than `MAX_SHAPE_VALUES` values. /// Never returns `None` or `Unavailable`. pub fn shape_of_body( content_type: Option<&str>, @@ -47,9 +68,8 @@ pub fn shape_of_body( // The one permitted use of `serde_json::Value`: a site's JSON body while its shape is extracted. serde_json::from_slice::(body) .ok() - .map(|v| BodyShape::Json { - root: json_shape(&v, markers, pair), - }) + .and_then(|v| json_shape(&v, markers, pair, &mut Budget::new())) + .map(|root| BodyShape::Json { root }) } _ => None, }, @@ -189,8 +209,10 @@ fn urlencoded_fields( markers: &[Marker], pair: Option<(&Marker, &Marker)>, ) -> Option> { + let mut budget = Budget::new(); let mut pieces = Vec::new(); for piece in body.split(|b| *b == b'&').filter(|p| !p.is_empty()) { + budget.take()?; let i = piece.iter().position(|b| *b == b'=')?; pieces.push((decode_percent(&piece[..i], true), &piece[i + 1..])); } @@ -234,28 +256,34 @@ fn add_hits_to_first_value(shape: &mut JsonShape, extra: Vec) -> bool } } +/// The shape of `v`, or `None` when it would hold more values than `budget` has left. fn json_shape( v: &serde_json::Value, markers: &[Marker], pair: Option<(&Marker, &Marker)>, -) -> JsonShape { + budget: &mut Budget, +) -> Option { use serde_json::Value; + budget.take()?; // Depth is bounded by serde_json's own recursion limit (128) at parse time. - match v { - Value::Object(map) => JsonShape::Object { - fields: map - .iter() - .map(|(k, v)| { - let (key, key_hits) = name(k.as_bytes(), markers, pair); - let mut value = json_shape(v, markers, pair); - add_hits_to_first_value(&mut value, key_hits); - JsonField { key, value } - }) - .collect(), - }, - Value::Array(items) => JsonShape::Array { - items: items.iter().map(|i| json_shape(i, markers, pair)).collect(), - }, + Some(match v { + Value::Object(map) => { + let mut fields = Vec::new(); + for (k, v) in map { + let (key, key_hits) = name(k.as_bytes(), markers, pair); + let mut value = json_shape(v, markers, pair, budget)?; + add_hits_to_first_value(&mut value, key_hits); + fields.push(JsonField { key, value }); + } + JsonShape::Object { fields } + } + Value::Array(items) => { + let mut shapes = Vec::new(); + for i in items { + shapes.push(json_shape(i, markers, pair, budget)?); + } + JsonShape::Array { items: shapes } + } Value::String(s) => JsonShape::Value { value: text_value(s.as_bytes(), markers, pair), }, @@ -283,7 +311,7 @@ fn json_shape( Value::Null => JsonShape::Value { value: value_shape(ValueKind::Null, 0, b"null", markers, pair), }, - } + }) } fn find(haystack: &[u8], needle: &[u8], from: usize) -> Option { @@ -318,11 +346,13 @@ fn multipart_parts( // The delimiter that closes a part is preceded by a line break. let closing = [eol, delim.as_slice()].concat(); + let mut budget = Budget::new(); let mut parts = Vec::new(); loop { if body[pos..].starts_with(b"--") { return Some(parts); } + budget.take()?; // Rest of the delimiter line (transport padding), then the headers. let headers_start = pos + body[pos..].iter().position(|b| *b == b'\n')? + 1; let headers_end = find(body, &blank_line, headers_start)?; @@ -829,9 +859,11 @@ mod tests { #[test] fn many_crlf_parts_are_read_in_linear_time() { // The earlier quadratic parse (CRLF bodies never matched the LF-only terminator, so every - // part scanned the rest of the body) took minutes at this size; this one takes milliseconds. + // part scanned the rest of the body) took minutes at 20,000 parts. Decision change: a + // body of more parts than `MAX_SHAPE_VALUES` is now Opaque, so this is the most parts + // that are still read. let mut body = Vec::new(); - for _ in 0..20_000 { + for _ in 0..MAX_SHAPE_VALUES { body.extend_from_slice(b"--b\r\nContent-Disposition: form-data; name=f\r\n\r\nv\r\n"); } body.extend_from_slice(b"--b--\r\n"); @@ -845,7 +877,7 @@ mod tests { let BodyShape::Multipart { parts } = s else { panic!("not multipart") }; - assert_eq!(parts.len(), 20_000); + assert_eq!(parts.len(), MAX_SHAPE_VALUES); // LF-only bodies too. let lf = String::from_utf8_lossy(&body).replace("\r\n", "\n"); let s = shape_of_body( @@ -857,7 +889,7 @@ mod tests { let BodyShape::Multipart { parts } = s else { panic!("not multipart") }; - assert_eq!(parts.len(), 20_000); + assert_eq!(parts.len(), MAX_SHAPE_VALUES); assert!( started.elapsed() < std::time::Duration::from_secs(5), "multipart parse is not linear" @@ -1071,4 +1103,99 @@ mod tests { assert!(markers.contains(&hit(Role::Password, Encoding::AsTyped))); assert!(markers.contains(&hit(Role::Username, Encoding::AsTyped))); } + + #[test] + fn a_big_json_array_is_opaque_quickly_and_small() { + // About 1 MiB of small numbers: the case that made a 33 MB shape. + let mut body = String::from("["); + while body.len() < MAX_BODY_BYTES - 16 { + body.push_str("1,"); + } + body.push_str("1]"); + assert!(body.len() <= MAX_BODY_BYTES); + let started = std::time::Instant::now(); + let s = shape_of_body(Some("application/json"), body.as_bytes(), &markers(), None); + assert!( + started.elapsed() < std::time::Duration::from_secs(1), + "took {:?}", + started.elapsed() + ); + let BodyShape::Opaque { len, .. } = &s else { + panic!("not opaque") + }; + assert_eq!(usize::try_from(*len).unwrap(), body.len()); + assert!(serialized(&s).len() < 1024); + } + + #[test] + fn a_shape_of_exactly_the_cap_is_kept_and_one_more_value_is_opaque() { + // An array is one node and each item another. + let items = |n: usize| format!("[{}]", vec!["1"; n].join(",")); + let ok = shape_of_body( + Some("application/json"), + items(MAX_SHAPE_VALUES - 1).as_bytes(), + &[], + None, + ); + assert!(matches!(ok, BodyShape::Json { .. })); + let over = shape_of_body( + Some("application/json"), + items(MAX_SHAPE_VALUES).as_bytes(), + &[], + None, + ); + assert!(matches!(over, BodyShape::Opaque { .. })); + // Urlencoded fields and multipart parts have the same cap. + let form = |n: usize| { + (0..n) + .map(|i| format!("k{i}=v")) + .collect::>() + .join("&") + }; + assert!(matches!( + shape_of_body(Some(URLENC), form(MAX_SHAPE_VALUES).as_bytes(), &[], None), + BodyShape::Urlencoded { .. } + )); + assert!(matches!( + shape_of_body( + Some(URLENC), + form(MAX_SHAPE_VALUES + 1).as_bytes(), + &[], + None + ), + BodyShape::Opaque { .. } + )); + let multipart = |n: usize| { + let mut b = Vec::new(); + for _ in 0..n { + b.extend_from_slice(b"--b\r\nContent-Disposition: form-data; name=f\r\n\r\nv\r\n"); + } + b.extend_from_slice(b"--b--\r\n"); + b + }; + assert!(matches!( + shape_of_body( + Some("multipart/form-data; boundary=b"), + &multipart(MAX_SHAPE_VALUES + 1), + &[], + None + ), + BodyShape::Opaque { .. } + )); + } + + #[test] + fn a_capped_body_still_reports_whole_body_hits() { + let m = [pw("Tr0ub4dor-9x")]; + let mut body = String::from("[\"Tr0ub4dor-9x\""); + for _ in 0..MAX_SHAPE_VALUES { + body.push_str(",1"); + } + body.push(']'); + let s = shape_of_body(Some("application/json"), body.as_bytes(), &m, None); + let BodyShape::Opaque { markers, .. } = s else { + panic!("not opaque") + }; + assert!(markers.contains(&hit(Role::Password, Encoding::AsTyped))); + } } diff --git a/crates/kharon-teach/src/capture/mod.rs b/crates/kharon-teach/src/capture/mod.rs index 013232a..5eddf1c 100644 --- a/crates/kharon-teach/src/capture/mod.rs +++ b/crates/kharon-teach/src/capture/mod.rs @@ -23,6 +23,11 @@ use resolve::{Elem, resolve}; /// Once this many body bytes are buffered, every further body is buffered as `Unavailable`. pub const MAX_BUFFERED_BODY_BYTES: usize = 32 << 20; +/// Steps kept. Later ones are counted in `StrippedCounts::steps_dropped` and not recorded. Typed +/// values are still tracked: only the step list stops growing. +pub const MAX_STEPS: usize = 10_000; +/// Requests buffered. Later ones are counted in `requests_seen` and not buffered. +pub const MAX_BUFFERED_REQUESTS: usize = 5_000; /// A frame as it was when an event arrived. struct FrameInfo { @@ -77,6 +82,11 @@ pub struct Capture { main_origin: Option, final_url: Option, steps: Vec, + steps_dropped: u64, + max_steps: usize, + max_requests: usize, + /// The first sequence number of the first element known to be a password. + first_password_seq: Option, elems: Vec, reqs: Vec, seq: u64, @@ -134,6 +144,10 @@ impl Capture { main_origin: raw_origin(start_url), final_url: None, steps: Vec::new(), + steps_dropped: 0, + max_steps: MAX_STEPS, + max_requests: MAX_BUFFERED_REQUESTS, + first_password_seq: None, elems: Vec::new(), reqs: Vec::new(), seq: 0, @@ -257,13 +271,22 @@ impl Capture { } } let info = self.frame_info(frame); - self.steps.push(Pending { + self.add_step(Pending { t_ms, frame: info, what: What::Navigation(frame.url.clone()), }); } + /// Keeps the step, or counts it when the step list is full. + fn add_step(&mut self, step: Pending) { + if self.steps.len() < self.max_steps { + self.steps.push(step); + } else { + self.steps_dropped += 1; + } + } + fn page( &mut self, t_ms: u64, @@ -290,7 +313,7 @@ impl Capture { has_step: false, first_seq: self.seq + 1, last_seq: 0, - seqs: Vec::new(), + last_before_password: None, }); let i = self.elems.len() - 1; self.note_report(i, &element, &info); @@ -299,7 +322,9 @@ impl Capture { }; self.seq += 1; self.elems[i].last_seq = self.seq; - self.elems[i].seqs.push(self.seq); + if self.first_password_seq.is_none_or(|s| self.seq < s) { + self.elems[i].last_before_password = Some(self.seq); + } if let Some(v) = value { self.elems[i].value = v; } @@ -320,7 +345,7 @@ impl Capture { None => return, }, }; - self.steps.push(Pending { + self.add_step(Pending { t_ms, frame: info, what, @@ -330,6 +355,11 @@ impl Capture { fn note_report(&mut self, i: usize, element: &RawElement, info: &FrameInfo) { let e = &mut self.elems[i]; e.was_password |= element.is_password_now; + if e.was_password { + let first = e.first_seq; + self.first_password_seq = Some(self.first_password_seq.map_or(first, |s| s.min(first))); + } + let e = &mut self.elems[i]; e.shadow |= element.in_shadow_root; if e.was_password && !info.main && info.cross_origin { self.framed_login = true; @@ -341,6 +371,9 @@ impl Capture { if is_captcha_url(&b.url) { self.captcha = true; } + if self.reqs.len() >= self.max_requests { + return; + } if let RawBody::Bytes(bytes) = &b.body { if bytes.len() > MAX_BODY_BYTES || self.buffered_body_bytes.saturating_add(bytes.len()) > self.body_budget @@ -470,6 +503,7 @@ impl Capture { query_values_removed: tally.query_values_removed, path_segments_replaced: tally.path_segments_replaced, typed_values_removed: resolved.markers.len() as u64, + steps_dropped: self.steps_dropped, }, requests, } @@ -1620,6 +1654,87 @@ mod tests { let _ = rec.to_json(); } + impl Capture { + /// A capture with small step and request limits, for the tests of those limits. + fn with_limits(start_url: &str, max_steps: usize, max_requests: usize) -> Self { + Self { + max_steps, + max_requests, + ..Self::new(start_url, "0.0.0") + } + } + } + + fn session_with_limits(max_steps: usize, max_requests: usize) -> Session { + let mut s = Session::unstarted(LOGIN); + s.cap = Capture::with_limits(LOGIN, max_steps, max_requests); + s.start(); + s + } + + #[test] + fn steps_past_the_limit_are_counted_but_their_values_still_count_as_typed() { + // Step 1 is the first navigation, step 2 the user name; the password input and the + // clicks after them do not fit. + let rec = session_with_limits(2, MAX_BUFFERED_REQUESTS) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .click(&button()) + .click(&button()) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish(); + assert_eq!(rec.steps.len(), 2); + assert_eq!(rec.stripped.steps_dropped, 3); + // The dropped password input is still a typed value: it is found and never written. + assert_eq!(rec.stripped.typed_values_removed, 2); + assert!( + rec.requests + .iter() + .flat_map(StoredRequest::hits) + .any(|h| h.role == Role::Password) + ); + assert_holds_no_typed_value(&rec.to_json(), &[USER, PASS]); + // Below the limit nothing is dropped. + assert_eq!(scenario("form_post").stripped.steps_dropped, 0); + assert_eq!(MAX_STEPS, 10_000); + } + + #[test] + fn requests_past_the_limit_are_counted_and_not_buffered() { + let mut s = session_with_limits(MAX_STEPS, 3).type_into(&user_el(), USER); + for i in 0..6 { + s = s.post_form(&format!("https://ex.test/p{i}"), "a=1"); + } + let rec = s.finish(); + assert_eq!(rec.stripped.requests_seen, 6); + // Main-frame navigations are always stored, so what is stored is what was buffered. + let paths: Vec<&str> = rec.requests.iter().map(|r| r.url.path.as_str()).collect(); + assert_eq!(paths, ["/p0", "/p1", "/p2"]); + assert_eq!(MAX_BUFFERED_REQUESTS, 5_000); + } + + #[test] + fn an_element_typed_into_endlessly_keeps_its_place_in_the_username_choice() { + // Thousands of events on one field, then the password: the user name is still found + // from the last thing typed before the password, with no per-event record kept. + let mut s = Session::new(LOGIN); + for _ in 0..5_000 { + s = s.type_into(&user_el(), USER); + } + let rec = s + .type_into(&pw_el(), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish(); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(rec.classification.shape, Shape::FormPost); + } + fn session_with_budget(budget: usize) -> Session { let mut s = Session::unstarted(LOGIN); s.cap = Capture::with_body_budget(LOGIN, "0.0.0", budget); diff --git a/crates/kharon-teach/src/capture/resolve.rs b/crates/kharon-teach/src/capture/resolve.rs index b1d3b32..aa5d843 100644 --- a/crates/kharon-teach/src/capture/resolve.rs +++ b/crates/kharon-teach/src/capture/resolve.rs @@ -17,8 +17,12 @@ pub(super) struct Elem { /// Sequence numbers of the first and the latest value-bearing event. pub first_seq: u64, pub last_seq: u64, - /// Sequence numbers of every value-bearing event. - pub seqs: Vec, + /// The latest value-bearing event seen before any password element was known. Kept as a + /// running value so an element's bookkeeping has a fixed size however much a page types. + /// If an element only turns out to be a password after other fields were typed into, this + /// can be later than the first password input; the username choice then falls back on the + /// element's last input. + pub last_before_password: Option, } impl Elem { @@ -74,7 +78,7 @@ fn pick(cands: &[usize], elems: &[Elem], first_password_seq: Option) -> Opt let last_before = |i: usize| -> Option { match first_password_seq { None => Some(elems[i].last_seq), - Some(s) => elems[i].seqs.iter().copied().filter(|q| *q < s).max(), + Some(s) => elems[i].last_before_password.filter(|q| *q < s), } }; cands diff --git a/crates/kharon-teach/src/cli.rs b/crates/kharon-teach/src/cli.rs index 5403e56..e2e9ad3 100644 --- a/crates/kharon-teach/src/cli.rs +++ b/crates/kharon-teach/src/cli.rs @@ -278,6 +278,12 @@ pub fn summary(r: &Recording) -> String { s.requests_seen, s.request_bodies_reduced )); + if s.steps_dropped > 0 { + out.push(format!( + "Not recorded: {} step(s) past the recorder's limit.", + s.steps_dropped + )); + } let mut text = out.join("\n"); text.push('\n'); text @@ -438,6 +444,7 @@ pub(crate) mod tests { query_values_removed: 3, path_segments_replaced: 1, typed_values_removed: 5, + steps_dropped: 0, }, } } @@ -462,6 +469,10 @@ pub(crate) mod tests { assert!(!s.contains('{') && !s.contains('"')); let quiet = summary(&recording()); assert!(!quiet.contains("Caution") && !quiet.contains("Flag:")); + assert!(!quiet.contains("Not recorded")); + let mut capped = recording(); + capped.stripped.steps_dropped = 7; + assert!(summary(&capped).contains("Not recorded: 7 step(s) past the recorder's limit.")); assert!(quiet.contains("does not look like a candidate")); } diff --git a/crates/kharon-teach/src/marker.rs b/crates/kharon-teach/src/marker.rs index 54501f3..b60154a 100644 --- a/crates/kharon-teach/src/marker.rs +++ b/crates/kharon-teach/src/marker.rs @@ -7,6 +7,8 @@ use md5::Md5; use memchr::memmem; use sha1::Sha1; use sha2::{Digest, Sha256, Sha512}; +use std::sync::OnceLock; + use zeroize::Zeroizing; /// Markers shorter than this many characters produce low-confidence hits. @@ -31,11 +33,23 @@ pub const LOW_CONFIDENCE_BELOW: u32 = 6; pub struct Marker { value: Zeroizing, role: Role, + /// The encoded forms searched for, built on first use and then reused by every search. + /// Zeroized buffers; dropped with the marker. + forms: OnceLock)>>, } impl Marker { pub fn new(value: Zeroizing, role: Role) -> Self { - Self { value, role } + Self { + value, + role, + forms: OnceLock::new(), + } + } + + /// The forms to search for, in `Encoding` order. Built once. + fn forms(&self) -> &[(Encoding, Vec)] { + self.forms.get_or_init(|| build_forms(self)) } pub fn role(&self) -> Role { @@ -56,8 +70,9 @@ impl Marker { self.value.as_str() == other.value.as_str() } + /// The same value under another role. The forms are rebuilt: only a password has digests. pub fn with_role(self, role: Role) -> Self { - Self { role, ..self } + Self::new(self.value, role) } /// Byte range of the first place this marker's value occurs in `haystack`, comparing ASCII letters @@ -163,14 +178,15 @@ pub fn search(haystack: &[u8], marker: &Marker) -> Vec { low_confidence, }); }; - for (encoding, forms) in needles(marker) { + for (encoding, forms) in marker.forms() { + let encoding = *encoding; if encoding == Encoding::AsTyped { - if haystacks.in_raw(&forms) { + if haystacks.in_raw(forms) { hit(Encoding::AsTyped); - } else if haystacks.in_decoded(&forms) { + } else if haystacks.in_decoded(forms) { hit(Encoding::Percent); } - } else if haystacks.in_raw(&forms) || haystacks.in_decoded(&forms) { + } else if haystacks.in_raw(forms) || haystacks.in_decoded(forms) { hit(encoding); } } @@ -239,7 +255,8 @@ pub(crate) fn found_as_whole_token(haystack: &[u8], marker: &Marker) -> bool { return false; } let haystacks = Haystacks::new(haystack); - needles(marker) + marker + .forms() .iter() .any(|(_, forms)| contains_whole_token(&haystacks, forms)) } @@ -280,7 +297,7 @@ fn contains(haystack: &[u8], needle: &[u8]) -> bool { } /// The search forms per encoding, in `Encoding` order. -fn needles(marker: &Marker) -> Vec<(Encoding, Vec)> { +fn build_forms(marker: &Marker) -> Vec<(Encoding, Vec)> { let value = marker.value.as_bytes(); let (standard, url_safe) = base64_forms(value); let standard = dedup(standard, &[value]); @@ -460,6 +477,26 @@ mod tests { search(h, m).into_iter().map(|x| x.encoding).collect() } + #[test] + fn many_searches_with_one_marker_are_fast() { + // The encoded forms (four digests, base64, hex, JSON) are built once per marker, not once + // per search. Measured in a debug build: rebuilding them for each search took about 4.2 s + // for this many searches, reusing them about 0.8 s. + let m = pw("Tr0ub4dor-9x"); + let hay = b"user=kh.throwaway7&csrf=abc123&password=Tr0ub4dor-9x"; + let started = std::time::Instant::now(); + let mut found = 0; + for _ in 0..40_000 { + found += search(hay, &m).len(); + } + assert_eq!(found, 40_000); + assert!( + started.elapsed() < std::time::Duration::from_secs(2), + "40,000 searches took {:?}", + started.elapsed() + ); + } + #[test] fn as_typed() { assert_eq!( From e766387c27fd72624e49662f578c7c61fd4f69af Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 07:18:18 -0500 Subject: [PATCH 31/36] fix(teach): honest messages, a panic hook, snap browsers and safe show output (#361) The never-started message and the notice page say to open the link in the first tab. The end prompt confirms on Enter, discards on q or quit and asks again otherwise. A panic hook prints only the place. show replaces control and bidi characters from the file. A snap browser is skipped, with its own error when it is the only one. Source scans now cover all shipped code, and req_tch_002 has a fixture that puts typed values into kept text. Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-teach/src/browser.rs | 138 +++++++++++++++++- crates/kharon-teach/src/capture/mod.rs | 16 +- crates/kharon-teach/src/cli.rs | 50 ++++++- crates/kharon-teach/src/lib.rs | 80 +++++++--- crates/kharon-teach/src/main.rs | 24 +++ crates/kharon-teach/src/notice.rs | 13 +- .../tests/req_tch_001_own_browser.rs | 19 +-- .../tests/req_tch_002_no_typed_value.rs | 51 +++++++ .../tests/req_tch_006_profile_and_visible.rs | 76 +++++++++- crates/kharon-teach/tests/support/mod.rs | 51 ++++++- 10 files changed, 462 insertions(+), 56 deletions(-) diff --git a/crates/kharon-teach/src/browser.rs b/crates/kharon-teach/src/browser.rs index 653de21..32a79e7 100644 --- a/crates/kharon-teach/src/browser.rs +++ b/crates/kharon-teach/src/browser.rs @@ -10,6 +10,10 @@ use std::time::Duration; pub enum BrowserError { #[error("no Chromium-family browser found; pass --browser ")] NotFound, + #[error( + "the only browser found is a snap package, which cannot use the recorder's profile folder; install Google Chrome or pass --browser " + )] + SnapOnly, #[error("browser not found at {0}")] BadPath(PathBuf), #[error("the browser did not start: {0}")] @@ -39,6 +43,16 @@ pub fn find_browser(explicit: Option<&Path>) -> Result { } pub fn find_browser_in(explicit: Option<&Path>, path_var: &OsStr) -> Result { + find_browser_with(explicit, path_var, &is_snap_browser) +} + +/// [`find_browser_in`] with the snap test given, so a test can stand in for the file system. +/// An explicit path is used as given, snap or not: the person chose it. +pub(crate) fn find_browser_with( + explicit: Option<&Path>, + path_var: &OsStr, + is_snap: &dyn Fn(&Path) -> bool, +) -> Result { if let Some(p) = explicit { return if p.is_file() { Ok(p.to_owned()) @@ -46,15 +60,44 @@ pub fn find_browser_in(explicit: Option<&Path>, path_var: &OsStr) -> Result` link resolves to. +pub(crate) fn is_snap_path(path: &Path) -> bool { + path.starts_with("/snap") || path.file_name().is_some_and(|n| n == "snap") +} + +/// Whether the candidate browser is a snap: its path or its canonical path is a snap path, or it +/// is a short wrapper script that starts one (Ubuntu's `chromium-browser`). +fn is_snap_browser(cand: &Path) -> bool { + use std::io::Read; + if is_snap_path(cand) || std::fs::canonicalize(cand).is_ok_and(|c| is_snap_path(&c)) { + return true; + } + let mut head = Vec::new(); + let read = std::fs::File::open(cand).and_then(|f| f.take(4096).read_to_end(&mut head)); + read.is_ok() && head.starts_with(b"#!") && head.windows(6).any(|w| w == b"/snap/") } fn is_executable_file(p: &Path) -> bool { @@ -821,6 +864,95 @@ mod tests { assert_eq!(found, dir.join("chromium")); } + #[test] + fn snap_paths_are_recognised() { + for snap in [ + "/snap/bin/chromium", + "/snap/chromium/current/usr/lib/chromium-browser/chrome", + // What `/snap/bin/chromium` resolves to once canonicalised. + "/usr/bin/snap", + ] { + assert!(is_snap_path(Path::new(snap)), "{snap}"); + } + for other in [ + "/usr/bin/chromium", + "/usr/bin/google-chrome", + "/opt/google/chrome/chrome", + "/home/me/snapshots/chrome", + "/usr/bin/snapd", + ] { + assert!(!is_snap_path(Path::new(other)), "{other}"); + } + } + + fn executable(dir: &Path, name: &str, body: &str) -> PathBuf { + let f = dir.join(name); + std::fs::write(&f, body).unwrap(); + std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o755)).unwrap(); + f + } + + #[test] + fn a_snap_candidate_is_skipped_for_the_next_one_in_order() { + let sb = Sandbox::new("snaporder"); + let a = sb.mkdir("a"); + let b = sb.mkdir("b"); + executable(&a, "chromium", "#!/bin/sh\n"); + let chrome = executable(&b, "google-chrome", "#!/bin/sh\n"); + let path = std::env::join_paths([&a, &b]).unwrap(); + // The stub calls only the first candidate a snap. + let is_snap = |p: &Path| p == a.join("chromium"); + assert_eq!(find_browser_with(None, &path, &is_snap).unwrap(), chrome); + // With nothing to fall back on, the error says why. + let only_a = std::env::join_paths([&a]).unwrap(); + assert!(matches!( + find_browser_with(None, &only_a, &is_snap), + Err(BrowserError::SnapOnly) + )); + // No candidate at all is still NotFound. + let empty = std::env::join_paths([sb.mkdir("empty")]).unwrap(); + assert!(matches!( + find_browser_with(None, &empty, &is_snap), + Err(BrowserError::NotFound) + )); + } + + #[test] + fn an_explicit_snap_browser_is_used_as_given() { + let sb = Sandbox::new("snapexplicit"); + let f = executable(sb.path(), "chromium", "#!/bin/sh\n"); + let always = |_: &Path| true; + assert_eq!( + find_browser_with(Some(&f), OsStr::new(""), &always).unwrap(), + f + ); + } + + #[test] + fn the_snap_only_message_says_what_to_do() { + assert_eq!( + BrowserError::SnapOnly.to_string(), + "the only browser found is a snap package, which cannot use the recorder's profile folder; install Google Chrome or pass --browser " + ); + } + + #[test] + fn a_wrapper_script_that_starts_a_snap_is_a_snap_and_a_plain_script_is_not() { + let sb = Sandbox::new("snapwrap"); + let wrapper = executable( + sb.path(), + "chromium-browser", + "#!/bin/sh\nexec /snap/bin/chromium \"$@\"\n", + ); + let plain = executable( + sb.path(), + "chromium", + "#!/bin/sh\nexec /opt/chrome \"$@\"\n", + ); + assert!(is_snap_browser(&wrapper)); + assert!(!is_snap_browser(&plain)); + } + #[test] fn explicit_missing_browser_is_bad_path() { assert!(matches!( diff --git a/crates/kharon-teach/src/capture/mod.rs b/crates/kharon-teach/src/capture/mod.rs index 5eddf1c..d889945 100644 --- a/crates/kharon-teach/src/capture/mod.rs +++ b/crates/kharon-teach/src/capture/mod.rs @@ -163,14 +163,6 @@ impl Capture { } } - #[cfg(test)] - fn with_body_budget(start_url: &str, tool_version: &str, budget: usize) -> Self { - Self { - body_budget: budget, - ..Self::new(start_url, tool_version) - } - } - pub fn started(&self) -> bool { self.started } @@ -1655,6 +1647,14 @@ mod tests { } impl Capture { + /// A capture with a small body budget, for the test of that budget. + fn with_body_budget(start_url: &str, tool_version: &str, budget: usize) -> Self { + Self { + body_budget: budget, + ..Self::new(start_url, tool_version) + } + } + /// A capture with small step and request limits, for the tests of those limits. fn with_limits(start_url: &str, max_steps: usize, max_requests: usize) -> Self { Self { diff --git a/crates/kharon-teach/src/cli.rs b/crates/kharon-teach/src/cli.rs index e2e9ad3..5a1e88c 100644 --- a/crates/kharon-teach/src/cli.rs +++ b/crates/kharon-teach/src/cli.rs @@ -181,8 +181,26 @@ pub fn check_out_path(p: &Path) -> Result<(), Refusal> { check_dir_writable(p.parent().unwrap_or(Path::new(""))) } +/// `text` from a recording file with every character a terminal could act on replaced by +/// U+FFFD: control characters (escape sequences start with one), the line and paragraph +/// separators, and the bidirectional controls that can make text read differently from how it +/// is stored. A recording sent by someone else must not be able to drive the terminal. +fn plain(text: &str) -> String { + text.chars() + .map(|c| { + if c.is_control() + || matches!(c, '\u{2028}' | '\u{2029}' | '\u{202A}'..='\u{202E}' | '\u{2066}'..='\u{2069}') + { + '\u{FFFD}' + } else { + c + } + }) + .collect() +} + fn site(u: &StrippedUrl) -> String { - format!("{}{}", u.origin, u.path) + plain(&format!("{}{}", u.origin, u.path)) } /// What a recording says, in plain words, one fact per line. It is built from the recording @@ -213,7 +231,8 @@ pub fn summary(r: &Recording) -> String { ), Shape::Transformed { digest } => match digest { Some(d) => format!( - "The page changes the password ({d} digest) before it sends it, so kharon cannot send it as typed." + "The page changes the password ({} digest) before it sends it, so kharon cannot send it as typed.", + plain(d) ), None => "The page changes the password before it sends it, so kharon cannot send it as typed.".to_owned(), }, @@ -260,7 +279,7 @@ pub fn summary(r: &Recording) -> String { } } for n in &r.classification.notes { - out.push(format!("Note: {n}")); + out.push(format!("Note: {}", plain(n))); } if r.listener_world == ListenerWorld::Main { out.push( @@ -476,6 +495,31 @@ pub(crate) mod tests { assert!(quiet.contains("does not look like a candidate")); } + #[test] + fn show_never_prints_a_control_character_from_the_file() { + let mut r = recording(); + r.start_url.path = "/login\u{1b}[31mred".into(); + r.start_url.origin = "https://ex.test\u{7}".into(); + r.classification.notes = + vec!["a note \u{1b}[2J with escapes \u{202E}and bidi\u{2028}".into()]; + r.classification.shape = Shape::Transformed { + digest: Some("sha\u{1b}[0m".into()), + }; + let s = summary(&r); + for c in s.chars().filter(|c| *c != '\n') { + assert!(!c.is_control(), "a control character {:?} was printed", c); + } + for bad in ['\u{202E}', '\u{2028}', '\u{2029}', '\u{2066}', '\u{2069}'] { + assert!(!s.contains(bad), "{bad:?}"); + } + assert!(s.contains("/login\u{FFFD}[31mred")); + assert!(s.contains("Note: a note \u{FFFD}[2J with escapes \u{FFFD}and bidi\u{FFFD}")); + // Ordinary text, including non-ASCII, passes through. + let mut r = recording(); + r.classification.notes = vec!["héllo wörld".into()]; + assert!(summary(&r).contains("Note: héllo wörld")); + } + #[test] fn cli_has_no_headless_or_attach_option() { let names: Vec<_> = Cli::command() diff --git a/crates/kharon-teach/src/lib.rs b/crates/kharon-teach/src/lib.rs index 7e01951..a74cbdb 100644 --- a/crates/kharon-teach/src/lib.rs +++ b/crates/kharon-teach/src/lib.rs @@ -223,6 +223,26 @@ fn harden_process() -> Result<(), String> { Ok(()) } +/// One line from the person, with the reader handed back so the caller can ask again. +async fn read_answer(mut stdin: R) -> (String, std::io::Result, R) { + let mut line = String::new(); + let read = stdin.read_line(&mut line).await; + (line, read, stdin) +} + +/// The answer a line at the end prompt gives: an empty line confirms, `q` or `quit` (any case) +/// discards, anything else is no answer and the prompt is repeated. +fn end_choice(line: &str) -> Option { + let answer = line.trim(); + if answer.is_empty() { + Some(End::Confirmed) + } else if answer.eq_ignore_ascii_case("q") || answer.eq_ignore_ascii_case("quit") { + Some(End::Discard) + } else { + None + } +} + enum Ended { Finished(Result), Interrupted, @@ -289,11 +309,9 @@ pub async fn run_with( )); let mut forward = hooks.ready; let (mut got_ready, mut stdin_done) = (false, false); - let stdin_line = async { - let mut line = String::new(); - let read = stdin.read_line(&mut line).await; - (line, read) - }; + // The reader goes into the future and comes back with the line, so another line can be + // asked for without a second read ever being dropped half way (a read is not cancel safe). + let stdin_line = read_answer(stdin); tokio::pin!(stdin_line); let ended = loop { tokio::select! { @@ -319,18 +337,24 @@ pub async fn run_with( } } } - (line, read) = &mut stdin_line, if got_ready && !stdin_done => { - stdin_done = true; + (line, read, back) = &mut stdin_line, if got_ready && !stdin_done => { // End of input is neither answer: keep recording until the window closes. if matches!(read, Ok(n) if n > 0) { - let how = if line.trim().eq_ignore_ascii_case("q") { - End::Discard - } else { - End::Confirmed - }; - if let Some(tx) = end_tx.take() { - let _ = tx.send(how); + match end_choice(&line) { + Some(how) => { + stdin_done = true; + if let Some(tx) = end_tx.take() { + let _ = tx.send(how); + } + } + None => { + let _ = writeln!(stdout, "{}", notice::END_RETRY); + let _ = stdout.flush(); + stdin_line.set(read_answer(back)); + } } + } else { + stdin_done = true; } } _ = sigint.recv() => break Ended::Interrupted, @@ -387,10 +411,7 @@ pub async fn run_with( Exit::Done } Ok(Outcome::NeverStarted) => { - let _ = writeln!( - stdout, - "The window was closed before the login page opened. Nothing was saved." - ); + let _ = writeln!(stdout, "{}", notice::NEVER_STARTED); Exit::Done } Err(e) => fail(&mut stderr, &e, Exit::Failed), @@ -546,6 +567,29 @@ mod tests { std::fs::remove_dir_all(&d).unwrap(); } + #[test] + fn the_end_prompt_reads_enter_q_and_quit_and_asks_again_otherwise() { + assert!(matches!(end_choice("\n"), Some(End::Confirmed))); + assert!(matches!(end_choice(" \r\n"), Some(End::Confirmed))); + for discard in ["q\n", "Q", " quit ", "QUIT\n", "Quit"] { + assert!( + matches!(end_choice(discard), Some(End::Discard)), + "{discard:?}" + ); + } + for again in ["no\n", "n", "discard", "qq", "y", "quit now", "exit"] { + assert!(end_choice(again).is_none(), "{again:?}"); + } + } + + #[test] + fn the_never_started_message_says_what_happened() { + assert_eq!( + notice::NEVER_STARTED, + "Recording never started: the login page was not opened in the first tab. Nothing was saved." + ); + } + #[test] fn the_write_picks_a_numbered_name_when_the_file_appears() { let d = private_dir("numbered"); diff --git a/crates/kharon-teach/src/main.rs b/crates/kharon-teach/src/main.rs index e34352b..6e59609 100644 --- a/crates/kharon-teach/src/main.rs +++ b/crates/kharon-teach/src/main.rs @@ -2,7 +2,18 @@ use clap::Parser; use kharon_teach::cli::Cli; use std::process::ExitCode; +/// The one line a panic prints: where it happened and nothing from its message. The message of +/// a panic can hold a slice of whatever text the program was handling. +fn panic_line(file: &str, line: u32) -> String { + format!("kharon-teach: internal error at {file}:{line}") +} + fn main() -> ExitCode { + // First of all, so that no panic ever reaches the default hook, which prints its payload. + std::panic::set_hook(Box::new(|info| match info.location() { + Some(l) => eprintln!("{}", panic_line(l.file(), l.line())), + None => eprintln!("kharon-teach: internal error"), + })); let cli = Cli::parse(); let runtime = match tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -32,3 +43,16 @@ fn main() -> ExitCode { runtime.shutdown_background(); exit.into() } + +#[cfg(test)] +mod tests { + use super::panic_line; + + #[test] + fn the_panic_line_names_the_place_and_nothing_else() { + assert_eq!( + panic_line("src/strip.rs", 134), + "kharon-teach: internal error at src/strip.rs:134" + ); + } +} diff --git a/crates/kharon-teach/src/notice.rs b/crates/kharon-teach/src/notice.rs index 6ad5a9b..e0409c4 100644 --- a/crates/kharon-teach/src/notice.rs +++ b/crates/kharon-teach/src/notice.rs @@ -12,6 +12,14 @@ nothing is sent anywhere. The result is one file on this computer.\n"; pub const CONSENT_PROMPT: &str = "Type yes to open the browser window and start: "; /// Printed once the browser window is showing the notice. pub const END_PROMPT: &str = "Press Enter once you are logged in. Type q to discard."; +/// Printed when a line at the end prompt is neither Enter, `q` nor `quit`. +pub const END_RETRY: &str = "Press Enter to save, or type q to discard."; +/// Printed when the session ended before the recording ever started: the login page was not +/// opened in the first tab (the window was closed, or the link was opened in another tab). +pub const NEVER_STARTED: &str = + "Recording never started: the login page was not opened in the first tab. Nothing was saved."; +/// Under the link on the notice page. +pub const OPEN_IN_THIS_TAB: &str = "Open the link in this tab."; /// Printed when consent was not given. pub const NOTHING_RECORDED: &str = "Nothing was recorded."; @@ -41,8 +49,9 @@ pub fn notice_page_html(login_url: &url::Url) -> String { let host = login_url.host_str().unwrap_or("the login page"); format!( "kharon-teach\ -
{notice}

Start recording and open {host}

", +
{notice}

Start recording and open {host}
{tab}

", notice = escape_html(NOTICE), + tab = escape_html(OPEN_IN_THIS_TAB), href = escape_html(login_url.as_str()), host = escape_html(host), ) @@ -71,6 +80,8 @@ mod tests { assert_eq!(html.matches("
Open the link in this tab.

")); } #[test] diff --git a/crates/kharon-teach/tests/req_tch_001_own_browser.rs b/crates/kharon-teach/tests/req_tch_001_own_browser.rs index d8aaf1e..27d5097 100644 --- a/crates/kharon-teach/tests/req_tch_001_own_browser.rs +++ b/crates/kharon-teach/tests/req_tch_001_own_browser.rs @@ -86,24 +86,9 @@ async fn launches_on_its_own_profile_and_binds_loopback_only() { #[test] fn the_library_has_no_way_to_attach_elsewhere() { - fn shipped_files(dir: &Path, out: &mut Vec) { - for e in std::fs::read_dir(dir).unwrap() { - let p = e.unwrap().path(); - if p.is_dir() { - shipped_files(&p, out); - } else if p.extension().is_some_and(|x| x == "rs") { - out.push(p); - } - } - } - let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); - let mut files = Vec::new(); - shipped_files(&src, &mut files); - assert!(!files.is_empty()); let mut launch_src = None; - for f in files { - let text = std::fs::read_to_string(&f).unwrap(); - let shipped = text.split("#[cfg(test)]").next().unwrap(); + for (f, shipped) in support::shipped_sources() { + let shipped = shipped.as_str(); for line in shipped.lines() { for lit in ["ws://", "http://"] { assert!( diff --git a/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs b/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs index 2940b3b..7c81519 100644 --- a/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs +++ b/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs @@ -24,6 +24,57 @@ async fn no_typed_value_in_the_recording_for_any_fixture() { } } +/// The typed values are in text the recording KEEPS: the next page's path and query key, and the +/// key of the JSON body. They must be stripped there, not merely absent from the values. This +/// fails if `strip_url` or the name check returned its input unchanged. +#[tokio::test] +async fn typed_values_in_a_url_and_a_json_key_are_stripped_from_kept_text() { + use kharon_recording::{BodyShape, JsonShape, Role, Step}; + let Some(run) = Run::start("typed_in_text", "/typed").await else { + return; + }; + run.begin("/typed").await; + run.person.type_into("input[name=user]", USER).await; + run.person.type_into("input[name=pass]", PASS).await; + run.person.click("#go").await; + run.person.wait_for_path(&format!("/u/{USER}")).await; + let Outcome::Recorded(r) = run.finish(End::Confirmed).await else { + panic!("expected a recording"); + }; + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); + + // The navigation to /u/?=1 is in the file, with both typed places replaced. + let moved = |url: &kharon_recording::StrippedUrl| { + url.path == "/u/{id}" && url.query_keys == ["{typed}"] + }; + assert!( + r.steps + .iter() + .any(|s| matches!(s, Step::Navigation { url, .. } if moved(url))), + "no navigation step with the typed path replaced" + ); + assert!(r.requests.iter().any(|q| moved(&q.url))); + assert!(r.final_url.as_ref().is_some_and(moved)); + + // The login posts JSON keyed by the user name: the key is {typed} and its hit is not lost. + let login = r + .requests + .iter() + .find(|q| q.url.path == "/api/typed") + .expect("the login request is stored"); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = &login.body + else { + panic!("the body is not a JSON object"); + }; + assert_eq!(fields.len(), 1); + assert_eq!(fields[0].key, "{typed}"); + let hits = login.hits(); + assert!(hits.iter().any(|h| h.role == Role::Password)); + assert!(hits.iter().any(|h| h.role == Role::Username)); +} + #[tokio::test] async fn nothing_typed_reaches_the_file_or_the_output() { let Some(p) = support::run_program("whole_program_no_typed_value", "/form").await else { diff --git a/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs index 8ca1035..1c1fa20 100644 --- a/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs +++ b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs @@ -279,6 +279,34 @@ async fn a_directory_the_program_did_not_make_survives_the_sweep() { } } +#[test] +fn the_source_scan_covers_all_shipped_code() { + // A test-only item above the test module would hide shipped code from every scan. + assert!(support::split_shipped("fn a() {}\n#[cfg(test)]\nmod tests {}\n").is_ok()); + assert!(support::split_shipped("fn a() {}\n#[cfg(test)]\npub(crate) mod tests {}\n").is_ok()); + assert!( + support::split_shipped( + "#[cfg(test)]\nfn helper() {}\nfn b() {}\n#[cfg(test)]\nmod tests {}\n" + ) + .is_err() + ); + assert!(support::split_shipped("fn a() {}\n#[cfg(test)]\nfn helper() {}\n").is_err()); + assert_eq!( + support::split_shipped("fn a() {}\n#[cfg(test)]\nmod tests {}\n").unwrap(), + "fn a() {}\n" + ); + // The real sources: capture's `finish` and `request` are scanned, and no file has a + // test-only item outside its test module (`shipped_sources` panics when one does). + let sources = support::shipped_sources(); + let capture = sources + .iter() + .find(|(p, _)| p.ends_with("capture/mod.rs")) + .expect("capture/mod.rs"); + assert!(capture.1.contains("pub fn finish(")); + assert!(capture.1.contains("fn request(")); + assert!(!capture.1.contains("with_body_budget")); +} + #[test] fn the_shipped_main_never_asks_for_headless() { let sources = support::shipped_sources(); @@ -491,8 +519,9 @@ async fn a_browser_that_quits_by_itself_ends_the_session_and_the_profile_stays_g /// The shipped binary, past `Ready`, in a browser without a display. struct Live { child: std::process::Child, - /// Held open and never written to: end of input must not end the recording. - _stdin: std::process::ChildStdin, + /// Held open and, unless a test writes an answer, never written to: end of input must not + /// end the recording. + stdin: std::process::ChildStdin, stdout: std::sync::Arc>, dir: PathBuf, out: PathBuf, @@ -555,7 +584,7 @@ impl Live { let live = Live { profile: PathBuf::new(), child, - _stdin: stdin, + stdin, stdout, dir, out, @@ -647,6 +676,47 @@ async fn sigquit_after_the_prompt_exits_1_and_leaves_no_browser() { signal_after_the_prompt("QUIT", "sigquit_after_prompt").await; } +#[tokio::test] +async fn a_line_that_is_not_an_answer_asks_again_and_quit_discards() { + use std::io::Write; + let Some(mut live) = Live::start("end_prompt_retry").await else { + return; + }; + for not_an_answer in ["no\n", "discard\n"] { + let before = live + .printed() + .matches("Press Enter to save, or type q to discard.") + .count(); + live.stdin.write_all(not_an_answer.as_bytes()).unwrap(); + live.stdin.flush().unwrap(); + let limit = std::time::Instant::now() + Duration::from_secs(10); + while live + .printed() + .matches("Press Enter to save, or type q to discard.") + .count() + == before + { + assert!( + std::time::Instant::now() < limit, + "no new prompt after {not_an_answer:?}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + // Still recording: nothing was confirmed or discarded. + assert_eq!( + live.exit_within(1).await, + None, + "{not_an_answer:?} ended it" + ); + } + live.stdin.write_all(b"QUIT\n").unwrap(); + live.stdin.flush().unwrap(); + assert_eq!(live.exit_within(10).await, Some(Some(0))); + assert!(live.printed().contains("Discarded. Nothing was saved.")); + assert!(!live.out.exists(), "a file was written"); + assert!(gone(&live.profile).await, "the profile is still there"); +} + #[tokio::test] async fn closing_the_window_after_the_prompt_saves_and_exits_without_enter() { use std::os::unix::fs::PermissionsExt; diff --git a/crates/kharon-teach/tests/support/mod.rs b/crates/kharon-teach/tests/support/mod.rs index 4601d29..7ecf853 100644 --- a/crates/kharon-teach/tests/support/mod.rs +++ b/crates/kharon-teach/tests/support/mod.rs @@ -122,6 +122,17 @@ const POPUP: &str = "popup\ const FRAME_OUTER: &str = "outer\ "; +/// Puts the typed values into text the recording keeps: a JSON key, and the next URL's path and +/// query key. +const TYPED_IN_TEXT: &str = "typed\ +"; + const HOME: &str = "home

signed in

"; fn response(status: &str, extra: &str, body: &str, ctype: &str) -> String { @@ -158,6 +169,9 @@ fn route(method: &str, path: &str, host: &str) -> String { response("200 OK", "", &FRAME_OUTER.replace("{HOST}", &other), html) } ("GET", "/hashed") => response("200 OK", "", HASHED, html), + ("GET", "/typed") => response("200 OK", "", TYPED_IN_TEXT, html), + ("POST", "/api/typed") => response("200 OK", "", "{\"ok\":true}", "application/json"), + ("GET", p) if p.starts_with("/u/") => response("200 OK", "", HOME, html), ("GET", "/home") => response("200 OK", "", HOME, html), _ => response("404 Not Found", "", "not found", "text/plain"), } @@ -961,8 +975,36 @@ pub fn rust_files(dir: &std::path::Path) -> Vec { out } -/// Shipped code only: for each file under `crates/kharon-teach/src`, the text before its first -/// `#[cfg(test)]`. +/// The shipped part of a source file: the text before its test MODULE (`#[cfg(test)]` followed +/// by `mod tests`, with any visibility). A `#[cfg(test)]` item anywhere before it would hide +/// shipped code from every scan, so that is an error. +pub fn split_shipped(text: &str) -> Result<&str, String> { + let mut from = 0; + while let Some(at) = text[from..].find("#[cfg(test)]") { + let at = from + at; + let after = text[at + "#[cfg(test)]".len()..].trim_start(); + let is_module = ["mod tests", "pub mod tests", "pub(crate) mod tests"] + .iter() + .any(|m| after.starts_with(m)); + if is_module { + return if text[..at].contains("#[cfg(test)]") { + Err("a #[cfg(test)] item sits above the test module".to_owned()) + } else { + Ok(&text[..at]) + }; + } + from = at + 1; + } + // No test module: the whole file is shipped, so it must have no test-only item either. + if text.contains("#[cfg(test)]") { + Err("a #[cfg(test)] item with no test module after it".to_owned()) + } else { + Ok(text) + } +} + +/// Shipped code only: for each file under `crates/kharon-teach/src`, the text before its test +/// module (see [`split_shipped`]). Panics when a `#[cfg(test)]` item sits above that module. pub fn shipped_sources() -> Vec<(PathBuf, String)> { let files = rust_files(&std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src")); assert!(!files.is_empty()); @@ -970,7 +1012,10 @@ pub fn shipped_sources() -> Vec<(PathBuf, String)> { .into_iter() .map(|p| { let text = std::fs::read_to_string(&p).unwrap(); - let shipped = text.split("#[cfg(test)]").next().unwrap().to_owned(); + let shipped = match split_shipped(&text) { + Ok(s) => s.to_owned(), + Err(why) => panic!("{}: {why}", p.display()), + }; (p, shipped) }) .collect() From ab5a1d4bf13362b1ab097b15b8ce013b6e56498a Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 07:20:03 -0500 Subject: [PATCH 32/36] docs(spec): bring the REQ-TCH rows and the user page up to the final build (#361) The user page and REQ-TCH-002, -003 and -006 now describe what is built: SIGHUP and SIGQUIT, the sweep rule, a killed recorder leaving its window open, unavailable bodies, names kept only when they look like names, the shape and step caps, the end prompt, the never-started message, snap browsers, the cross_site rule and its after-login note, and the two-step transformed rule. The summary says a form or script sends the password as typed or encoded. The limits that remain are listed under what it cannot do. Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- SPEC.md | 6 ++--- crates/kharon-teach/src/cli.rs | 12 +++++---- docs/teach-mode.md | 48 ++++++++++++++++++++++------------ 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/SPEC.md b/SPEC.md index d333a53..689dce2 100644 --- a/SPEC.md +++ b/SPEC.md @@ -216,11 +216,11 @@ The agent keeps its own CDP session (Playwright/puppeteer `connectOverCDP`). kha | ID | Requirement | AC | |---|---|---| | REQ-TCH-001 | The recorder launches the browser itself, in its own process group, on a profile directory it created (mode `0700`, with an owner file). It has no option to attach to another browser or to name a profile. The browser's debugging port is loopback only. | `crates/kharon-teach/tests/req_tch_001_own_browser.rs`: a launched browser runs with `--user-data-dir` set to the profile the recorder made, and the debugging port listens only on loopback. A scan of the shipped source finds no `ws://` or `http://` address that is not the loopback CDP one, and the fields of `Launch` are `browser`, `first_url`, `headless` and `profile`. | -| REQ-TCH-002 | No typed value is written to the recording file, to any other file of the recorder's own, or to its output. This covers every value typed in the window, not only the user name and the password. A name that contains a typed value is replaced with `{typed}` (a field or part name, a host label, a content type, a scheme). A path segment that contains one is replaced with `{id}`. An element attribute or selector that holds one is left out. | `crates/kharon-teach/tests/req_tch_002_no_typed_value.rs`: for each of the four fixture logins, a byte search of the recording for the user name and the password, in every encoding the recorder knows, finds nothing. The same search over the whole program's file, standard output and standard error finds nothing. A page that copies a typed value into an attribute, and the listener script's own reports, carry none of it. | -| REQ-TCH-003 | The file holds no cookie, no request or response header value other than the content type and the `Authorization` scheme (the browser's own name and user-agent string are recorded, as they are not header values the page sent), no response body, no query value and no request-body value. A URL scheme and an `Authorization` scheme are kept only from a known list. The content of a `data:` URL is never kept. | `crates/kharon-teach/tests/req_tch_003_file_contents.rs`: the file is mode `0600`, reads back as a recording, keeps a query key and not its value, and holds no cookie, no response text, no page title and none of eight named headers; the response's `Set-Cookie` shows only as a flag. The scheme lists and the `data:` rule are checked by unit tests in `crates/kharon-teach/src/strip.rs`. | +| REQ-TCH-002 | No typed value is written to the recording file, to any other file of the recorder's own, or to its output. This covers every value typed in the window, not only the user name and the password. A name that contains a typed value is replaced with `{typed}` (a field or part name, a JSON key, a host label, a content type, a scheme), and the hits found in it stay on the field. A path segment that contains one is replaced with `{id}`. An element attribute or selector that holds one is left out. A value shorter than six characters is removed from kept text where it stands as a whole word, in any of its encodings. | `crates/kharon-teach/tests/req_tch_002_no_typed_value.rs`: for each of the four fixture logins, a byte search of the recording for the user name and the password, in every encoding the recorder knows, finds nothing. The same search over the whole program's file, standard output and standard error finds nothing. A page that copies a typed value into an attribute, and the listener script's own reports, carry none of it. A fifth fixture puts the typed values into a path, a query key and a JSON key: the file holds none of them, the path is `/u/{id}`, the query key and the JSON key are `{typed}`, and the hits in the key are kept. The whole-string rule, the encoded forms of short values and the name rules are checked by unit tests in `crates/kharon-teach/src/strip.rs`, `body.rs` and `marker.rs`. | +| REQ-TCH-003 | The file holds no cookie, no request or response header value other than the content type and the `Authorization` scheme (the browser's own name and user-agent string are recorded, as they are not header values the page sent), no response body, no query value and no request-body value. A field name, part name, JSON key or query key is kept only when it looks like a name (at most 64 characters of letters, digits and `_ . : - [ ]`, and not an identifier); otherwise it is written `{other}`. A body that is not what its content type says, or that would hold more than 2,000 values, is kept as a length only. A body over 1 MiB is `unavailable`. A URL scheme and an `Authorization` scheme are kept only from a known list. The content of a `data:` URL is never kept. | `crates/kharon-teach/tests/req_tch_003_file_contents.rs`: the file is mode `0600`, reads back as a recording, keeps a query key and not its value, and holds no cookie, no response text, no page title and none of eight named headers; the response's `Set-Cookie` shows only as a flag. The scheme lists and the `data:` rule are checked by unit tests in `crates/kharon-teach/src/strip.rs`. The name rule, the 2,000-value cap and the token-shaped bodies, keys and queries are checked by unit tests in `strip.rs` and `crates/kharon-teach/src/body.rs`. | | REQ-TCH-004 | The recorder holds no broker credential, listens on no socket, and opens no network connection except loopback CDP to the browser it launched. | `crates/kharon-teach/tests/req_tch_004_no_network.rs`: the shipped source has no listener or bind call. The dependency tree has no TLS or HTTP client crate, no `kharond` and no `kharon-connector`. The only dial is in `cdp.rs`, to a `ws://127.0.0.1:` address. | | REQ-TCH-005 | The four fixture logins classify as `form_post`, `script_post`, `two_step` and `transformed`. | `crates/kharon-teach/tests/req_tch_005_shapes.rs`: a real Chromium logs in to each of the four fixture pages, and each recording carries the shape named for it and the rung ceiling `milliseconds` (the form, script and hashed pages) or `agent_continues` (the two-step page). Further cases cover a multipart body, a blob body, a pop-up and a login in a frame. | -| REQ-TCH-006 | The browser profile is deleted on every exit the recorder can see (confirm, discard, window closed, browser killed, SIGINT, SIGTERM). A leftover profile is deleted at the next start unless its owner is still running. The browser is always visible: no flag and no environment variable changes that. | `crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs`: the profile is gone after each of those exits, and a leftover is removed at the next start. After a login the browser's password and autofill tables are empty. A scan of the shipped source finds no way to ask for headless, and `PATH` is the only environment variable read. The owner-still-running rule is checked by unit tests in `crates/kharon-teach/src/browser.rs`. | +| REQ-TCH-006 | The browser and its profile are removed on every exit the recorder can see (confirm, discard, window closed, browser killed, SIGINT, SIGTERM, SIGHUP, SIGQUIT). A recorder killed without that chance (SIGKILL, a crash) leaves its browser window open; the profile is then removed at a later start, once that window is closed. A start removes a leftover directory only when the recorder made it: its name is `kharon-teach-` and 20 letters and digits, it is a real directory owned by the same user, its owner file names a process that is not running, and no browser is running on it. Any other directory is left alone. The browser is always visible: no flag and no environment variable changes that. | `crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs`: the profile is gone after each of those exits, and no process is left with the profile on its command line after SIGINT, SIGTERM, SIGHUP and SIGQUIT, both before the browser is ready and after the end prompt. A leftover with a dead owner is removed at the next start, and directories without an owner file, with a junk owner file or with another name shape are not. After a login the browser's password and autofill tables are empty. A scan of the shipped source finds no way to ask for headless, and `PATH` is the only environment variable read. The sweep rule (name shape, owner file, live owner, live browser, symlinks) and the clearing of the browser's process group id are checked by unit tests in `crates/kharon-teach/src/browser.rs`. | ## 4. Agent-facing API diff --git a/crates/kharon-teach/src/cli.rs b/crates/kharon-teach/src/cli.rs index 5a1e88c..a49c89c 100644 --- a/crates/kharon-teach/src/cli.rs +++ b/crates/kharon-teach/src/cli.rs @@ -218,14 +218,14 @@ pub fn summary(r: &Recording) -> String { out.push(format!( "How the login works: {}", match &r.classification.shape { - Shape::FormPost => "One page; the form sends the password to the site as typed.".to_owned(), + Shape::FormPost => "One page; the form sends the password to the site as typed or encoded.".to_owned(), Shape::ScriptPost => - "One page; the page's script sends the password to the site as typed.".to_owned(), + "One page; the page's script sends the password to the site as typed or encoded.".to_owned(), Shape::TwoStep { password_leg } => format!( "Two steps; the user name goes first, then the password, which {}.", match password_leg { - PasswordLeg::FormPost => "a form sends to the site as typed", - PasswordLeg::ScriptPost => "the page's script sends to the site as typed", + PasswordLeg::FormPost => "a form sends to the site as typed or encoded", + PasswordLeg::ScriptPost => "the page's script sends to the site as typed or encoded", PasswordLeg::Transformed => "the page changes before it sends it", } ), @@ -477,7 +477,9 @@ pub(crate) mod tests { r.listener_world = ListenerWorld::Main; let s = summary(&r); assert!(s.contains("https://ex.test/login")); - assert!(s.contains("One page; the form sends the password to the site as typed.")); + assert!( + s.contains("One page; the form sends the password to the site as typed or encoded.") + ); assert!(s.contains("password is in the page for milliseconds")); assert!(s.contains("A captcha")); assert!(s.contains("Network swap: this login looks like")); diff --git a/docs/teach-mode.md b/docs/teach-mode.md index 4c3b973..33753f0 100644 --- a/docs/teach-mode.md +++ b/docs/teach-mode.md @@ -12,6 +12,8 @@ Use a throwaway account. Never use a real one. The program reads everything you Linux only for now. A Chromium-family browser must be installed: Chromium, Google Chrome, Microsoft Edge or Brave. The program looks for them on your `PATH` in that order, or you can name one with `--browser`. +A browser installed as a snap package cannot use the folder the program makes for its profile, because a snap gets its own private temporary folder. On Ubuntu the default Chromium is a snap. The program skips a snap browser and tries the next one. If a snap is the only browser it finds, it stops and prints "the only browser found is a snap package, which cannot use the recorder's profile folder; install Google Chrome or pass --browser ". A browser you name with `--browser` is used as you gave it. + The program always opens a visible window. There is no way to turn that off. ## Recording a login @@ -36,7 +38,7 @@ Type yes to open the browser window and start: Type `yes` and press Enter. Anything else ends the program with "Nothing was recorded." and opens nothing. -A browser window opens on a new, empty profile. Its first page repeats the notice and has one link, "Start recording and open" followed by the site's name. Recording starts when you click it. Nothing before that click is recorded. +A browser window opens on a new, empty profile. Its first page repeats the notice and has one link, "Start recording and open" followed by the site's name. Under the link it says "Open the link in this tab." Recording starts when you click it in that tab. Nothing before that click is recorded. Opening the link in a new tab, or typing an address in another tab, does not start the recording. The terminal then prints: @@ -47,14 +49,18 @@ Press Enter once you are logged in. Type q to discard. Log in by hand in the window. When you are signed in, go back to the terminal. A recording can end in three ways: - Press Enter. The recording is saved and marked as a login that worked. -- Type `q` and press Enter. The recording is thrown away and nothing is saved. +- Type `q` or `quit` (in any letter case) and press Enter. The recording is thrown away and nothing is saved. - Close the browser window. The recording is saved and marked as a login that was not confirmed. If you close the window before you click the link, nothing is saved. -If you press Enter before you click the link, nothing is saved, and the program prints "The window was closed before the login page opened. Nothing was saved." even though the window is still open. +Any other line you type is not an answer. The program prints "Press Enter to save, or type q to discard." and waits for another line. If the terminal's input ends without a line (it is closed or piped from a file that has run out), the program treats that as neither answer and keeps recording until the window closes. + +If the session ends before the recording started, because you pressed Enter before you clicked the link, closed the window first, or opened the link in another tab, nothing is saved. The program prints "Recording never started: the login page was not opened in the first tab. Nothing was saved." -Ctrl-C, or a stop signal, ends the program with "Interrupted. Nothing was saved." and a failure exit code. This line and every refusal go to the error output, each starting with "kharon-teach: ". +Ctrl-C, a stop request (SIGTERM), closing the terminal window (SIGHUP) or Ctrl-\ (SIGQUIT) ends the program with "Interrupted. Nothing was saved." and a failure exit code. This line and every refusal go to the error output, each starting with "kharon-teach: ". -On every one of these, the program closes the browser and deletes the profile it made. If an earlier run was killed and left a profile behind, the next run deletes it and says "Removed 1 leftover browser profile(s) from an earlier run." before the Enter prompt. +On every one of these, the program closes the browser and deletes the profile it made. The browser runs in its own process group, so that closing it also closes its helper processes. + +If the program itself is killed without a chance to react (SIGKILL, a crash, a power cut), it cannot clean up. Then the browser window stays open, still holding the throwaway session, until you close it. The profile folder is deleted by a later run, and only after that window is closed. A later run deletes a folder only if the program made it: its name is `kharon-teach-` followed by exactly 20 letters and digits, it holds the owner file the program writes with its own process number, that process is no longer running, and no browser is running on it. Any other folder, even one named `kharon-teach-something`, is left alone. When it deletes one, it says "Removed 1 leftover browser profile(s) from an earlier run." before the Enter prompt. When a recording is saved, the program prints a summary of it and then "Saved:" and the path. The file is named after the site and the time, for example `login.example.com-20261118T061500Z.kharon-recording.json`, and is written in the current directory. Only you can read it (mode `0600`). If a file of that name already exists, the program adds `.1`, then `.2`, up to `.9`, and does not overwrite anything. For a generated name the number goes before `.kharon-recording.json`. For a name you gave with `--out` it goes at the end. @@ -81,22 +87,25 @@ The file describes the structure of the login. It is one JSON document (a plain- | What the browser showed | What the file keeps | |---|---| | What you typed | The role (user name or password) and the length. Every field you typed in counts, not only those two. | -| Request bodies | The shape: the names of the fields, and for each value its kind, its length and, where a typed value was found, its role and how it was encoded. Never the value. A body the program cannot read, or one over 1 MiB, is kept as a length only. | +| Request bodies | The shape: the names of the fields, and for each value its kind, its length and, where a typed value was found, its role and how it was encoded. Never the value. A body the program cannot read as the kind its content type says, and a body that would make a shape of more than 2,000 values, is kept as a length only, with the role and encoding of any typed value found in it. A body over 1 MiB is not read at all and is marked `unavailable`: the file says only that its body could not be read. | | Query strings (`?a=b`) | The names only. | +| Names that do not look like names | A field, part or query name is kept only when it looks like a name: at most 64 characters, all of them letters, digits or one of `_ . : - [ ]`, and not all digits, not a UUID and not 16 or more hex characters. A query piece with no `=` must also be at most 32 characters. Anything else is written `{other}`, because it is more likely a value than a name. A urlencoded body with a piece that has no `=`, or in which most names do not look like names, is not treated as fields at all; it is kept as a length only. | | Paths (`/a/b`) | The path, with a part replaced by `{id}` when it is all digits, looks like a UUID, is 16 or more hex characters, or contains something you typed. | -| Names that contained something you typed | Replaced by `{typed}`. This covers the names of fields and parts, the labels of a host name and the content type. | +| Names that contained something you typed | Replaced by `{typed}`. This covers the names of fields and parts, the labels of a host name and the content type. A typed value found in a name is still recorded as a hit on that field, so the login is read correctly. | | Web addresses | The scheme is kept only if it is a common one (http, https, ws, wss, data, blob, about, file, chrome, chrome-extension, devtools). Any other is written `{other}`. The content of a `data:` address is never kept. | | Headers | The content type only. For `Authorization`, only the scheme (for example Basic or Bearer), and only from a known list; any other scheme is written `{other}`. Whether a typed value was in it is recorded. | | Cookies | Whether a response set one. Nothing else. | | Responses | For each stored request, the status number (for example 200) and the content type of the reply. | | Response bodies | Never read. | -| Page text, titles and HTML | Never read. Only the attributes of the fields you used are kept: tag, type, `id`, `name`, `autocomplete`, placeholder, aria-label, the form it is in, and a list of ways to find it again. An attribute or way of finding a field that held something you typed is left out. | +| Page text, titles and HTML | Never read. Only the attributes of the fields you used, and of the buttons and links you clicked, are kept: tag, type, `id`, `name`, `autocomplete`, placeholder, aria-label, the form it is in, and a list of ways to find it again. An attribute or way of finding an element that held something you typed is left out. | + +Only three kinds of request are stored: pages the main window moved to, requests in which a typed value was found, and any request other than GET to the login site after your first keystroke. All other requests are counted and dropped. A typed value of fewer than six characters can match by chance, so such a match decides nothing when a longer, surer match of the same kind (user name or password) exists in the recording. -Only three kinds of request are stored: pages the main window moved to, requests in which a typed value was found, and any request other than GET to the login site after your first keystroke. All other requests are counted and dropped. +The program holds at most 5,000 requests and keeps at most 10,000 steps. Requests past the first 5,000 are counted and not looked at. Steps past 10,000 are not written, but they are counted, and what you typed in them is still found and removed. The file also records the program version, the browser's name and user agent (the line a browser sends to say what it is), the window size, the start and end times, how the recording ended, and the steps you took: pages visited, fields typed in (by length), clicks, form submissions and Enter presses. -At the end of the file, a `stripped` block counts what was removed: how many requests the browser made and how many were kept, how many typed values, query values and path parts were dropped, and how many bodies were reduced to a shape. The summary prints the same counts. +At the end of the file, a `stripped` block counts what was removed: how many requests the browser made and how many were kept, how many typed values, query values and path parts were dropped, how many bodies were reduced to a shape, and how many steps were dropped for being past the limit. The summary prints the same counts, and a line "Not recorded: N step(s) past the recorder's limit." when any steps were dropped. ## Reading the result @@ -106,8 +115,8 @@ Shape of the login ("How the login works"): | Shape | Meaning | |---|---| -| `form_post` | One page. The form sends the password to the site as typed. | -| `script_post` | One page. The page's script sends the password to the site as typed. | +| `form_post` | One page. The form sends the password to the site as typed or encoded. Encoded means changed back and forth without losing it, for example percent-encoded or base64. | +| `script_post` | One page. The page's script sends the password to the site as typed or encoded. | | `two_step` | The user name goes first, then the password on a later step. The summary says how the password was sent: by a form, by a script, or changed by the page first. | | `transformed` | The page changes the password before sending it, so no request holds it as typed. If the change was a known digest, the summary names it. A digest is a fixed-length scramble of the password that cannot be turned back (md5, sha1, sha256 or sha512). | | `unrecognized` | None of the above. This also covers a recording in which no password field was used, one in which the password was sent in a way the program does not know, one in which request bodies were unavailable and the password was found nowhere, and one with a flag below that needs a person to look. | @@ -118,7 +127,7 @@ Strongest protection ("Strongest protection"), the best that kharon could do for |---|---| | `milliseconds` | kharon could fill and submit in one step, so the password is in the page for milliseconds. | | `agent_continues` | The agent has to continue after the fill. This is the case for two-step logins, logins with a captcha or a one-time-code step, and one-page logins where the password field was not in a form and no Enter key and no click on a button followed it. | -| `needs_person` | A person has to look at the recording. | +| `needs_person` | A person has to look at the recording. This includes a two-step login whose password step is changed by the page, unless the change is a digest the program can name. With a named digest, the result is `agent_continues` and a note names the digest. | "Network swap" says whether the password was found, as typed or encoded but not hashed, in exactly one stored request. Such a login might let kharon put the password in as the request leaves the browser, so the page never holds it. kharon cannot do this yet. The line records which logins might allow it. @@ -128,7 +137,7 @@ Flags, each printed as "Flag:" with a sentence: |---|---| | `captcha` | A captcha was on the page. The program knows five captcha hosts, plus Google's. | | `otp_step` | There was a field for a one-time code after the password. | -| `cross_site` | The login went through another site, as single sign-on does. | +| `cross_site` | The browser moved to another web site before the password was sent, as single sign-on does. Only a move to another web site (an `http` or `https` page of another registrable domain) counts; a browser error page does not. A move after the password was sent is not a flag. A login that lands on another site afterwards is still read normally, and the summary adds the note "After the login the browser went on to another site." | | `popup` | The login opened another window. | | `framed_login` | A login field was inside a frame from another web address. | | `shadow_dom` | A login field was inside a shadow root, a part of the page that hides its fields from ordinary lookups. | @@ -150,12 +159,19 @@ Before you do, run `kharon-teach show ` and read what it says. The `stripp - It records in a visible desktop browser. The agent's browser is usually headless, and a site may lay itself out differently there. The file records the window size and the user agent so a mismatch can be seen later. - A site with bot detection may treat a fresh profile with a debugging port differently from your everyday browser. - If you type a real password despite the notice, the program still writes it nowhere, but it read it, and the browser held it until the profile was deleted. -- If the program is killed or the computer loses power, the profile stays on disk until the next start deletes it. +- If the program is killed without a chance to react (SIGKILL, a crash, a power cut), the browser window stays open with the throwaway session in it. Its profile is deleted by a later run, only after that window is closed. +- While a recording runs, the browser's debugging port listens on 127.0.0.1. Any program on the computer, including another user's program on a shared machine, can connect to it and read what is typed. Record only on a computer you do not share. +- A browser installed as a snap package is skipped, because it cannot use the profile folder (see "Before you start"). This has not been tried on an Ubuntu machine yet. If the program fails to start the browser there, pass `--browser` with Google Chrome or a Chromium that is not a snap. - It does not detect passkey logins. - It does not follow web workers or service workers. - It finds a typed value in a request only in forms it knows: as typed, percent-encoded, escaped for JSON, base64, base64url and hex, and for the password the md5, sha1, sha256 and sha512 digests in hex or base64. A site that lowercases or re-encodes a typed value before sending it may not be recognised as the user name. - A typed value that a site sends as a JSON number is looked for as its text. If the site writes the number differently from how it was typed, it is not found. -- A typed value shorter than six characters is searched for, but a match is marked low confidence, because it may be chance. +- A typed value shorter than six characters, such as a PIN, is searched for, but a match is marked low confidence, because it may be chance. In text the program keeps (paths, query names, hosts) it is removed only where it stands as a whole word, so that it does not damage ordinary words. In requests its matches are weak hints, and they can point at the wrong request when the same few characters appear by chance in another one. +- If you mistype the password and the page reloads before you get it right, the recording reads as a sign-up form (two password values) and needs a person. Start the recording again instead. +- Logins the page's script cannot see are not recognised: the browser's own Basic or Digest login dialog, passkeys, and fields drawn on a canvas or typed into a `contenteditable` element. +- A captcha script that only loads on the page (reCAPTCHA v3, Cloudflare Turnstile) sets the `captcha` flag and lowers the result to `agent_continues`, even when no challenge was shown. +- A password sent over a WebSocket, through a service worker or from another tab is not found, and neither is a change outside the encodings and digests listed above. Such a login comes out as `transformed` or `unrecognized`. +- Opening the link on the notice page in a new tab, or typing another address, means the recording never starts. The page says to open the link in the same tab. - Once 32 MiB of request bodies are held, later bodies are recorded as unavailable. - A `transformed` login with no named digest is mostly a site that encrypts the password in the page with a key the site supplies, as banks often do. It is an expected result and is marked `needs_person`. - A recording made with the listener in the page's own script world, not the isolated one, is marked as such and is less trustworthy. From 3c14c673e8de9e4150a10120d01ac2b66fb41fe2 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Wed, 18 Nov 2026 07:43:07 -0500 Subject: [PATCH 33/36] fix(teach): give the path dependency a version, as cargo deny requires deny.toml forbids wildcard dependencies, and a path dependency with no version counts as one. The other crates in the workspace already write the version next to the path. Refs #361 Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) --- crates/kharon-teach/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/kharon-teach/Cargo.toml b/crates/kharon-teach/Cargo.toml index 89f16a5..660344e 100644 --- a/crates/kharon-teach/Cargo.toml +++ b/crates/kharon-teach/Cargo.toml @@ -16,7 +16,7 @@ name = "kharon-teach" path = "src/main.rs" [dependencies] -kharon-recording = { path = "../kharon-recording" } +kharon-recording = { version = "0.0.0", path = "../kharon-recording" } zeroize = { version = "1", features = ["serde"] } sha1 = "0.10" sha2 = "0.10" From 1cb3f7bc8a020cd26d32c69397ae7941010e20a6 Mon Sep 17 00:00:00 2001 From: "cycle-five-duet[bot]" <352198809+cycle-five-duet[bot]@users.noreply.github.com> Date: Thu, 19 Nov 2026 14:41:39 +0000 Subject: [PATCH 34/36] fix(teach): encoded host labels, a real CSS unescape, cross_site wording (#364) Nyx's lead calls on #364, items 2 to 4. - strip_host now runs kept_hit on each label, as strip_path does for segments, so a label that is the hex of a typed value (or holds the combined credential) becomes {typed}. Base64 cannot survive the parser's lowercasing; hex is the real case. - selector_is_clean undoes CSS escapes properly (CSS Syntax 3, 4.3.7) instead of deleting every backslash, so `#\39 x...` is read as `#9x...`. - docs/teach-mode.md: cross_site is set when the first move to another site came before the LAST request that carried the password, which is what the code does. Tests: a hex host label, hex escapes (short and six-digit), other escaped chars, U+FFFD for 0, a surrogate and past U+10FFFF, a trailing backslash, and two selectors that hold a typed value. Co-Authored-By: Claude Opus 5.5 --- crates/kharon-teach/src/capture/build.rs | 116 ++++++++++++++++++++++- crates/kharon-teach/src/strip.rs | 26 ++++- docs/teach-mode.md | 2 +- 3 files changed, 136 insertions(+), 8 deletions(-) diff --git a/crates/kharon-teach/src/capture/build.rs b/crates/kharon-teach/src/capture/build.rs index 101b5a0..2edf16a 100644 --- a/crates/kharon-teach/src/capture/build.rs +++ b/crates/kharon-teach/src/capture/build.rs @@ -9,6 +9,7 @@ use super::{Buffered, FrameInfo, RawBody, RawElement, RawForm}; use crate::body::shape_of_body; use crate::marker::Marker; use crate::strip::{TYPED, dedup_hits, hits_in, holds_typed, reduce_authorization, strip_url}; +use zeroize::Zeroizing; /// What is known while stripping: every typed value, and the credential pair once chosen. #[derive(Clone, Copy)] @@ -80,10 +81,55 @@ fn clean(attr: Option<&String>, ctx: Ctx<'_>) -> Option { attr.filter(|a| !ctx.holds_typed(a)).cloned() } -/// A selector holding a typed value, also once CSS escapes (`\.`) are undone, is removed. +/// `s` with CSS escapes undone (CSS Syntax 3, 4.3.7): `\` + 1-6 hex digits (+ one optional +/// whitespace) is that code point (0, a surrogate or past U+10FFFF is U+FFFD); `\` + any other +/// char is that char. A `\` before a newline or at the end is dropped. +fn css_unescape(s: &str) -> Zeroizing { + let mut out = Zeroizing::new(String::with_capacity(s.len())); + let mut it = s.chars().peekable(); + while let Some(c) = it.next() { + if c != '\\' { + out.push(c); + continue; + } + let mut hex = Zeroizing::new(String::new()); + while hex.len() < 6 { + match it.peek() { + Some(h) if h.is_ascii_hexdigit() => { + hex.push(*h); + it.next(); + } + _ => break, + } + } + if hex.is_empty() { + match it.next() { + Some('\n') | None => {} + Some(o) => out.push(o), + } + continue; + } + let cp = u32::from_str_radix(&hex, 16).unwrap_or(0); + out.push(char::from_u32(cp).filter(|_| cp != 0).unwrap_or('\u{FFFD}')); + match it.peek() { + Some('\r') => { + it.next(); + if it.peek() == Some(&'\n') { + it.next(); + } + } + Some(' ' | '\t' | '\n' | '\u{c}') => { + it.next(); + } + _ => {} + } + } + out +} + +/// A selector holding a typed value, also once CSS escapes are undone, is removed. fn selector_is_clean(selector: &str, ctx: Ctx<'_>) -> bool { - let unescaped: String = selector.chars().filter(|c| *c != '\\').collect(); - !ctx.holds_typed(selector) && !ctx.holds_typed(&unescaped) + !ctx.holds_typed(selector) && !ctx.holds_typed(&css_unescape(selector)) } const INPUT_TYPES: [&str; 22] = [ @@ -257,3 +303,67 @@ pub(super) fn reduce_request(b: &Buffered, ctx: Ctx<'_>) -> Option<(StoredReques }; Some((stored, tally, reduced_body)) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::pw; + + fn un(s: &str) -> String { + css_unescape(s).to_string() + } + + #[test] + fn a_hex_escape_is_its_code_point() { + assert_eq!(un(r"#\39 x"), "#9x"); + assert_eq!(un(r"#\000039x"), "#9x"); + } + + #[test] + fn any_other_escaped_char_is_itself() { + assert_eq!(un(r"#a\.b"), "#a.b"); + assert_eq!(un(r"#a\\b"), r"#a\b"); + } + + #[test] + fn zero_a_surrogate_or_past_the_last_code_point_is_the_replacement_char() { + assert_eq!(un(r"#\0 x"), "#\u{FFFD}x"); + assert_eq!(un(r"#\D800 x"), "#\u{FFFD}x"); + assert_eq!(un(r"#\110000 x"), "#\u{FFFD}x"); + } + + #[test] + fn a_trailing_backslash_is_dropped() { + assert_eq!(un(r"#a\"), "#a"); + } + + #[test] + fn a_selector_with_a_hex_escaped_typed_value_is_left_out() { + let m = [pw("9xTr0ub4dor")]; + let ctx = Ctx { + markers: &m, + pair: None, + }; + assert!(!selector_is_clean(r"#\39 xTr0ub4dor", ctx)); + } + + #[test] + fn a_selector_with_an_escaped_backslash_in_a_typed_value_is_left_out() { + let m = [pw(r"Tr0ub\4dor")]; + let ctx = Ctx { + markers: &m, + pair: None, + }; + assert!(!selector_is_clean(r"#Tr0ub\\4dor", ctx)); + } + + #[test] + fn a_selector_with_no_typed_value_is_kept() { + let m = [pw("9xTr0ub4dor")]; + let ctx = Ctx { + markers: &m, + pair: None, + }; + assert!(selector_is_clean(r"#login\.form > input:nth-child(2)", ctx)); + } +} diff --git a/crates/kharon-teach/src/strip.rs b/crates/kharon-teach/src/strip.rs index 2263091..6bb0144 100644 --- a/crates/kharon-teach/src/strip.rs +++ b/crates/kharon-teach/src/strip.rs @@ -316,7 +316,7 @@ pub fn strip_url( let port = parsed.port().map(|p| format!(":{p}")).unwrap_or_default(); format!( "{scheme}://{}{port}", - strip_host(host, markers, &host_ranges) + strip_host(host, markers, pair, &host_ranges) ) } None => format!("{scheme}:"), @@ -354,8 +354,15 @@ pub fn strip_url( /// (ASCII case-insensitively: the parser lowercases hosts) and every label an occurrence touches /// is replaced; `extra` holds matches found over the whole URL, in host coordinates. Adjacent /// replaced labels become ONE `{typed}` label, so a value that spans labels (`kh.throwaway7`) -/// leaves no part of itself behind. -fn strip_host(host: &str, markers: &[Marker], extra: &[Range]) -> String { +/// leaves no part of itself behind. A label that holds a typed value in an encoded form (hex, +/// say; base64 cannot survive the lowercasing), or the combined credential, is replaced too, by +/// `kept_hit`, the same as a path segment. +fn strip_host( + host: &str, + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, + extra: &[Range], +) -> String { let mut ranges = occurrences(host, markers); ranges.extend_from_slice(extra); let mut spans = Vec::new(); @@ -364,7 +371,10 @@ fn strip_host(host: &str, markers: &[Marker], extra: &[Range]) -> String spans.push(offset..offset + label.len()); offset += label.len() + 1; } - let typed = touched(&ranges, &spans); + let mut typed = touched(&ranges, &spans); + for (t, label) in typed.iter_mut().zip(host.split('.')) { + *t |= kept_hit(label.as_bytes(), markers, pair); + } let mut labels: Vec<&str> = Vec::new(); let mut previous_typed = false; for (label, is_typed) in host.split('.').zip(typed) { @@ -748,6 +758,14 @@ mod tests { assert_eq!(s.value.query_keys, ["{typed}"]); } + #[test] + fn a_host_label_that_is_an_encoded_typed_value_is_replaced() { + let m = [user("kh.throwaway7")]; + let label = hex::encode("kh.throwaway7"); + let s = strip_url(&format!("https://{label}.ex.test/login"), &m, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + } + #[test] fn a_host_label_that_holds_a_typed_value_is_replaced() { let m = [user("kh.throwaway7")]; diff --git a/docs/teach-mode.md b/docs/teach-mode.md index 33753f0..9debebf 100644 --- a/docs/teach-mode.md +++ b/docs/teach-mode.md @@ -137,7 +137,7 @@ Flags, each printed as "Flag:" with a sentence: |---|---| | `captcha` | A captcha was on the page. The program knows five captcha hosts, plus Google's. | | `otp_step` | There was a field for a one-time code after the password. | -| `cross_site` | The browser moved to another web site before the password was sent, as single sign-on does. Only a move to another web site (an `http` or `https` page of another registrable domain) counts; a browser error page does not. A move after the password was sent is not a flag. A login that lands on another site afterwards is still read normally, and the summary adds the note "After the login the browser went on to another site." | +| `cross_site` | The browser moved to another web site before the password was last sent, as single sign-on does. Only a move to another web site (an `http` or `https` page of another registrable domain) counts; a browser error page does not. A move after the last request that carried the password is not a flag. A login that lands on another site afterwards is still read normally, and the summary adds the note "After the login the browser went on to another site." | | `popup` | The login opened another window. | | `framed_login` | A login field was inside a frame from another web address. | | `shadow_dom` | A login field was inside a shadow root, a part of the page that hides its fields from ordinary lookups. | From 61d9ecb26df3ea6624f03629899f35d304aabca5 Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Fri, 20 Nov 2026 07:01:48 -0500 Subject: [PATCH 35/36] docs(spec): REQ-TCH-003 says opaque bodies keep length and markers (#364) Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) Claude-Session: https://claude.ai/code/session_017zqkE9yPuSSE3mYPnw3NAp --- SPEC.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPEC.md b/SPEC.md index 689dce2..98b3ffd 100644 --- a/SPEC.md +++ b/SPEC.md @@ -217,7 +217,7 @@ The agent keeps its own CDP session (Playwright/puppeteer `connectOverCDP`). kha |---|---|---| | REQ-TCH-001 | The recorder launches the browser itself, in its own process group, on a profile directory it created (mode `0700`, with an owner file). It has no option to attach to another browser or to name a profile. The browser's debugging port is loopback only. | `crates/kharon-teach/tests/req_tch_001_own_browser.rs`: a launched browser runs with `--user-data-dir` set to the profile the recorder made, and the debugging port listens only on loopback. A scan of the shipped source finds no `ws://` or `http://` address that is not the loopback CDP one, and the fields of `Launch` are `browser`, `first_url`, `headless` and `profile`. | | REQ-TCH-002 | No typed value is written to the recording file, to any other file of the recorder's own, or to its output. This covers every value typed in the window, not only the user name and the password. A name that contains a typed value is replaced with `{typed}` (a field or part name, a JSON key, a host label, a content type, a scheme), and the hits found in it stay on the field. A path segment that contains one is replaced with `{id}`. An element attribute or selector that holds one is left out. A value shorter than six characters is removed from kept text where it stands as a whole word, in any of its encodings. | `crates/kharon-teach/tests/req_tch_002_no_typed_value.rs`: for each of the four fixture logins, a byte search of the recording for the user name and the password, in every encoding the recorder knows, finds nothing. The same search over the whole program's file, standard output and standard error finds nothing. A page that copies a typed value into an attribute, and the listener script's own reports, carry none of it. A fifth fixture puts the typed values into a path, a query key and a JSON key: the file holds none of them, the path is `/u/{id}`, the query key and the JSON key are `{typed}`, and the hits in the key are kept. The whole-string rule, the encoded forms of short values and the name rules are checked by unit tests in `crates/kharon-teach/src/strip.rs`, `body.rs` and `marker.rs`. | -| REQ-TCH-003 | The file holds no cookie, no request or response header value other than the content type and the `Authorization` scheme (the browser's own name and user-agent string are recorded, as they are not header values the page sent), no response body, no query value and no request-body value. A field name, part name, JSON key or query key is kept only when it looks like a name (at most 64 characters of letters, digits and `_ . : - [ ]`, and not an identifier); otherwise it is written `{other}`. A body that is not what its content type says, or that would hold more than 2,000 values, is kept as a length only. A body over 1 MiB is `unavailable`. A URL scheme and an `Authorization` scheme are kept only from a known list. The content of a `data:` URL is never kept. | `crates/kharon-teach/tests/req_tch_003_file_contents.rs`: the file is mode `0600`, reads back as a recording, keeps a query key and not its value, and holds no cookie, no response text, no page title and none of eight named headers; the response's `Set-Cookie` shows only as a flag. The scheme lists and the `data:` rule are checked by unit tests in `crates/kharon-teach/src/strip.rs`. The name rule, the 2,000-value cap and the token-shaped bodies, keys and queries are checked by unit tests in `strip.rs` and `crates/kharon-teach/src/body.rs`. | +| REQ-TCH-003 | The file holds no cookie, no request or response header value other than the content type and the `Authorization` scheme (the browser's own name and user-agent string are recorded, as they are not header values the page sent), no response body, no query value and no request-body value. A field name, part name, JSON key or query key is kept only when it looks like a name (at most 64 characters of letters, digits and `_ . : - [ ]`, and not an identifier); otherwise it is written `{other}`. A body that is not what its content type says, or that would hold more than 2,000 values, is kept as `opaque`: its length and the role and encoding of each typed value found in it, never its content. A body over 1 MiB is `unavailable`. A URL scheme and an `Authorization` scheme are kept only from a known list. The content of a `data:` URL is never kept. | `crates/kharon-teach/tests/req_tch_003_file_contents.rs`: the file is mode `0600`, reads back as a recording, keeps a query key and not its value, and holds no cookie, no response text, no page title and none of eight named headers; the response's `Set-Cookie` shows only as a flag. The scheme lists and the `data:` rule are checked by unit tests in `crates/kharon-teach/src/strip.rs`. The name rule, the 2,000-value cap and the token-shaped bodies, keys and queries are checked by unit tests in `strip.rs` and `crates/kharon-teach/src/body.rs`. | | REQ-TCH-004 | The recorder holds no broker credential, listens on no socket, and opens no network connection except loopback CDP to the browser it launched. | `crates/kharon-teach/tests/req_tch_004_no_network.rs`: the shipped source has no listener or bind call. The dependency tree has no TLS or HTTP client crate, no `kharond` and no `kharon-connector`. The only dial is in `cdp.rs`, to a `ws://127.0.0.1:` address. | | REQ-TCH-005 | The four fixture logins classify as `form_post`, `script_post`, `two_step` and `transformed`. | `crates/kharon-teach/tests/req_tch_005_shapes.rs`: a real Chromium logs in to each of the four fixture pages, and each recording carries the shape named for it and the rung ceiling `milliseconds` (the form, script and hashed pages) or `agent_continues` (the two-step page). Further cases cover a multipart body, a blob body, a pop-up and a login in a frame. | | REQ-TCH-006 | The browser and its profile are removed on every exit the recorder can see (confirm, discard, window closed, browser killed, SIGINT, SIGTERM, SIGHUP, SIGQUIT). A recorder killed without that chance (SIGKILL, a crash) leaves its browser window open; the profile is then removed at a later start, once that window is closed. A start removes a leftover directory only when the recorder made it: its name is `kharon-teach-` and 20 letters and digits, it is a real directory owned by the same user, its owner file names a process that is not running, and no browser is running on it. Any other directory is left alone. The browser is always visible: no flag and no environment variable changes that. | `crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs`: the profile is gone after each of those exits, and no process is left with the profile on its command line after SIGINT, SIGTERM, SIGHUP and SIGQUIT, both before the browser is ready and after the end prompt. A leftover with a dead owner is removed at the next start, and directories without an owner file, with a junk owner file or with another name shape are not. After a login the browser's password and autofill tables are empty. A scan of the shipped source finds no way to ask for headless, and `PATH` is the only environment variable read. The sweep rule (name shape, owner file, live owner, live browser, symlinks) and the clearing of the browser's process group id are checked by unit tests in `crates/kharon-teach/src/browser.rs`. | From 59291d70d660a429488d839d0f8eb8f98ea0b64b Mon Sep 17 00:00:00 2001 From: Cycle Five Date: Fri, 20 Nov 2026 11:00:28 -0500 Subject: [PATCH 36/36] fix(teach): replace IDN host labels holding typed values (#364) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A host label written in punycode (xn--jos-dma for josé) holds none of the typed value's bytes, so strip_host kept it. The host is now also compared in its Unicode form, lowercased, when any label starts with xn--. If that form cannot be lined up with the labels, every xn-- label is replaced. A value typed in a decomposed form (NFD) is still not recognised. Co-Authored-By: Claude & Lothrop (cycle.five@proton.me) Claude-Session: https://claude.ai/code/session_017zqkE9yPuSSE3mYPnw3NAp --- Cargo.lock | 1 + crates/kharon-teach/Cargo.toml | 1 + crates/kharon-teach/src/marker.rs | 15 ++++++++ crates/kharon-teach/src/strip.rs | 57 +++++++++++++++++++++++++++++-- 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e5b3331..b956d85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1087,6 +1087,7 @@ dependencies = [ "clap", "futures-util", "hex", + "idna", "kharon-recording", "md-5", "memchr", diff --git a/crates/kharon-teach/Cargo.toml b/crates/kharon-teach/Cargo.toml index 660344e..24bce9c 100644 --- a/crates/kharon-teach/Cargo.toml +++ b/crates/kharon-teach/Cargo.toml @@ -26,6 +26,7 @@ hex = "0.4" percent-encoding = "2" memchr = "2" url = "2" +idna = "1" serde_json = { version = "1", features = ["raw_value"] } serde = { version = "1", features = ["derive"] } thiserror = "2" diff --git a/crates/kharon-teach/src/marker.rs b/crates/kharon-teach/src/marker.rs index b60154a..58cb43a 100644 --- a/crates/kharon-teach/src/marker.rs +++ b/crates/kharon-teach/src/marker.rs @@ -92,6 +92,21 @@ impl Marker { .position(|w| w.eq_ignore_ascii_case(needle)) .map(|start| start..start + needle.len()) } + + /// Byte ranges of every place this marker's value, lowercased (Unicode lowercasing), occurs + /// in `lowered`, which the caller has already lowercased. Empty for an empty or all-ASCII + /// value: `find_ignore_ascii_case` covers those without copying the value. Used for an + /// internationalised host name in its Unicode form. The lowercase copy is zeroized. + pub(crate) fn find_lowercased_in(&self, lowered: &str) -> Vec> { + if self.value.is_empty() || self.value.is_ascii() { + return Vec::new(); + } + let needle = Zeroizing::new(self.value.to_lowercase()); + lowered + .match_indices(needle.as_str()) + .map(|(start, found)| start..start + found.len()) + .collect() + } } type Needle = Zeroizing>; diff --git a/crates/kharon-teach/src/strip.rs b/crates/kharon-teach/src/strip.rs index 6bb0144..41cc115 100644 --- a/crates/kharon-teach/src/strip.rs +++ b/crates/kharon-teach/src/strip.rs @@ -237,8 +237,7 @@ pub(crate) fn holds_typed( /// /// A URL that cannot be a base (`data:`, `blob:`, `about:`, `mailto:`) keeps only its scheme. /// The scheme is kept only when it is in a short known list, else it is `{other}`. -/// Known limits: an internationalised host is compared in its punycode form, so a non-ASCII -/// typed value inside such a label is not recognised; the port is not tested. +/// Known limit: the port is not tested. pub fn strip_url( url: &str, markers: &[Marker], @@ -375,6 +374,41 @@ fn strip_host( for (t, label) in typed.iter_mut().zip(host.split('.')) { *t |= kept_hit(label.as_bytes(), markers, pair); } + // An internationalised label arrives as punycode (`xn--jos-dma` for `josé`), which holds + // none of the typed value's bytes, so the host is compared again in its Unicode form, + // lowercased. Known limit: a value typed in a decomposed form (NFD) is not recognised, + // because the host's Unicode form is always composed (NFC). + if host.split('.').any(|label| label.starts_with("xn--")) { + let (unicode, result) = idna::domain_to_unicode(host); + let unicode = Zeroizing::new(unicode); + if result.is_err() || unicode.split('.').count() != spans.len() { + // The Unicode form cannot be lined up with the labels: fail closed. + for (t, label) in typed.iter_mut().zip(host.split('.')) { + *t |= label.starts_with("xn--"); + } + } else { + let lowered_labels: Vec> = unicode + .split('.') + .map(|label| Zeroizing::new(label.to_lowercase())) + .collect(); + let mut lowered = Zeroizing::new(String::with_capacity(unicode.len())); + for (i, label) in lowered_labels.iter().enumerate() { + if i > 0 { + lowered.push('.'); + } + lowered.push_str(label); + } + let lowered_spans = spans_of(lowered_labels.iter().map(|label| label.len())); + let mut lowered_ranges = occurrences(&lowered, markers); + for m in markers { + lowered_ranges.extend(m.find_lowercased_in(&lowered)); + } + let hit = touched(&lowered_ranges, &lowered_spans); + for ((t, hit), label) in typed.iter_mut().zip(hit).zip(&lowered_labels) { + *t |= hit || kept_hit(label.as_bytes(), markers, pair); + } + } + } let mut labels: Vec<&str> = Vec::new(); let mut previous_typed = false; for (label, is_typed) in host.split('.').zip(typed) { @@ -782,6 +816,25 @@ mod tests { assert_eq!(s.value.origin, "https://{typed}.ex.{typed}.test"); } + #[test] + fn an_idn_host_label_holding_a_typed_value_is_replaced() { + // The parser writes the host as punycode: `josé` is `xn--jos-dma`. + let m = [user("josé")]; + let s = strip_url("https://xn--jos-dma.ex.test/login", &m, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + // Typed in another case: the host's Unicode form is lowercase. + let m = [user("JOSÉ")]; + let s = strip_url("https://xn--jos-dma.ex.test/login", &m, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + // A value that spans two internationalised labels. + let m = [user("josé.ñu")]; + let s = strip_url("https://josé.ñu.ex.test/login", &m, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + // An internationalised label that holds no typed value is kept as it is. + let s = strip_url("https://xn--bcher-kva.ex.test/login", &m, None).unwrap(); + assert_eq!(s.value.origin, "https://xn--bcher-kva.ex.test"); + } + #[test] fn basic_auth_is_reduced_to_scheme_and_hits() { let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x");