diff --git a/Cargo.lock b/Cargo.lock index 5b8b951..b956d85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1070,6 +1070,43 @@ dependencies = [ "zeroize", ] +[[package]] +name = "kharon-recording" +version = "0.0.0" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "kharon-teach" +version = "0.0.0" +dependencies = [ + "base64ct", + "clap", + "futures-util", + "hex", + "idna", + "kharon-recording", + "md-5", + "memchr", + "percent-encoding", + "publicsuffix", + "rand", + "rustix", + "serde", + "serde_json", + "sha1", + "sha2", + "thiserror 2.0.21", + "time", + "tokio", + "tokio-tungstenite", + "url", + "zeroize", +] + [[package]] name = "kharond" version = "0.0.0" @@ -1166,6 +1203,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.3" diff --git a/Cargo.toml b/Cargo.toml index ee2fa89..60369e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,8 @@ members = [ "crates/kharon-cli", "crates/kharon-connector", "crates/kharon-agent-connector", + "crates/kharon-recording", + "crates/kharon-teach", ] # SPEC §8 process hardening: a release build aborts on panic instead of diff --git a/README.md b/README.md index fc7cfd4..72a54fd 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Vaultwarden fill broker: lets AI agents fill credentials without ever seeing the - [`docs/broker-lifecycle.md`](docs/broker-lifecycle.md): host install of kharond, backup and restore of `/data`, upgrades, and the secrets inventory. - [`docs/vault-onboarding.md`](docs/vault-onboarding.md): the customer's Bitwarden organization, collection, dedicated account, and API key. - [`docs/agent-runtime-setup.md`](docs/agent-runtime-setup.md): the agent side on a hosted or root runtime, and handing over credentials. +- [`docs/teach-mode.md`](docs/teach-mode.md): `kharon-teach`, a program that records how a login works while a person logs in by hand with a throwaway account, and writes a file with no typed value in it (#361). - [`docs/hermes-end-to-end.md`](docs/hermes-end-to-end.md): Hermes after the connector is installed: the MCP client, what the agent and the owner do in a fill, and what each failure looks like. A record of one live run. - [`docs/acceptance-checks.md`](docs/acceptance-checks.md): checks a new managed deploy must pass before it carries anyone but a pilot (edge, one real fill, REQ-GOAL-002 canaries). - [`deploy/compose/README.md`](deploy/compose/README.md): running kharond with Docker Compose. diff --git a/SPEC.md b/SPEC.md index c89e288..98b3ffd 100644 --- a/SPEC.md +++ b/SPEC.md @@ -209,6 +209,19 @@ The agent keeps its own CDP session (Playwright/puppeteer `connectOverCDP`). kha | REQ-ARCH-005 | Target ownership. The broker creates one browser context per agent (`Target.createBrowserContext`) and hands the agent its `browserContextId` via `kharon_context` / `POST /v1/context` (#65): the call connects if needed (the broker stays lazy, no connect at startup), creates or returns the agent's cached context, and replies `browser_context_id`. It is idempotent while the CDP connection lives; a reconnect or a lock invalidates the id (the next call returns a new one, and fills into tabs of the old context get `target_not_owned`). It is lock-gated (`423` while locked), runs under the `fill_deadline`, and is audited like a fill (`context_open` before any CDP write, then exactly one `context` outcome; REQ-AUD-003†). `kharon_status` reports the cached id or `null` and never connects. `fill` is accepted only if `cdp_target_id` is in the requesting agent's context. Targets in the default context are refused unless `allow_default_context=true`, for single-agent setups. | E2E: agent A fills into a tab in agent B's context → `target_not_owned`. Default-context tab with the flag off → refused. Daemon + CDP stub: no connection before the first `kharon_context`; two calls → same id, one `createBrowserContext`; after a reconnect or lock+unlock → new id; `423` while locked; `status.browser_context_id` is `null` before the first call and after invalidation. | | REQ-ARCH-004 | Chromium launched with password manager and autofill save disabled. | Profile `Login Data` contains 0 rows after 10 fills. | +### 3.4 Teach mode: recording a demonstrated login + +`kharon-teach` is a separate program. A person runs it on their own computer, with a throwaway account, to show kharon how one site's login works. It opens its own visible browser, reads what the page does while the person logs in by hand, and writes one file that describes the structure of the login. The file holds no typed value. It shares nothing with a fill: it has no broker credential, no vault access and no connection to a broker. It needs to hear what the page does, so its CDP client enables the `Runtime`, `Network` and `Page` domains. The broker's client enables none of them (REQ-ARCH-003). The recorder therefore has its own client and never uses the broker's. Turning a recording into a login method, and replaying one, is later work (#361). The design is in [`docs/superpowers/specs/2026-11-18-teach-recorder-design.md`](docs/superpowers/specs/2026-11-18-teach-recorder-design.md). The user page is [`docs/teach-mode.md`](docs/teach-mode.md). + +| ID | Requirement | AC | +|---|---|---| +| REQ-TCH-001 | The recorder launches the browser itself, in its own process group, on a profile directory it created (mode `0700`, with an owner file). It has no option to attach to another browser or to name a profile. The browser's debugging port is loopback only. | `crates/kharon-teach/tests/req_tch_001_own_browser.rs`: a launched browser runs with `--user-data-dir` set to the profile the recorder made, and the debugging port listens only on loopback. A scan of the shipped source finds no `ws://` or `http://` address that is not the loopback CDP one, and the fields of `Launch` are `browser`, `first_url`, `headless` and `profile`. | +| REQ-TCH-002 | No typed value is written to the recording file, to any other file of the recorder's own, or to its output. This covers every value typed in the window, not only the user name and the password. A name that contains a typed value is replaced with `{typed}` (a field or part name, a JSON key, a host label, a content type, a scheme), and the hits found in it stay on the field. A path segment that contains one is replaced with `{id}`. An element attribute or selector that holds one is left out. A value shorter than six characters is removed from kept text where it stands as a whole word, in any of its encodings. | `crates/kharon-teach/tests/req_tch_002_no_typed_value.rs`: for each of the four fixture logins, a byte search of the recording for the user name and the password, in every encoding the recorder knows, finds nothing. The same search over the whole program's file, standard output and standard error finds nothing. A page that copies a typed value into an attribute, and the listener script's own reports, carry none of it. A fifth fixture puts the typed values into a path, a query key and a JSON key: the file holds none of them, the path is `/u/{id}`, the query key and the JSON key are `{typed}`, and the hits in the key are kept. The whole-string rule, the encoded forms of short values and the name rules are checked by unit tests in `crates/kharon-teach/src/strip.rs`, `body.rs` and `marker.rs`. | +| REQ-TCH-003 | The file holds no cookie, no request or response header value other than the content type and the `Authorization` scheme (the browser's own name and user-agent string are recorded, as they are not header values the page sent), no response body, no query value and no request-body value. A field name, part name, JSON key or query key is kept only when it looks like a name (at most 64 characters of letters, digits and `_ . : - [ ]`, and not an identifier); otherwise it is written `{other}`. A body that is not what its content type says, or that would hold more than 2,000 values, is kept as `opaque`: its length and the role and encoding of each typed value found in it, never its content. A body over 1 MiB is `unavailable`. A URL scheme and an `Authorization` scheme are kept only from a known list. The content of a `data:` URL is never kept. | `crates/kharon-teach/tests/req_tch_003_file_contents.rs`: the file is mode `0600`, reads back as a recording, keeps a query key and not its value, and holds no cookie, no response text, no page title and none of eight named headers; the response's `Set-Cookie` shows only as a flag. The scheme lists and the `data:` rule are checked by unit tests in `crates/kharon-teach/src/strip.rs`. The name rule, the 2,000-value cap and the token-shaped bodies, keys and queries are checked by unit tests in `strip.rs` and `crates/kharon-teach/src/body.rs`. | +| REQ-TCH-004 | The recorder holds no broker credential, listens on no socket, and opens no network connection except loopback CDP to the browser it launched. | `crates/kharon-teach/tests/req_tch_004_no_network.rs`: the shipped source has no listener or bind call. The dependency tree has no TLS or HTTP client crate, no `kharond` and no `kharon-connector`. The only dial is in `cdp.rs`, to a `ws://127.0.0.1:` address. | +| REQ-TCH-005 | The four fixture logins classify as `form_post`, `script_post`, `two_step` and `transformed`. | `crates/kharon-teach/tests/req_tch_005_shapes.rs`: a real Chromium logs in to each of the four fixture pages, and each recording carries the shape named for it and the rung ceiling `milliseconds` (the form, script and hashed pages) or `agent_continues` (the two-step page). Further cases cover a multipart body, a blob body, a pop-up and a login in a frame. | +| REQ-TCH-006 | The browser and its profile are removed on every exit the recorder can see (confirm, discard, window closed, browser killed, SIGINT, SIGTERM, SIGHUP, SIGQUIT). A recorder killed without that chance (SIGKILL, a crash) leaves its browser window open; the profile is then removed at a later start, once that window is closed. A start removes a leftover directory only when the recorder made it: its name is `kharon-teach-` and 20 letters and digits, it is a real directory owned by the same user, its owner file names a process that is not running, and no browser is running on it. Any other directory is left alone. The browser is always visible: no flag and no environment variable changes that. | `crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs`: the profile is gone after each of those exits, and no process is left with the profile on its command line after SIGINT, SIGTERM, SIGHUP and SIGQUIT, both before the browser is ready and after the end prompt. A leftover with a dead owner is removed at the next start, and directories without an owner file, with a junk owner file or with another name shape are not. After a login the browser's password and autofill tables are empty. A scan of the shipped source finds no way to ask for headless, and `PATH` is the only environment variable read. The sweep rule (name shape, owner file, live owner, live browser, symlinks) and the clearing of the browser's process group id are checked by unit tests in `crates/kharon-teach/src/browser.rs`. | + ## 4. Agent-facing API Exposed as MCP (tools below) and HTTP `/v1/*`, with identical semantics. Auth: `Authorization: Bearer kh_agent__`. Only the Argon2id hash of the token is stored. One token per agent identity. The owner mints the token out of band with `kharon token mint ` (§7); the daemon does not. A dashboard instead connects with MCP OAuth and calls `POST /mcp` with a short-lived access token, `kh_at_…`, issued by the deploy's own authorization server (§4.1). diff --git a/crates/kharon-recording/Cargo.toml b/crates/kharon-recording/Cargo.toml new file mode 100644 index 0000000..f4d4d2f --- /dev/null +++ b/crates/kharon-recording/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "kharon-recording" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true +description = "Typed recording format for kharon teach mode (#361)" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "2" diff --git a/crates/kharon-recording/src/classify.rs b/crates/kharon-recording/src/classify.rs new file mode 100644 index 0000000..4c7be35 --- /dev/null +++ b/crates/kharon-recording/src/classify.rs @@ -0,0 +1,807 @@ +//! Classification of a recorded login: its shape and the strongest rung it can take. + +use serde::{Deserialize, Serialize}; + +use crate::format::{ElementDescriptor, Step}; +use crate::request::{Deciding, ResourceKind, Role, StoredRequest}; + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct Flags { + pub captcha: bool, + pub otp_step: bool, + pub cross_site: bool, + pub popup: bool, + pub framed_login: bool, + pub shadow_dom: bool, + pub multiple_passwords: bool, + pub body_unavailable: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PasswordLeg { + FormPost, + ScriptPost, + Transformed, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Shape { + FormPost, + ScriptPost, + TwoStep { password_leg: PasswordLeg }, + Transformed { digest: Option }, + Unrecognized, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RungCeiling { + Milliseconds, + AgentContinues, + NeedsPerson, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Classification { + pub shape: Shape, + pub rung_ceiling: RungCeiling, + pub network_swap_candidate: bool, + pub notes: Vec, +} + +/// A plain (not digest) password hit that decides anything: a weak one counts only when no +/// confident password hit exists anywhere (see `Deciding`). +fn plain_hit(r: &StoredRequest, d: &Deciding) -> bool { + r.hits() + .iter() + .any(|h| h.role == Role::Password && !h.encoding.is_digest() && d.counts(h)) +} + +fn has_hit(r: &StoredRequest, role: Role, d: &Deciding) -> bool { + r.hits().iter().any(|h| h.role == role && d.counts(h)) +} + +/// Restates the three TOTP tests in `resolveField` in +/// `crates/kharond/src/cdp/page_js.rs`; keep the two in step. +pub fn is_otp_descriptor(e: &ElementDescriptor) -> bool { + e.autocomplete + .as_deref() + .is_some_and(|a| a.contains("one-time")) + || e.name + .as_deref() + .is_some_and(|n| n.to_lowercase().contains("otp")) + || e.input_type.as_deref() == Some("tel") +} + +fn t_ms(step: &Step) -> u64 { + match step { + Step::Navigation { t_ms, .. } + | Step::Input { t_ms, .. } + | Step::Click { t_ms, .. } + | Step::Submit { t_ms, .. } + | Step::EnterKey { t_ms, .. } => *t_ms, + } +} + +fn unrecognized(note: &str) -> Classification { + Classification { + shape: Shape::Unrecognized, + rung_ceiling: RungCeiling::NeedsPerson, + network_swap_candidate: false, + notes: vec![note.to_string()], + } +} + +/// Names the shape of a recorded login and the strongest rung it can take. +pub fn classify(steps: &[Step], requests: &[StoredRequest], flags: &Flags) -> Classification { + let deciding = Deciding::from_hits(requests.iter().flat_map(|r| r.hits())); + let swap_candidate = requests.iter().filter(|r| plain_hit(r, &deciding)).count() == 1; + + // Rule 1. + let blocking = [ + ( + flags.cross_site, + "The login goes to another site partway through, so a person has to look at it.", + ), + ( + flags.popup, + "The login opens a pop-up window, so a person has to look at it.", + ), + ( + flags.framed_login, + "The login form sits inside a frame, so a person has to look at it.", + ), + ( + flags.multiple_passwords, + "The page has more than one password field, so a person has to look at it.", + ), + ]; + let notes: Vec = blocking + .iter() + .filter(|(on, _)| *on) + .map(|(_, n)| n.to_string()) + .collect(); + if !notes.is_empty() { + return Classification { + shape: Shape::Unrecognized, + rung_ceiling: RungCeiling::NeedsPerson, + network_swap_candidate: false, + notes, + }; + } + + // Rule 2. + let first_password = steps.iter().find_map(|s| match s { + Step::Input { + t_ms, + role: Role::Password, + element, + .. + } => Some((*t_ms, element)), + _ => None, + }); + let Some((p, password_element)) = first_password else { + return unrecognized("no password field was used"); + }; + + // Rule 3. + let username_t = steps.iter().find_map(|s| match s { + Step::Input { + t_ms, + role: Role::Username, + .. + } => Some(*t_ms), + _ => None, + }); + let navigated = username_t.is_some_and(|u| { + steps.iter().any(|s| { + matches!(s, Step::Navigation { frame, .. } if frame.main) && t_ms(s) > u && t_ms(s) < p + }) + }); + let username_only_request = requests.iter().any(|r| { + r.t_ms < p + && has_hit(r, Role::Username, &deciding) + && !has_hit(r, Role::Password, &deciding) + }); + let two_step = navigated || username_only_request; + + // Rule 4. + let mut notes = Vec::new(); + let earliest_plain = requests + .iter() + .filter(|r| plain_hit(r, &deciding)) + .min_by_key(|r| r.t_ms); + let leg = match earliest_plain { + Some(r) => match r.resource { + ResourceKind::Document if r.main_frame_navigation => PasswordLeg::FormPost, + ResourceKind::Xhr | ResourceKind::Fetch => PasswordLeg::ScriptPost, + _ => { + notes.push( + "The password was sent in a way this tool does not know, so a person has to look at it." + .to_string(), + ); + return finish_unrecognized(notes); + } + }, + None => { + if flags.body_unavailable { + notes.push( + "The browser would not give up the request bodies, so the password could not be found in them." + .to_string(), + ); + return finish_unrecognized(notes); + } + PasswordLeg::Transformed + } + }; + let digest = requests + .iter() + .flat_map(|r| r.hits()) + .find(|h| h.role == Role::Password && h.encoding.is_digest() && deciding.counts(h)) + .and_then(|h| h.encoding.digest_name()) + .map(str::to_string); + + // Rule 5. + let shape = if two_step { + Shape::TwoStep { password_leg: leg } + } else { + match leg { + PasswordLeg::FormPost => Shape::FormPost, + PasswordLeg::ScriptPost => Shape::ScriptPost, + PasswordLeg::Transformed => Shape::Transformed { + digest: digest.clone(), + }, + } + }; + + // Rule 6. + let mut ceiling = RungCeiling::Milliseconds; + let mut lower = |to: RungCeiling, note: &str| { + if to == RungCeiling::NeedsPerson || ceiling == RungCeiling::Milliseconds { + ceiling = to; + } + notes.push(note.to_string()); + }; + if matches!(shape, Shape::TwoStep { .. }) { + lower( + RungCeiling::AgentContinues, + "The username and password are entered on separate steps, so the agent has to carry on after the first.", + ); + } + if matches!( + shape, + Shape::TwoStep { + password_leg: PasswordLeg::Transformed + } + ) { + // The leg is the same uncertainty as a one-page transformed login: the password was + // found nowhere in plain form. Only a named digest lets the agent carry on. + match &digest { + Some(name) => lower( + RungCeiling::AgentContinues, + &format!( + "The password is sent as a {name} digest, so the agent has to hash it the same way." + ), + ), + None => lower( + RungCeiling::NeedsPerson, + "The password is changed before it is sent in a way this tool cannot name, so a person has to look at it.", + ), + } + } + if flags.captcha { + lower( + RungCeiling::AgentContinues, + "The login shows a CAPTCHA, so the agent has to carry on from there.", + ); + } + if flags.otp_step { + lower( + RungCeiling::AgentContinues, + "The login asks for a one-time code, so the agent has to carry on from there.", + ); + } + let one_page = !matches!(shape, Shape::TwoStep { .. }); + if one_page && password_element.form.is_none() { + let can_submit = steps.iter().any(|s| { + t_ms(s) >= p + && match s { + Step::EnterKey { .. } => true, + Step::Click { element, .. } => { + element.tag.eq_ignore_ascii_case("button") + || element.input_type.as_deref() == Some("submit") + } + _ => false, + } + }); + if !can_submit { + lower( + RungCeiling::AgentContinues, + "The password field is not in a form and nothing was pressed to send it, so the agent has to find a way to submit.", + ); + } + } + if matches!(shape, Shape::Transformed { digest: None }) { + lower( + RungCeiling::NeedsPerson, + "The password is changed before it is sent in a way this tool cannot name, so a person has to look at it.", + ); + } + + Classification { + shape, + rung_ceiling: ceiling, + network_swap_candidate: swap_candidate, + notes, + } +} + +fn finish_unrecognized(notes: Vec) -> Classification { + Classification { + shape: Shape::Unrecognized, + rung_ceiling: RungCeiling::NeedsPerson, + network_swap_candidate: false, + notes, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::{FormRef, FrameRef, StrippedUrl}; + use crate::request::{BodyShape, Encoding, FieldShape, MarkerHit, ValueKind, ValueShape}; + + fn frame() -> FrameRef { + FrameRef { + main: true, + origin: "https://example.test".into(), + cross_origin: false, + } + } + + fn url() -> StrippedUrl { + StrippedUrl { + origin: "https://example.test".into(), + path: "/login".into(), + query_keys: vec![], + } + } + + fn element(id: u32, input_type: &str, in_form: bool) -> ElementDescriptor { + ElementDescriptor { + element_id: id, + tag: "input".into(), + input_type: Some(input_type.into()), + id: None, + name: None, + autocomplete: None, + placeholder: None, + aria_label: None, + was_password: input_type == "password", + form: in_form.then(|| FormRef { + index: 0, + action: None, + method: "post".into(), + }), + selectors: vec![], + in_shadow_root: false, + } + } + + fn input(t: u64, role: Role, in_form: bool) -> Step { + let ty = if role == Role::Password { + "password" + } else { + "text" + }; + Step::Input { + t_ms: t, + frame: frame(), + element: element(t as u32, ty, in_form), + value_len: 6, + role, + } + } + + fn nav(t: u64) -> Step { + Step::Navigation { + t_ms: t, + frame: frame(), + url: url(), + } + } + + fn hit(role: Role, encoding: Encoding) -> MarkerHit { + MarkerHit { + role, + encoding, + low_confidence: false, + } + } + + fn req(t: u64, resource: ResourceKind, hits: Vec) -> StoredRequest { + StoredRequest { + t_ms: t, + method: "POST".into(), + url: url(), + resource, + main_frame_navigation: resource == ResourceKind::Document, + content_type: None, + authorization: None, + url_markers: vec![], + body: BodyShape::Urlencoded { + fields: vec![FieldShape { + key: "k".into(), + value: ValueShape { + kind: ValueKind::String, + len: 6, + markers: hits, + }, + }], + }, + response: None, + } + } + + fn plain_pw() -> MarkerHit { + hit(Role::Password, Encoding::AsTyped) + } + + fn one_page_steps() -> Vec { + vec![ + nav(0), + input(10, Role::Username, true), + input(20, Role::Password, true), + ] + } + + fn clean_form_post() -> Classification { + classify( + &one_page_steps(), + &[req(100, ResourceKind::Document, vec![plain_pw()])], + &Flags::default(), + ) + } + + #[test] + fn form_post() { + let c = clean_form_post(); + assert_eq!(c.shape, Shape::FormPost); + assert_eq!(c.rung_ceiling, RungCeiling::Milliseconds); + assert!(c.network_swap_candidate); + } + + #[test] + fn script_post() { + let c = classify( + &one_page_steps(), + &[req( + 100, + ResourceKind::Fetch, + vec![hit(Role::Password, Encoding::JsonEscaped)], + )], + &Flags::default(), + ); + assert_eq!(c.shape, Shape::ScriptPost); + assert_eq!(c.rung_ceiling, RungCeiling::Milliseconds); + assert!(c.network_swap_candidate); + } + + #[test] + fn two_step_by_navigation() { + let steps = vec![ + nav(0), + input(10, Role::Username, true), + nav(50), + input(60, Role::Password, true), + ]; + let c = classify( + &steps, + &[req(100, ResourceKind::Document, vec![plain_pw()])], + &Flags::default(), + ); + assert_eq!( + c.shape, + Shape::TwoStep { + password_leg: PasswordLeg::FormPost + } + ); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); + assert!(!c.notes.is_empty()); + } + + #[test] + fn two_step_by_request() { + let steps = vec![ + nav(0), + input(10, Role::Username, true), + input(60, Role::Password, true), + ]; + let reqs = [ + req( + 30, + ResourceKind::Xhr, + vec![hit(Role::Username, Encoding::AsTyped)], + ), + req(100, ResourceKind::Fetch, vec![plain_pw()]), + ]; + let c = classify(&steps, &reqs, &Flags::default()); + assert_eq!( + c.shape, + Shape::TwoStep { + password_leg: PasswordLeg::ScriptPost + } + ); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); + } + + #[test] + fn two_step_with_a_transformed_leg_is_agent_continues() { + let steps = vec![ + nav(0), + input(10, Role::Username, true), + nav(50), + input(60, Role::Password, true), + ]; + let c = classify( + &steps, + &[req( + 100, + ResourceKind::Fetch, + vec![hit(Role::Password, Encoding::Sha256Hex)], + )], + &Flags::default(), + ); + assert_eq!( + c.shape, + Shape::TwoStep { + password_leg: PasswordLeg::Transformed + } + ); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); + assert!( + c.notes.iter().any(|n| n.contains("sha256 digest")), + "{:?}", + c.notes + ); + } + + #[test] + fn two_step_with_a_transformed_leg_and_no_named_digest_needs_a_person() { + let steps = vec![ + nav(0), + input(10, Role::Username, true), + nav(50), + input(60, Role::Password, true), + ]; + let c = classify( + &steps, + &[req(100, ResourceKind::Fetch, vec![])], + &Flags::default(), + ); + assert_eq!( + c.shape, + Shape::TwoStep { + password_leg: PasswordLeg::Transformed + } + ); + assert_eq!(c.rung_ceiling, RungCeiling::NeedsPerson); + assert!( + c.notes.iter().any(|n| n.contains("cannot name")), + "{:?}", + c.notes + ); + } + + fn weak(role: Role, encoding: Encoding) -> MarkerHit { + MarkerHit { + role, + encoding, + low_confidence: true, + } + } + + #[test] + fn a_weak_password_hit_does_not_pick_the_leg_when_a_confident_one_exists() { + // A short password matched by chance in an earlier request of a kind no leg is made + // from; the request that really carried it is a form post. + let reqs = [ + req( + 30, + ResourceKind::Other, + vec![weak(Role::Password, Encoding::AsTyped)], + ), + req(100, ResourceKind::Document, vec![plain_pw()]), + ]; + let c = classify(&one_page_steps(), &reqs, &Flags::default()); + assert_eq!(c.shape, Shape::FormPost); + assert!( + c.network_swap_candidate, + "only the confident request counts" + ); + // With no confident hit anywhere, the weak ones decide, as before. + let only_weak = [req( + 100, + ResourceKind::Document, + vec![weak(Role::Password, Encoding::AsTyped)], + )]; + let c = classify(&one_page_steps(), &only_weak, &Flags::default()); + assert_eq!(c.shape, Shape::FormPost); + assert!(c.network_swap_candidate); + } + + #[test] + fn a_weak_username_hit_does_not_make_a_two_step_when_a_confident_one_exists() { + let reqs = [ + req( + 15, + ResourceKind::Xhr, + vec![weak(Role::Username, Encoding::AsTyped)], + ), + req( + 100, + ResourceKind::Document, + vec![hit(Role::Username, Encoding::AsTyped), plain_pw()], + ), + ]; + let c = classify(&one_page_steps(), &reqs, &Flags::default()); + assert_eq!(c.shape, Shape::FormPost); + // Alone, a weak username-only request still reads as a first step. + let alone = [ + req( + 15, + ResourceKind::Xhr, + vec![weak(Role::Username, Encoding::AsTyped)], + ), + req(100, ResourceKind::Document, vec![plain_pw()]), + ]; + let c = classify(&one_page_steps(), &alone, &Flags::default()); + assert!(matches!(c.shape, Shape::TwoStep { .. })); + } + + #[test] + fn transformed_with_digest() { + let c = classify( + &one_page_steps(), + &[req( + 100, + ResourceKind::Fetch, + vec![hit(Role::Password, Encoding::Sha256Hex)], + )], + &Flags::default(), + ); + assert_eq!( + c.shape, + Shape::Transformed { + digest: Some("sha256".into()) + } + ); + assert_eq!(c.rung_ceiling, RungCeiling::Milliseconds); + assert!(!c.network_swap_candidate); + } + + #[test] + fn transformed_unknown() { + let c = classify( + &one_page_steps(), + &[req(100, ResourceKind::Fetch, vec![])], + &Flags::default(), + ); + assert_eq!(c.shape, Shape::Transformed { digest: None }); + assert_eq!(c.rung_ceiling, RungCeiling::NeedsPerson); + assert!(!c.notes.is_empty()); + } + + #[test] + fn password_nowhere_and_body_unavailable_is_unrecognized() { + let flags = Flags { + body_unavailable: true, + ..Flags::default() + }; + let c = classify( + &one_page_steps(), + &[req(100, ResourceKind::Fetch, vec![])], + &flags, + ); + assert_eq!(c.shape, Shape::Unrecognized); + assert_eq!(c.rung_ceiling, RungCeiling::NeedsPerson); + assert!(!c.notes.is_empty()); + } + + #[test] + fn each_blocking_flag_wins() { + let setters: [fn(&mut Flags); 4] = [ + |f| f.cross_site = true, + |f| f.popup = true, + |f| f.framed_login = true, + |f| f.multiple_passwords = true, + ]; + for set in setters { + let mut flags = Flags::default(); + set(&mut flags); + let c = classify( + &one_page_steps(), + &[req(100, ResourceKind::Document, vec![plain_pw()])], + &flags, + ); + assert_eq!(c.shape, Shape::Unrecognized); + assert_eq!(c.rung_ceiling, RungCeiling::NeedsPerson); + assert!(!c.network_swap_candidate); + assert!(!c.notes.is_empty()); + } + } + + #[test] + fn captcha_caps_a_form_post_at_agent_continues() { + let flags = Flags { + captcha: true, + ..Flags::default() + }; + let c = classify( + &one_page_steps(), + &[req(100, ResourceKind::Document, vec![plain_pw()])], + &flags, + ); + assert_eq!(c.shape, Shape::FormPost); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); + assert!(!c.notes.is_empty()); + } + + #[test] + fn otp_step_caps_at_agent_continues() { + let flags = Flags { + otp_step: true, + ..Flags::default() + }; + let c = classify( + &one_page_steps(), + &[req(100, ResourceKind::Document, vec![plain_pw()])], + &flags, + ); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); + assert!(!c.notes.is_empty()); + } + + fn formless_steps() -> Vec { + vec![ + nav(0), + input(10, Role::Username, false), + input(20, Role::Password, false), + ] + } + + #[test] + fn no_way_to_submit_caps_at_agent_continues() { + let c = classify( + &formless_steps(), + &[req(100, ResourceKind::Fetch, vec![plain_pw()])], + &Flags::default(), + ); + assert_eq!(c.shape, Shape::ScriptPost); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); + assert!(!c.notes.is_empty()); + } + + #[test] + fn enter_key_keeps_milliseconds_without_a_form() { + let mut steps = formless_steps(); + steps.push(Step::EnterKey { + t_ms: 30, + frame: frame(), + element: element(2, "password", false), + }); + let c = classify( + &steps, + &[req(100, ResourceKind::Fetch, vec![plain_pw()])], + &Flags::default(), + ); + assert_eq!(c.rung_ceiling, RungCeiling::Milliseconds); + assert!(c.notes.is_empty()); + } + + #[test] + fn two_plain_hits_is_not_a_swap_candidate() { + let c = classify( + &one_page_steps(), + &[ + req(100, ResourceKind::Document, vec![plain_pw()]), + req(110, ResourceKind::Fetch, vec![plain_pw()]), + ], + &Flags::default(), + ); + assert!(!c.network_swap_candidate); + } + + #[test] + fn no_password_step_is_unrecognized() { + let steps = vec![nav(0), input(10, Role::Username, true)]; + let c = classify( + &steps, + &[req(100, ResourceKind::Document, vec![plain_pw()])], + &Flags::default(), + ); + assert_eq!(c.shape, Shape::Unrecognized); + assert_eq!(c.rung_ceiling, RungCeiling::NeedsPerson); + assert!(c.notes.iter().any(|n| n == "no password field was used")); + } + + #[test] + fn otp_descriptor_matches_the_three_tests_and_nothing_else() { + let base = element(1, "text", true); + assert!(!is_otp_descriptor(&base)); + let mut a = base.clone(); + a.autocomplete = Some("one-time-code".into()); + assert!(is_otp_descriptor(&a)); + let mut n = base.clone(); + n.name = Some("User_OTP".into()); + assert!(is_otp_descriptor(&n)); + let t = element(1, "tel", true); + assert!(is_otp_descriptor(&t)); + let mut other = base; + other.autocomplete = Some("username".into()); + other.name = Some("email".into()); + assert!(!is_otp_descriptor(&other)); + } +} diff --git a/crates/kharon-recording/src/format.rs b/crates/kharon-recording/src/format.rs new file mode 100644 index 0000000..1ef4943 --- /dev/null +++ b/crates/kharon-recording/src/format.rs @@ -0,0 +1,527 @@ +//! The recording file: a typed, value-free record of a demonstrated login. + +use serde::{Deserialize, Serialize}; + +use crate::classify::{Classification, Flags}; +use crate::request::{Role, StoredRequest}; + +pub const FORMAT_VERSION: u32 = 1; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StrippedUrl { + /// scheme://host[:port] + pub origin: String, + pub path: String, + pub query_keys: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct FrameRef { + pub main: bool, + pub origin: String, + pub cross_origin: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct FormRef { + pub index: u32, + pub action: Option, + pub method: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ElementDescriptor { + pub element_id: u32, + pub tag: String, + pub input_type: Option, + pub id: Option, + pub name: Option, + pub autocomplete: Option, + pub placeholder: Option, + pub aria_label: Option, + /// True when the element was a password input (type=password) when recorded. + pub was_password: bool, + pub form: Option, + /// Candidate CSS selectors, most stable first. + pub selectors: Vec, + pub in_shadow_root: bool, +} + +/// Every `t_ms` is milliseconds since the recording started. +/// +/// An `Input` step is recorded once per element, at its first input event, and +/// its `value_len` is updated in place as the value changes. `change` events +/// add no step. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Step { + Navigation { + t_ms: u64, + frame: FrameRef, + url: StrippedUrl, + }, + Input { + t_ms: u64, + frame: FrameRef, + element: ElementDescriptor, + /// Characters typed so far. + value_len: u32, + role: Role, + }, + Click { + t_ms: u64, + frame: FrameRef, + element: ElementDescriptor, + }, + Submit { + t_ms: u64, + frame: FrameRef, + form: FormRef, + }, + EnterKey { + t_ms: u64, + frame: FrameRef, + element: ElementDescriptor, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EndedBy { + Confirmed, + WindowClosed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ListenerWorld { + Isolated, + Main, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Viewport { + pub width: u32, + pub height: u32, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct BrowserInfo { + pub product: String, + pub user_agent: String, + pub viewport: Viewport, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StrippedCounts { + /// Requests observed on the wire. + pub requests_seen: u64, + /// Requests kept in the file. + pub requests_stored: u64, + /// Request bodies replaced by a value-free shape. + pub request_bodies_reduced: u64, + /// URL query values dropped (keys are kept). + pub query_values_removed: u64, + /// URL path segments replaced by a placeholder. + pub path_segments_replaced: u64, + /// Typed input values discarded (only lengths are kept). + pub typed_values_removed: u64, + /// Steps past the recorder's limit that were not written to `steps`. + #[serde(default)] + pub steps_dropped: u64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Recording { + /// `to_json` always writes `FORMAT_VERSION`, whatever this field holds. + pub format_version: u32, + pub tool_version: String, + pub browser: BrowserInfo, + pub listener_world: ListenerWorld, + /// RFC 3339, UTC. + pub started_at: String, + /// RFC 3339, UTC. + pub ended_at: String, + pub ended_by: EndedBy, + pub success_confirmed: bool, + pub start_url: StrippedUrl, + pub final_url: Option, + pub steps: Vec, + pub requests: Vec, + pub flags: Flags, + pub classification: Classification, + pub stripped: StrippedCounts, +} + +#[derive(Debug, thiserror::Error)] +pub enum FormatError { + #[error("unsupported recording format_version {0}")] + UnsupportedVersion(u32), + #[error("not a kharon recording: {0}")] + Parse(#[from] serde_json::Error), +} + +#[derive(Deserialize)] +struct VersionProbe { + format_version: u32, +} + +impl Recording { + /// Pretty JSON with a trailing newline. + pub fn to_json(&self) -> String { + let mut copy = self.clone(); + copy.format_version = FORMAT_VERSION; + let mut text = serde_json::to_string_pretty(©) + .expect("a Recording is plain data and always serializes"); + text.push('\n'); + text + } + + /// The version is checked before the full parse. + pub fn from_json(text: &str) -> Result { + let probe: VersionProbe = serde_json::from_str(text)?; + if probe.format_version != FORMAT_VERSION { + return Err(FormatError::UnsupportedVersion(probe.format_version)); + } + Ok(serde_json::from_str(text)?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::classify::{PasswordLeg, RungCeiling, Shape}; + use crate::request::{ + AuthShape, BodyShape, Encoding, FieldShape, JsonField, JsonShape, MarkerHit, ResourceKind, + ValueKind, ValueShape, + }; + + fn url(path: &str) -> StrippedUrl { + StrippedUrl { + origin: "https://example.test".into(), + path: path.into(), + query_keys: vec![], + } + } + + fn frame() -> FrameRef { + FrameRef { + main: true, + origin: "https://example.test".into(), + cross_origin: false, + } + } + + fn hit(role: Role) -> MarkerHit { + MarkerHit { + role, + encoding: Encoding::AsTyped, + low_confidence: false, + } + } + + fn value(markers: Vec) -> ValueShape { + ValueShape { + kind: ValueKind::String, + len: 8, + markers, + } + } + + fn request(body: BodyShape) -> StoredRequest { + StoredRequest { + t_ms: 120, + method: "POST".into(), + url: url("/login"), + resource: ResourceKind::Document, + main_frame_navigation: true, + content_type: Some("application/x-www-form-urlencoded".into()), + authorization: None, + url_markers: vec![], + body, + response: None, + } + } + + fn sample() -> Recording { + let element = ElementDescriptor { + element_id: 2, + tag: "input".into(), + input_type: Some("password".into()), + id: Some("pw".into()), + name: Some("password".into()), + autocomplete: None, + placeholder: None, + aria_label: None, + was_password: true, + form: Some(FormRef { + index: 0, + action: Some(url("/login")), + method: "post".into(), + }), + selectors: vec!["#pw".into()], + in_shadow_root: false, + }; + let body = BodyShape::Urlencoded { + fields: vec![FieldShape { + key: "password".into(), + value: value(vec![hit(Role::Password)]), + }], + }; + Recording { + format_version: FORMAT_VERSION, + tool_version: "0.0.0".into(), + browser: BrowserInfo { + product: "Chrome/1".into(), + user_agent: "UA".into(), + viewport: Viewport { + width: 1280, + height: 800, + }, + }, + listener_world: ListenerWorld::Isolated, + started_at: "2026-11-18T00:00:00Z".into(), + ended_at: "2026-11-18T00:01:00Z".into(), + ended_by: EndedBy::Confirmed, + success_confirmed: true, + start_url: url("/login"), + final_url: Some(url("/home")), + steps: vec![ + Step::Navigation { + t_ms: 0, + frame: frame(), + url: url("/login"), + }, + Step::Input { + t_ms: 50, + frame: frame(), + element, + value_len: 8, + role: Role::Password, + }, + ], + requests: vec![request(body)], + flags: Flags::default(), + classification: Classification { + shape: Shape::TwoStep { + password_leg: PasswordLeg::FormPost, + }, + rung_ceiling: RungCeiling::Milliseconds, + network_swap_candidate: false, + notes: vec![], + }, + stripped: StrippedCounts { + requests_seen: 3, + requests_stored: 1, + request_bodies_reduced: 1, + query_values_removed: 0, + path_segments_replaced: 0, + typed_values_removed: 2, + steps_dropped: 0, + }, + } + } + + fn request_with_three_hits() -> StoredRequest { + let nested = JsonShape::Object { + fields: vec![JsonField { + key: "auth".into(), + value: JsonShape::Array { + items: vec![JsonShape::Object { + fields: vec![JsonField { + key: "p".into(), + value: JsonShape::Value { + value: value(vec![hit(Role::Password)]), + }, + }], + }], + }, + }], + }; + let mut r = request(BodyShape::Json { root: nested }); + r.url_markers = vec![hit(Role::Username)]; + r.authorization = Some(AuthShape { + scheme: "Bearer".into(), + markers: vec![hit(Role::Password)], + }); + r + } + + #[test] + fn round_trips() { + let r = sample(); + assert_eq!(Recording::from_json(&r.to_json()).unwrap(), r); + } + + #[test] + fn round_trips_nested_classification_shape() { + let c = Classification { + shape: Shape::Transformed { + digest: Some("sha256".into()), + }, + rung_ceiling: RungCeiling::NeedsPerson, + network_swap_candidate: true, + notes: vec!["n".into()], + }; + let text = serde_json::to_string(&c).unwrap(); + assert!( + text.contains("\"shape\":{\"kind\":\"transformed\""), + "{text}" + ); + assert_eq!(serde_json::from_str::(&text).unwrap(), c); + } + + #[test] + fn refuses_an_unknown_version() { + let text = sample() + .to_json() + .replace("\"format_version\": 1", "\"format_version\": 2"); + assert!(matches!( + Recording::from_json(&text), + Err(FormatError::UnsupportedVersion(2)) + )); + } + + #[test] + fn refuses_a_file_with_no_version() { + assert!(matches!( + Recording::from_json("{}"), + Err(FormatError::Parse(_)) + )); + } + + #[test] + fn wire_names_are_snake_case_and_tagged() { + let j = sample().to_json(); + for needle in [ + "\"kind\": \"navigation\"", + "\"role\": \"password\"", + "\"kind\": \"urlencoded\"", + "\"encoding\": \"as_typed\"", + "\"ended_by\": \"confirmed\"", + ] { + assert!(j.contains(needle), "missing {needle}"); + } + } + + #[test] + fn hits_walks_url_authorization_and_nested_json() { + assert_eq!(request_with_three_hits().hits().len(), 3); + } + + #[test] + fn digest_names() { + assert!(Encoding::Sha256Hex.is_digest()); + assert_eq!(Encoding::Md5Base64.digest_name(), Some("md5")); + assert!(!Encoding::Hex.is_digest()); + } + + #[test] + fn to_json_always_writes_the_current_version() { + let mut r = sample(); + r.format_version = 99; + let back = Recording::from_json(&r.to_json()).unwrap(); + assert_eq!(back.format_version, FORMAT_VERSION); + } + + #[test] + fn every_variant_round_trips() { + let mut r = sample(); + let Step::Input { element, .. } = r.steps[1].clone() else { + unreachable!() + }; + let form = element.form.clone().unwrap(); + r.steps.extend([ + Step::Click { + t_ms: 60, + frame: frame(), + element: element.clone(), + }, + Step::Submit { + t_ms: 70, + frame: frame(), + form, + }, + Step::EnterKey { + t_ms: 80, + frame: frame(), + element, + }, + ]); + let json_root = JsonShape::Object { + fields: vec![JsonField { + key: "p".into(), + value: JsonShape::Value { + value: value(vec![hit(Role::Password)]), + }, + }], + }; + let bodies = [ + BodyShape::None, + BodyShape::Unavailable, + BodyShape::Json { root: json_root }, + BodyShape::Multipart { + parts: vec![FieldShape { + key: "f".into(), + value: value(vec![hit(Role::Username)]), + }], + }, + BodyShape::Opaque { + len: 9, + markers: vec![hit(Role::Password)], + }, + ]; + r.requests = bodies.into_iter().map(request).collect(); + let shapes = [ + Shape::FormPost, + Shape::ScriptPost, + Shape::TwoStep { + password_leg: PasswordLeg::ScriptPost, + }, + Shape::TwoStep { + password_leg: PasswordLeg::Transformed, + }, + Shape::Transformed { digest: None }, + Shape::Transformed { + digest: Some("md5".into()), + }, + Shape::Unrecognized, + ]; + for shape in shapes { + r.classification.shape = shape; + assert_eq!(Recording::from_json(&r.to_json()).unwrap(), r); + } + } + + #[test] + fn two_step_wire_form() { + let c = sample().classification; + let text = serde_json::to_string(&c).unwrap(); + assert!( + text.contains(r#""shape":{"kind":"two_step","password_leg":"form_post"}"#), + "{text}" + ); + } + + #[test] + fn hits_covers_multipart_and_opaque() { + let m = request(BodyShape::Multipart { + parts: vec![ + FieldShape { + key: "a".into(), + value: value(vec![hit(Role::Username)]), + }, + FieldShape { + key: "b".into(), + value: value(vec![hit(Role::Password)]), + }, + ], + }); + assert_eq!(m.hits().len(), 2); + let o = request(BodyShape::Opaque { + len: 4, + markers: vec![hit(Role::Password)], + }); + assert_eq!(o.hits().len(), 1); + } +} diff --git a/crates/kharon-recording/src/lib.rs b/crates/kharon-recording/src/lib.rs new file mode 100644 index 0000000..5e8a110 --- /dev/null +++ b/crates/kharon-recording/src/lib.rs @@ -0,0 +1,17 @@ +//! Typed recording format for kharon teach mode (#361). + +pub mod classify; +pub mod format; +pub mod request; + +pub use classify::{ + Classification, Flags, PasswordLeg, RungCeiling, Shape, classify, is_otp_descriptor, +}; +pub use format::{ + BrowserInfo, ElementDescriptor, EndedBy, FORMAT_VERSION, FormRef, FormatError, FrameRef, + ListenerWorld, Recording, Step, StrippedCounts, StrippedUrl, Viewport, +}; +pub use request::{ + AuthShape, BodyShape, Deciding, Encoding, FieldShape, JsonField, JsonShape, MarkerHit, + ResourceKind, ResponseShape, Role, StoredRequest, ValueKind, ValueShape, +}; diff --git a/crates/kharon-recording/src/request.rs b/crates/kharon-recording/src/request.rs new file mode 100644 index 0000000..422de50 --- /dev/null +++ b/crates/kharon-recording/src/request.rs @@ -0,0 +1,250 @@ +//! Reduced request shapes. Values are never stored, only lengths and marker hits. + +use serde::{Deserialize, Serialize}; + +use crate::format::StrippedUrl; + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Role { + None, + Username, + Password, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Encoding { + AsTyped, + Percent, + JsonEscaped, + Base64, + Base64Url, + Hex, + Md5Hex, + Md5Base64, + Sha1Hex, + Sha1Base64, + Sha256Hex, + Sha256Base64, + Sha512Hex, + Sha512Base64, +} + +impl Encoding { + pub fn is_digest(self) -> bool { + self.digest_name().is_some() + } + + pub fn digest_name(self) -> Option<&'static str> { + match self { + Self::Md5Hex | Self::Md5Base64 => Some("md5"), + Self::Sha1Hex | Self::Sha1Base64 => Some("sha1"), + Self::Sha256Hex | Self::Sha256Base64 => Some("sha256"), + Self::Sha512Hex | Self::Sha512Base64 => Some("sha512"), + Self::AsTyped + | Self::Percent + | Self::JsonEscaped + | Self::Base64 + | Self::Base64Url + | Self::Hex => None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct MarkerHit { + pub role: Role, + pub encoding: Encoding, + /// True when the match is weak (the marker is shorter than `LOW_CONFIDENCE_BELOW` characters), so a reader treats it as a hint. + pub low_confidence: bool, +} + +/// Which weak hits still decide anything. A weak (`low_confidence`) hit of a role counts only +/// when no confident hit of that role was found anywhere in the recording: a short value matches +/// by chance in unrelated requests, and must not outvote the request that really carried it. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct Deciding { + confident_username: bool, + confident_password: bool, +} + +impl Deciding { + /// Looks at every hit given. + pub fn from_hits<'a>(hits: impl IntoIterator) -> Self { + let mut d = Self::default(); + for h in hits.into_iter().filter(|h| !h.low_confidence) { + match h.role { + Role::Username => d.confident_username = true, + Role::Password => d.confident_password = true, + Role::None => {} + } + } + d + } + + /// Whether `hit` decides anything. A hit with no role counts only when confident. + pub fn counts(&self, hit: &MarkerHit) -> bool { + let confident_elsewhere = match hit.role { + Role::Username => self.confident_username, + Role::Password => self.confident_password, + Role::None => true, + }; + !hit.low_confidence || !confident_elsewhere + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ValueKind { + String, + Number, + Bool, + Null, + File, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ValueShape { + pub kind: ValueKind, + /// Characters for a string value, bytes for a file. + pub len: u32, + pub markers: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct FieldShape { + pub key: String, + pub value: ValueShape, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum JsonShape { + Object { fields: Vec }, + Array { items: Vec }, + Value { value: ValueShape }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct JsonField { + pub key: String, + pub value: JsonShape, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum BodyShape { + None, + Unavailable, + Urlencoded { + fields: Vec, + }, + Json { + root: JsonShape, + }, + Multipart { + parts: Vec, + }, + Opaque { + /// Bytes. + len: u64, + markers: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AuthShape { + pub scheme: String, + pub markers: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ResourceKind { + Document, + Xhr, + Fetch, + Other, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ResponseShape { + pub status: u16, + pub content_type: Option, + pub set_cookie: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StoredRequest { + /// Milliseconds since the recording started. + pub t_ms: u64, + pub method: String, + pub url: StrippedUrl, + pub resource: ResourceKind, + pub main_frame_navigation: bool, + pub content_type: Option, + pub authorization: Option, + pub url_markers: Vec, + pub body: BodyShape, + pub response: Option, +} + +impl JsonShape { + fn collect<'a>(&'a self, out: &mut Vec<&'a MarkerHit>) { + match self { + Self::Object { fields } => fields.iter().for_each(|f| f.value.collect(out)), + Self::Array { items } => items.iter().for_each(|i| i.collect(out)), + Self::Value { value } => out.extend(&value.markers), + } + } +} + +impl StoredRequest { + /// Every marker hit in the URL, the authorization header and every level of the body. + pub fn hits(&self) -> Vec<&MarkerHit> { + let mut out: Vec<&MarkerHit> = self.url_markers.iter().collect(); + if let Some(auth) = &self.authorization { + out.extend(&auth.markers); + } + match &self.body { + BodyShape::None | BodyShape::Unavailable => {} + BodyShape::Urlencoded { fields } | BodyShape::Multipart { parts: fields } => { + out.extend(fields.iter().flat_map(|f| &f.value.markers)); + } + BodyShape::Json { root } => root.collect(&mut out), + BodyShape::Opaque { markers, .. } => out.extend(markers), + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn h(role: Role, low_confidence: bool) -> MarkerHit { + MarkerHit { + role, + encoding: Encoding::AsTyped, + low_confidence, + } + } + + #[test] + fn a_weak_hit_counts_only_when_no_confident_hit_of_its_role_exists() { + let all = [ + h(Role::Password, true), + h(Role::Password, false), + h(Role::Username, true), + ]; + let d = Deciding::from_hits(all.iter()); + assert!(!d.counts(&h(Role::Password, true))); + assert!(d.counts(&h(Role::Password, false))); + // No confident username hit anywhere: the weak one decides. + assert!(d.counts(&h(Role::Username, true))); + // A hit with no role counts only when confident. + assert!(d.counts(&h(Role::None, false))); + assert!(!d.counts(&h(Role::None, true))); + } +} diff --git a/crates/kharon-teach/Cargo.toml b/crates/kharon-teach/Cargo.toml new file mode 100644 index 0000000..24bce9c --- /dev/null +++ b/crates/kharon-teach/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "kharon-teach" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true +description = "Teach-mode recorder for kharon (#361)" + +[lib] +name = "kharon_teach" +path = "src/lib.rs" + +[[bin]] +name = "kharon-teach" +path = "src/main.rs" + +[dependencies] +kharon-recording = { version = "0.0.0", path = "../kharon-recording" } +zeroize = { version = "1", features = ["serde"] } +sha1 = "0.10" +sha2 = "0.10" +md-5 = "0.10" +base64ct = { version = "1", features = ["alloc"] } +hex = "0.4" +percent-encoding = "2" +memchr = "2" +url = "2" +idna = "1" +serde_json = { version = "1", features = ["raw_value"] } +serde = { version = "1", features = ["derive"] } +thiserror = "2" +rand = "0.8" +tokio = { version = "1", features = ["fs", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time", "io-std", "io-util"] } +publicsuffix = "2.3" +tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "handshake"] } +time = { version = "0.3", features = ["formatting"] } +clap = { version = "4", features = ["derive"] } +futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } + +[dev-dependencies] +tokio = { version = "1", features = ["test-util"] } + +[target.'cfg(unix)'.dependencies] +rustix = { version = "1", default-features = false, features = ["std", "process"] } diff --git a/crates/kharon-teach/src/body.rs b/crates/kharon-teach/src/body.rs new file mode 100644 index 0000000..c750e5d --- /dev/null +++ b/crates/kharon-teach/src/body.rs @@ -0,0 +1,1201 @@ +//! Request bodies reduced to shapes: where a typed value sat, never what it was. +//! +//! The site's JSON body is read as a `serde_json::Value` here, the one place a site payload may +//! be, while its shape is extracted. Known limit: the parsed `Value` holds typed values in +//! ordinary `String`s that serde_json does not zeroize. + +use kharon_recording::{ + BodyShape, FieldShape, JsonField, JsonShape, MarkerHit, ValueKind, ValueShape, +}; +use memchr::memmem; +use zeroize::Zeroizing; + +use crate::marker::Marker; +use crate::marker::percent_decode as decode_percent; +use crate::strip::{OTHER, TYPED, dedup_hits, hits_in, holds_typed, looks_like_a_name}; + +/// Bodies larger than this are not parsed: they are `Opaque`. +pub const MAX_BODY_BYTES: usize = 1 << 20; + +/// A body whose shape would hold more than this many values (JSON nodes, urlencoded fields, +/// multipart parts) is `Opaque`: a page can send telemetry batches of any size, and a shape +/// as large as the body helps nobody. +pub const MAX_SHAPE_VALUES: usize = 2_000; + +/// What is left of the value budget for one body. Building stops as soon as it runs out. +struct Budget(usize); + +impl Budget { + fn new() -> Self { + Self(MAX_SHAPE_VALUES) + } + + /// Takes one value, or `None` when none is left. + fn take(&mut self) -> Option<()> { + self.0 = self.0.checked_sub(1)?; + Some(()) + } +} + +/// `content_type` picks the reader: application/x-www-form-urlencoded, application/json (and any +json), +/// multipart/form-data; anything else, or a body that fails to parse as its declared type, is Opaque. +/// `pair` is (username, password) once both are known; see below. +/// +/// Values are searched after decoding (percent-decoded urlencoded value with `+` as space, a JSON +/// string's content, a JSON number's text, a multipart text part's bytes). An `Opaque` body is +/// searched as raw bytes. A name (key, part name) holding a typed value is written as `{typed}` +/// and its hits are added to its field; a name that does not look like a name (see +/// `strip::looks_like_a_name`) is written as `{other}`. A urlencoded body with a piece that has +/// no `=`, or whose names mostly do not look like names, is not urlencoded data and is `Opaque`. +/// So is a body whose shape would hold more than `MAX_SHAPE_VALUES` values. +/// Never returns `None` or `Unavailable`. +pub fn shape_of_body( + content_type: Option<&str>, + body: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> BodyShape { + let parsed = match content_type.map(ContentType::parse) { + Some(ct) if body.len() <= MAX_BODY_BYTES => match ct.essence.as_str() { + "application/x-www-form-urlencoded" => urlencoded_fields(body, markers, pair) + .map(|fields| BodyShape::Urlencoded { fields }), + "multipart/form-data" => ct + .boundary + .as_deref() + .and_then(|b| multipart_parts(body, b, markers, pair)) + .map(|parts| BodyShape::Multipart { parts }), + e if e == "application/json" || e.ends_with("+json") => { + // The one permitted use of `serde_json::Value`: a site's JSON body while its shape is extracted. + serde_json::from_slice::(body) + .ok() + .and_then(|v| json_shape(&v, markers, pair, &mut Budget::new())) + .map(|root| BodyShape::Json { root }) + } + _ => None, + }, + _ => None, + }; + parsed.unwrap_or_else(|| BodyShape::Opaque { + len: u64::try_from(body.len()).unwrap_or(u64::MAX), + markers: hits_in(body, markers, pair), + }) +} + +struct ContentType { + /// Lowercased, without parameters. + essence: String, + boundary: Option, +} + +impl ContentType { + fn parse(raw: &str) -> Self { + let (essence, params) = raw.split_once(';').unwrap_or((raw, "")); + let boundary = parameters(params) + .into_iter() + .find(|(k, _)| k == "boundary") + .map(|(_, v)| v) + .filter(|v| !v.is_empty()); + Self { + essence: essence.trim().to_ascii_lowercase(), + boundary, + } + } +} + +/// `; key=value; key="quoted value"` pairs, keys lowercased. Quotes may hold `;` and `\`-escapes. +fn parameters(text: &str) -> Vec<(String, String)> { + let mut out = Vec::new(); + let mut chars = text.chars().peekable(); + loop { + let mut key = String::new(); + while let Some(c) = chars.next_if(|c| *c != '=' && *c != ';') { + key.push(c); + } + let mut value = String::new(); + if chars.next_if_eq(&'=').is_some() { + if chars.next_if_eq(&'"').is_some() { + while let Some(c) = chars.next() { + match c { + '"' => break, + '\\' => value.extend(chars.next()), + c => value.push(c), + } + } + // Anything up to the next `;` after the closing quote is ignored. + while chars.next_if(|c| *c != ';').is_some() {} + } else { + while let Some(c) = chars.next_if(|c| *c != ';') { + value.push(c); + } + } + } + let key = key.trim().to_ascii_lowercase(); + if !key.is_empty() { + out.push((key, value.trim().to_owned())); + } + if chars.next().is_none() { + return out; + } + } +} + +/// Characters in `bytes` read as UTF-8, each invalid run counting as one. Makes no copy. +fn char_len(bytes: &[u8]) -> u32 { + let n: usize = bytes + .utf8_chunks() + .map(|c| c.valid().chars().count() + usize::from(!c.invalid().is_empty())) + .sum(); + u32::try_from(n).unwrap_or(u32::MAX) +} + +fn byte_len(bytes: &[u8]) -> u32 { + u32::try_from(bytes.len()).unwrap_or(u32::MAX) +} + +fn value_shape( + kind: ValueKind, + len: u32, + searched: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> ValueShape { + ValueShape { + kind, + len, + markers: hits_in(searched, markers, pair), + } +} + +fn text_value(bytes: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> ValueShape { + value_shape(ValueKind::String, char_len(bytes), bytes, markers, pair) +} + +/// A name as it is written to the file, and the marker hits found in it. A name that holds a +/// typed value (any encoding, or the text in any letter case, raw or percent-decoded; see +/// `strip::holds_typed`) is `{typed}`, and its hits are returned so the caller can attach them +/// to the field. A name that is not a typed value is kept only when it looks like a name, else +/// it is `{other}`. +fn name( + bytes: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> (String, Vec) { + if holds_typed(bytes, markers, pair) { + return (TYPED.to_owned(), hits_in(bytes, markers, pair)); + } + let text = String::from_utf8_lossy(bytes); + if looks_like_a_name(&text) { + (text.into_owned(), Vec::new()) + } else { + (OTHER.to_owned(), Vec::new()) + } +} + +/// Adds `extra` to `value.markers` without repeating a (role, encoding). +fn add_hits(value: &mut ValueShape, extra: Vec) { + if extra.is_empty() { + return; + } + let mut all = std::mem::take(&mut value.markers); + all.extend(extra); + value.markers = dedup_hits(all); +} + +/// The urlencoded fields, or `None` when the body is not urlencoded data: some piece has no `=`, +/// or more than half of the keys do not look like names. (jQuery sends JSON text under a +/// urlencoded content type; read as fields, the whole JSON would be one key.) +fn urlencoded_fields( + body: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> Option> { + let mut budget = Budget::new(); + let mut pieces = Vec::new(); + for piece in body.split(|b| *b == b'&').filter(|p| !p.is_empty()) { + budget.take()?; + let i = piece.iter().position(|b| *b == b'=')?; + pieces.push((decode_percent(&piece[..i], true), &piece[i + 1..])); + } + let odd_keys = pieces + .iter() + .filter(|(k, _)| !looks_like_a_name(&String::from_utf8_lossy(k))) + .count(); + if odd_keys * 2 > pieces.len() { + return None; + } + Some( + pieces + .into_iter() + .map(|(key, v)| { + let value = decode_percent(v, true); + let (key, key_hits) = name(&key, markers, pair); + let mut value = text_value(&value, markers, pair); + add_hits(&mut value, key_hits); + FieldShape { key, value } + }) + .collect(), + ) +} + +/// Adds `extra` to the first scalar value under `shape`, depth first. The format gives a JSON +/// object key no marker list of its own, so the hits of a typed value found in a key go to the +/// value under it, or to the first value inside it when that is an object or an array. An empty +/// object or array has nowhere to hold them; those hits are not recorded. +fn add_hits_to_first_value(shape: &mut JsonShape, extra: Vec) -> bool { + match shape { + JsonShape::Value { value } => { + add_hits(value, extra); + true + } + JsonShape::Array { items } => items + .iter_mut() + .any(|i| add_hits_to_first_value(i, extra.clone())), + JsonShape::Object { fields } => fields + .iter_mut() + .any(|f| add_hits_to_first_value(&mut f.value, extra.clone())), + } +} + +/// The shape of `v`, or `None` when it would hold more values than `budget` has left. +fn json_shape( + v: &serde_json::Value, + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, + budget: &mut Budget, +) -> Option { + use serde_json::Value; + budget.take()?; + // Depth is bounded by serde_json's own recursion limit (128) at parse time. + Some(match v { + Value::Object(map) => { + let mut fields = Vec::new(); + for (k, v) in map { + let (key, key_hits) = name(k.as_bytes(), markers, pair); + let mut value = json_shape(v, markers, pair, budget)?; + add_hits_to_first_value(&mut value, key_hits); + fields.push(JsonField { key, value }); + } + JsonShape::Object { fields } + } + Value::Array(items) => { + let mut shapes = Vec::new(); + for i in items { + shapes.push(json_shape(i, markers, pair, budget)?); + } + JsonShape::Array { items: shapes } + } + Value::String(s) => JsonShape::Value { + value: text_value(s.as_bytes(), markers, pair), + }, + Value::Number(n) => { + let text = Zeroizing::new(n.to_string()); + JsonShape::Value { + value: value_shape( + ValueKind::Number, + char_len(text.as_bytes()), + text.as_bytes(), + markers, + pair, + ), + } + } + Value::Bool(b) => JsonShape::Value { + value: value_shape( + ValueKind::Bool, + 0, + if *b { b"true" } else { b"false" }, + markers, + pair, + ), + }, + Value::Null => JsonShape::Value { + value: value_shape(ValueKind::Null, 0, b"null", markers, pair), + }, + }) +} + +fn find(haystack: &[u8], needle: &[u8], from: usize) -> Option { + if needle.is_empty() || from > haystack.len() { + return None; + } + memmem::find(&haystack[from..], needle).map(|i| from + i) +} + +/// Parts of a `multipart/form-data` body, or `None` when it is not well formed (no opening +/// delimiter, a part with no header end, or a part not closed by a delimiter). +/// +/// The line ending (CRLF or LF) is read once, from the first delimiter line, and used for every +/// part, so each search stops at its own terminator and the whole parse is linear in the body. +fn multipart_parts( + body: &[u8], + boundary: &str, + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> Option> { + let mut delim = b"--".to_vec(); + delim.extend_from_slice(boundary.as_bytes()); + + let mut pos = find(body, &delim, 0)? + delim.len(); + let first_newline = pos + body[pos..].iter().position(|b| *b == b'\n')?; + let eol: &[u8] = if first_newline > pos && body[first_newline - 1] == b'\r' { + b"\r\n" + } else { + b"\n" + }; + let blank_line = [eol, eol].concat(); + // The delimiter that closes a part is preceded by a line break. + let closing = [eol, delim.as_slice()].concat(); + + let mut budget = Budget::new(); + let mut parts = Vec::new(); + loop { + if body[pos..].starts_with(b"--") { + return Some(parts); + } + budget.take()?; + // Rest of the delimiter line (transport padding), then the headers. + let headers_start = pos + body[pos..].iter().position(|b| *b == b'\n')? + 1; + let headers_end = find(body, &blank_line, headers_start)?; + let content_start = headers_end + blank_line.len(); + let headers = &body[headers_start..headers_end]; + // The line break that ends an empty content is the blank line's own. + let content_end = find(body, &closing, content_start - eol.len())?; + let content = body.get(content_start..content_end).unwrap_or(&[]); + parts.push(part_shape(headers, content, markers, pair)); + pos = content_end + closing.len(); + } +} + +fn part_shape( + headers: &[u8], + content: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> FieldShape { + let mut part_name = Vec::new(); + let mut is_file = false; + for line in headers.split(|b| *b == b'\n') { + let line = String::from_utf8_lossy(line); + let Some((header, rest)) = line.split_once(':') else { + continue; + }; + if !header.trim().eq_ignore_ascii_case("content-disposition") { + continue; + } + for (key, value) in parameters(rest) { + match key.as_str() { + "name" => part_name = value.into_bytes(), + "filename" | "filename*" => is_file = true, + _ => {} + } + } + } + let value = if is_file { + ValueShape { + kind: ValueKind::File, + len: byte_len(content), + markers: Vec::new(), + } + } else { + text_value(content, markers, pair) + }; + let (key, key_hits) = name(&part_name, markers, pair); + let mut value = value; + add_hits(&mut value, key_hits); + FieldShape { key, value } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::strip::{reduce_authorization, strip_url}; + use crate::testing::{assert_holds_no_typed_value, pw, user}; + use base64ct::{Base64, Encoding as B64}; + use kharon_recording::{Encoding, MarkerHit, Role}; + + fn markers() -> Vec { + vec![user("kh.throwaway7"), pw("Tr0ub4dor-9x")] + } + fn hit(role: Role, encoding: Encoding) -> MarkerHit { + MarkerHit { + role, + encoding, + low_confidence: false, + } + } + fn shape(ct: &str, body: &[u8]) -> BodyShape { + shape_of_body(Some(ct), body, &markers(), None) + } + fn vs(kind: ValueKind, len: u32, markers: Vec) -> ValueShape { + ValueShape { kind, len, markers } + } + fn field(key: &str, value: ValueShape) -> FieldShape { + FieldShape { + key: key.into(), + value, + } + } + fn leaf(value: ValueShape) -> JsonShape { + JsonShape::Value { value } + } + fn jf(key: &str, value: JsonShape) -> JsonField { + JsonField { + key: key.into(), + value, + } + } + const URLENC: &str = "application/x-www-form-urlencoded"; + + #[test] + fn urlencoded_fields_with_roles() { + let s = shape( + URLENC, + b"user=kh.throwaway7&password=Tr0ub4dor-9x&remember=1", + ); + assert_eq!( + s, + BodyShape::Urlencoded { + fields: vec![ + field( + "user", + vs( + ValueKind::String, + 13, + vec![hit(Role::Username, Encoding::AsTyped)] + ) + ), + field( + "password", + vs( + ValueKind::String, + 12, + vec![hit(Role::Password, Encoding::AsTyped)] + ) + ), + field("remember", vs(ValueKind::String, 1, vec![])), + ] + } + ); + } + + #[test] + fn urlencoded_value_is_searched_after_decoding() { + let s = shape(URLENC, b"password=Tr0ub4dor%2D9x&q=a+b%zz"); + let BodyShape::Urlencoded { fields } = s else { + panic!("not urlencoded") + }; + assert_eq!( + fields[0].value.markers, + [hit(Role::Password, Encoding::AsTyped)] + ); + // "+" is a space; a lone "%zz" stays as typed: "a b%zz" is 6 characters. + assert_eq!(fields[1].value.len, 6); + } + + #[test] + fn json_nested() { + let s = shape( + "Application/JSON; charset=utf-8", + br#"{"auth":{"id":"kh.throwaway7","secret":"Tr0ub4dor-9x"},"n":3,"ok":true,"z":null}"#, + ); + let BodyShape::Json { root } = s else { + panic!("not json") + }; + let JsonShape::Object { mut fields } = root else { + panic!("not an object") + }; + fields.sort_by(|a, b| a.key.cmp(&b.key)); + let want = vec![ + jf( + "auth", + JsonShape::Object { + fields: vec![ + jf( + "id", + leaf(vs( + ValueKind::String, + 13, + vec![hit(Role::Username, Encoding::AsTyped)], + )), + ), + jf( + "secret", + leaf(vs( + ValueKind::String, + 12, + vec![hit(Role::Password, Encoding::AsTyped)], + )), + ), + ], + }, + ), + jf("n", leaf(vs(ValueKind::Number, 1, vec![]))), + jf("ok", leaf(vs(ValueKind::Bool, 0, vec![]))), + jf("z", leaf(vs(ValueKind::Null, 0, vec![]))), + ]; + let JsonShape::Object { fields: mut inner } = fields[0].value.clone() else { + panic!() + }; + inner.sort_by(|a, b| a.key.cmp(&b.key)); + fields[0].value = JsonShape::Object { fields: inner }; + assert_eq!(fields, want); + } + + #[test] + fn json_arrays_and_scalars_and_plus_json() { + let s = shape("application/vnd.api+json", br#"["Tr0ub4dor-9x", 12]"#); + assert_eq!( + s, + BodyShape::Json { + root: JsonShape::Array { + items: vec![ + leaf(vs( + ValueKind::String, + 12, + vec![hit(Role::Password, Encoding::AsTyped)] + )), + leaf(vs(ValueKind::Number, 2, vec![])), + ] + } + } + ); + } + + #[test] + fn a_json_number_holding_a_typed_value_is_a_hit() { + let m = [pw("424242")]; + let s = shape_of_body(Some("application/json"), br#"{"pin":424242}"#, &m, None); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = s + else { + panic!() + }; + assert_eq!( + fields[0].value, + leaf(vs( + ValueKind::Number, + 6, + vec![hit(Role::Password, Encoding::AsTyped)] + )) + ); + } + + #[test] + fn graphql_is_json() { + let s = shape( + "application/json", + br#"{"query":"mutation Login($password: String!) { login(password: $password) }","variables":{"password":"Tr0ub4dor-9x"}}"#, + ); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = s + else { + panic!() + }; + let vars = fields.iter().find(|f| f.key == "variables").unwrap(); + let JsonShape::Object { fields: inner } = &vars.value else { + panic!() + }; + assert_eq!(inner[0].key, "password"); + assert_eq!( + inner[0].value, + leaf(vs( + ValueKind::String, + 12, + vec![hit(Role::Password, Encoding::AsTyped)] + )) + ); + } + + #[test] + fn a_json_key_that_holds_a_typed_value_is_not_kept() { + let s = shape( + "application/json", + br#"{"users":{"kh.throwaway7":{"secret":"Tr0ub4dor-9x"}}}"#, + ); + let BodyShape::Json { root } = s else { + panic!() + }; + let text = serde_json::to_string(&root).unwrap(); + assert!(text.contains(r#""key":"{typed}""#), "{text}"); + assert!(text.contains(r#""key":"secret""#), "{text}"); + assert!(!text.contains("throwaway"), "{text}"); + assert!(text.contains("password"), "{text}"); + } + + #[test] + fn multipart_text_and_file() { + let body = b"--XyZ\r\nContent-Disposition: form-data; name=\"password\"\r\n\r\nTr0ub4dor-9x\r\n--XyZ\r\nContent-Disposition: form-data; name=\"avatar\"; filename=\"me.png\"\r\nContent-Type: image/png\r\n\r\nkh.throwaway7\r\n--XyZ--\r\n"; + let s = shape("multipart/form-data; boundary=XyZ", body); + assert_eq!( + s, + BodyShape::Multipart { + parts: vec![ + field( + "password", + vs( + ValueKind::String, + 12, + vec![hit(Role::Password, Encoding::AsTyped)] + ) + ), + field("avatar", vs(ValueKind::File, 13, vec![])), + ] + } + ); + } + + #[test] + fn a_multipart_part_name_that_holds_a_typed_value_is_not_kept() { + let body = + b"--b\r\nContent-Disposition: form-data; name=\"kh.throwaway7\"\r\n\r\nx\r\n--b--"; + let s = shape("multipart/form-data; boundary=\"b\"", body); + let BodyShape::Multipart { parts } = s else { + panic!() + }; + assert_eq!(parts[0].key, "{typed}"); + } + + #[test] + fn urlencoded_key_that_holds_a_typed_value_is_not_kept() { + let s = shape(URLENC, b"kh.throwaway7=1"); + let BodyShape::Urlencoded { fields } = s else { + panic!() + }; + assert_eq!(fields[0].key, "{typed}"); + } + + #[test] + fn unknown_type_is_opaque_with_whole_body_hit() { + let s = shape("application/octet-stream", b"xx Tr0ub4dor-9x yy"); + assert_eq!( + s, + BodyShape::Opaque { + len: 18, + markers: vec![hit(Role::Password, Encoding::AsTyped)] + } + ); + assert!(matches!( + shape_of_body(None, b"a=b", &[], None), + BodyShape::Opaque { len: 3, .. } + )); + } + + #[test] + fn bad_json_falls_back_to_opaque() { + assert!(matches!( + shape("application/json", b"{not json"), + BodyShape::Opaque { len: 9, .. } + )); + assert!(matches!( + shape("application/json", b""), + BodyShape::Opaque { len: 0, .. } + )); + assert_eq!(shape(URLENC, b""), BodyShape::Urlencoded { fields: vec![] }); + } + + #[test] + fn a_combined_credential_lands_on_the_value_that_holds_it() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let body = format!(r#"{{"auth":"{b64}","n":1}}"#); + let (u, p) = (user("kh.throwaway7"), pw("Tr0ub4dor-9x")); + let with = shape_of_body( + Some("application/json"), + body.as_bytes(), + &markers(), + Some((&u, &p)), + ); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = with + else { + panic!() + }; + let auth = fields.iter().find(|f| f.key == "auth").unwrap(); + assert_eq!( + auth.value, + leaf(vs( + ValueKind::String, + u32::try_from(b64.len()).unwrap(), + vec![ + hit(Role::Username, Encoding::Base64), + hit(Role::Password, Encoding::Base64) + ] + )) + ); + let without = shape("application/json", body.as_bytes()); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = without + else { + panic!() + }; + let auth = fields.iter().find(|f| f.key == "auth").unwrap(); + let JsonShape::Value { value } = &auth.value else { + panic!() + }; + assert!(value.markers.is_empty()); + } + + #[test] + fn an_opaque_body_holds_a_combined_credential_at_whole_body_level() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let (u, p) = (user("kh.throwaway7"), pw("Tr0ub4dor-9x")); + let s = shape_of_body(None, b64.as_bytes(), &[], Some((&u, &p))); + let BodyShape::Opaque { markers, .. } = s else { + panic!() + }; + assert_eq!(markers.len(), 2); + } + + #[test] + fn oversized_bodies_are_opaque() { + let big = vec![b'a'; MAX_BODY_BYTES + 1]; + assert!(matches!( + shape("application/json", &big), + BodyShape::Opaque { .. } + )); + } + + #[test] + fn hostile_input_never_panics() { + let mut nested = "[".repeat(200); + nested.push_str(&"]".repeat(200)); + assert!(matches!( + shape("application/json", nested.as_bytes()), + BodyShape::Opaque { .. } + )); + for ct in [ + "multipart/form-data; boundary=XyZ", + "multipart/form-data; boundary=", + "multipart/form-data", + "multipart/form-data; boundary=\"unterminated", + URLENC, + "application/json", + ] { + for body in [ + &b"--XyZ\r\nContent-Disposition: form-data; name=\"a"[..], + b"--XyZ\r\n\r\n", + b"--XyZ\r\nContent-Disposition: form-data; name=\"a\"\r\n\r\nvalue", + b"--XyZ--", + b"%", + b"a=%&b=%4&c=%zz", + b"\xff\xfe=\xc3\x28&\xf0\x9f", + b"{\"a\":\"\xff\"}", + b"", + ] { + let _ = shape(ct, body); + } + } + let _ = strip_url("https://ex.test/%", &markers(), None); + let _ = strip_url("https://ex.test/%zz?%=%&%ff", &markers(), None); + let _ = strip_url("data:text/plain,Tr0ub4dor-9x", &markers(), None); + let _ = reduce_authorization(b"Basic \xff\xfe", &markers(), None); + let _ = reduce_authorization(b"", &markers(), None); + let _ = reduce_authorization(b" ", &markers(), None); + } + + #[test] + fn a_site_cased_key_and_name_is_not_kept() { + let m = [user("Bob.Smith@Ex.Test")]; + // Two keys hold the value; two are ordinary, so the body is still urlencoded data. + let s = shape_of_body( + Some(URLENC), + b"a=1&b=2&bob.smith%40ex.test=1&BOB.SMITH@EX.TEST=2", + &m, + None, + ); + let BodyShape::Urlencoded { fields } = s else { + panic!() + }; + assert_eq!( + fields.iter().map(|f| f.key.as_str()).collect::>(), + ["a", "b", "{typed}", "{typed}"] + ); + // Decision change: a body whose keys all fail to look like names is Opaque. (The match + // is case-sensitive, so a site-cased value gives no hit here: a known limit.) + let s = shape_of_body( + Some(URLENC), + b"bob.smith%40ex.test=1&BOB.SMITH@EX.TEST=2", + &m, + None, + ); + assert!(matches!(s, BodyShape::Opaque { .. }), "not opaque"); + let s = shape_of_body( + Some("application/json"), + br#"{"bob.smith@ex.test":1}"#, + &m, + None, + ); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = s + else { + panic!() + }; + assert_eq!(fields[0].key, "{typed}"); + let body = + b"--b\r\nContent-Disposition: form-data; name=\"BOB.smith@ex.test\"\r\n\r\nx\r\n--b--"; + let s = shape_of_body(Some("multipart/form-data; boundary=b"), body, &m, None); + let BodyShape::Multipart { parts } = s else { + panic!() + }; + assert_eq!(parts[0].key, "{typed}"); + } + + #[test] + fn a_name_holding_the_combined_credential_is_not_kept() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let (u, p) = (user("kh.throwaway7"), pw("Tr0ub4dor-9x")); + let body = format!("{b64}=1"); + let s = shape_of_body(Some(URLENC), body.as_bytes(), &[], Some((&u, &p))); + let BodyShape::Urlencoded { fields } = s else { + panic!() + }; + assert_eq!(fields[0].key, "{typed}"); + } + + #[test] + fn many_crlf_parts_are_read_in_linear_time() { + // The earlier quadratic parse (CRLF bodies never matched the LF-only terminator, so every + // part scanned the rest of the body) took minutes at 20,000 parts. Decision change: a + // body of more parts than `MAX_SHAPE_VALUES` is now Opaque, so this is the most parts + // that are still read. + let mut body = Vec::new(); + for _ in 0..MAX_SHAPE_VALUES { + body.extend_from_slice(b"--b\r\nContent-Disposition: form-data; name=f\r\n\r\nv\r\n"); + } + body.extend_from_slice(b"--b--\r\n"); + assert!( + body.len() <= MAX_BODY_BYTES, + "the body must be parsed, not Opaque" + ); + // No markers: this measures the parse, not the per-value search. + let started = std::time::Instant::now(); + let s = shape_of_body(Some("multipart/form-data; boundary=b"), &body, &[], None); + let BodyShape::Multipart { parts } = s else { + panic!("not multipart") + }; + assert_eq!(parts.len(), MAX_SHAPE_VALUES); + // LF-only bodies too. + let lf = String::from_utf8_lossy(&body).replace("\r\n", "\n"); + let s = shape_of_body( + Some("multipart/form-data; boundary=b"), + lf.as_bytes(), + &[], + None, + ); + let BodyShape::Multipart { parts } = s else { + panic!("not multipart") + }; + assert_eq!(parts.len(), MAX_SHAPE_VALUES); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "multipart parse is not linear" + ); + } + + #[test] + fn serialized_shapes_never_contain_a_marker() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let values = [ + "kh.throwaway7", + "Tr0ub4dor-9x", + "Tr0ub4/dor", + "Bob.Smith@Ex.Test", + "Zed.Quincy9", + "ab?cd#ef@gh", + ]; + let m: Vec = vec![ + user(values[0]), + pw(values[1]), + pw(values[2]), + user(values[3]), + user(values[4]), + pw(values[5]), + ]; + let (u, p) = (user(values[0]), pw(values[1])); + let pair = Some((&u, &p)); + let json_pair = format!(r#"{{"auth":"{b64}","n":1}}"#); + let multipart = b"--XyZ\r\nContent-Disposition: form-data; name=\"password\"\r\n\r\nTr0ub4dor-9x\r\n--XyZ\r\nContent-Disposition: form-data; name=\"kh.throwaway7\"; filename=\"kh.throwaway7\"\r\n\r\nkh.throwaway7\r\n--XyZ\r\nContent-Disposition: form-data; name=\"bob.smith@ex.test\"\r\n\r\nTr0ub4/dor\r\n--XyZ--\r\n"; + let body = |ct: &str, bytes: &[u8]| shape_of_body(Some(ct), bytes, &m, pair); + let bodies: Vec = + vec![ + body(URLENC, b"user=kh.throwaway7&password=Tr0ub4dor-9x&remember=1"), + body(URLENC, b"password=Tr0ub4dor%2D9x&kh.throwaway7=1&p=Tr0ub4%2Fdor"), + body(URLENC, b"bob.smith%40ex.test=1&BOB.SMITH@EX.TEST=2"), + body(URLENC, format!("{b64}=1").as_bytes()), + body( + "application/json", + br#"{"auth":{"id":"kh.throwaway7","secret":"Tr0ub4dor-9x"},"n":3}"#, + ), + body( + "application/json", + br#"{"query":"x","variables":{"password":"Tr0ub4dor-9x"}}"#, + ), + body( + "application/json", + br#"{"users":{"kh.throwaway7":{"secret":"Tr0ub4dor-9x"},"bob.smith@ex.test":1}}"#, + ), + body("application/json", json_pair.as_bytes()), + body("application/json", b"{Tr0ub4dor-9x"), + body("multipart/form-data; boundary=XyZ", multipart), + body("application/octet-stream", b"Tr0ub4dor-9x kh.throwaway7"), + ]; + let mut outputs: Vec = bodies + .iter() + .map(|b| serde_json::to_string(b).unwrap()) + .collect(); + let urls = [ + "https://kh.throwaway7.ex.test/login", + // The HOST holds the value in a different case from the marker. + "https://zed.quincy9.ex.test/login", + "https://x-zed.quincy9-y.ex.test:81/login", + "https://ab?cd#ef@gh/x", + "https://ex.test/a/ab?cd#ef@gh", + "https://ab@gh.ex.test/ab?cd=1#ef", + "https://a.kh.throwaway7.ex.kh.throwaway7.test/", + "https://ex.test/u/kh.throwaway7/dash?kh.throwaway7=1&password=Tr0ub4dor-9x", + "https://ex.test/u/kh%2Ethrow%61way7/x?kh%2Ethrowaway7=1", + "https://ex.test/Tr0ub4dor-9x?x=Tr0ub4dor-9x#Tr0ub4dor-9x", + "https://kh.throwaway7:Tr0ub4dor-9x@ex.test/", + "https://ex.test/a/Tr0ub4/dor/z", + "https://ex.test/a/Tr0ub4%2Fdor/z?Tr0ub4%2Fdor=1", + "https://ex.test/u/bob.smith@ex.test/x?BOB.SMITH%40EX.TEST=1", + "data:text/plain,Tr0ub4dor-9x,kh.throwaway7", + "kh.throwaway7://ex.test/", + ]; + for url in urls { + let s = strip_url(url, &m, pair).unwrap(); + outputs.push(serde_json::to_string(&s.value).unwrap()); + } + let in_path = format!("https://ex.test/{b64}/x?{b64}=1"); + outputs + .push(serde_json::to_string(&strip_url(&in_path, &[], pair).unwrap().value).unwrap()); + for header in [ + format!("Basic {b64}"), + "Bearer Tr0ub4dor-9x".to_owned(), + "Tr0ub4dor-9x".to_owned(), + "kh.throwaway7".to_owned(), + "Basic Tr0ub4dor-9x".to_owned(), + ] { + let a = reduce_authorization(header.as_bytes(), &m, pair); + outputs.push(serde_json::to_string(&a).unwrap()); + } + for text in outputs { + assert_holds_no_typed_value(&text, &values); + } + } + + const TOKEN: &str = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk"; + + fn json_hits(s: &BodyShape) -> Vec { + fn walk(j: &JsonShape, out: &mut Vec) { + match j { + JsonShape::Object { fields } => fields.iter().for_each(|f| walk(&f.value, out)), + JsonShape::Array { items } => items.iter().for_each(|i| walk(i, out)), + JsonShape::Value { value } => out.extend(value.markers.clone()), + } + } + let mut out = Vec::new(); + if let BodyShape::Json { root } = s { + walk(root, &mut out); + } + out + } + + fn serialized(s: &BodyShape) -> String { + serde_json::to_string(s).unwrap() + } + + #[test] + fn a_urlencoded_body_that_is_one_token_is_opaque_and_not_kept() { + let s = shape_of_body(Some(URLENC), TOKEN.as_bytes(), &[], None); + assert!(matches!(s, BodyShape::Opaque { .. }), "{s:?}"); + assert!(!serialized(&s).contains("eyJhbGci")); + // One piece without `=` among ordinary ones does the same. + let body = format!("a=1&{TOKEN}&b=2"); + let s = shape_of_body(Some(URLENC), body.as_bytes(), &[], None); + assert!(matches!(s, BodyShape::Opaque { .. })); + assert!(!serialized(&s).contains("eyJhbGci")); + } + + #[test] + fn a_json_object_keyed_by_a_token_does_not_keep_it() { + let body = format!(r#"{{"{TOKEN}":1,"ok":2}}"#); + let s = shape_of_body(Some("application/json"), body.as_bytes(), &[], None); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = &s + else { + panic!("{s:?}") + }; + let mut keys: Vec<&str> = fields.iter().map(|f| f.key.as_str()).collect(); + keys.sort_unstable(); + assert_eq!(keys, ["ok", "{other}"]); + assert!(!serialized(&s).contains("eyJhbGci")); + } + + #[test] + fn a_multipart_part_named_like_a_token_does_not_keep_it() { + let body = + format!("--b\r\nContent-Disposition: form-data; name=\"{TOKEN}\"\r\n\r\nx\r\n--b--"); + let s = shape_of_body( + Some("multipart/form-data; boundary=b"), + body.as_bytes(), + &[], + None, + ); + let BodyShape::Multipart { parts } = &s else { + panic!("{s:?}") + }; + assert_eq!(parts[0].key, "{other}"); + assert!(!serialized(&s).contains("eyJhbGci")); + } + + #[test] + fn a_typed_value_found_in_a_name_keeps_its_hit() { + let m = [pw("Tr0ub4dor9x")]; + let plain = Some(hit(Role::Password, Encoding::AsTyped)); + // Urlencoded: the key is the password. + let s = shape_of_body(Some(URLENC), b"a=1&Tr0ub4dor9x=1", &m, None); + let BodyShape::Urlencoded { fields } = s else { + panic!() + }; + assert_eq!(fields[1].key, "{typed}"); + assert_eq!( + fields[1].value.markers, + plain.clone().into_iter().collect::>() + ); + assert!(fields[0].value.markers.is_empty()); + // JSON: the hit lands on the value under the key, or the first value inside it. + for body in [ + br#"{"Tr0ub4dor9x":"v"}"#.as_slice(), + br#"{"Tr0ub4dor9x":{"x":[1,2]}}"#.as_slice(), + ] { + let s = shape_of_body(Some("application/json"), body, &m, None); + assert_eq!(json_hits(&s), plain.clone().into_iter().collect::>()); + } + // Multipart. + let body = b"--b\r\nContent-Disposition: form-data; name=\"Tr0ub4dor9x\"\r\n\r\nx\r\n--b--"; + let s = shape_of_body(Some("multipart/form-data; boundary=b"), body, &m, None); + let BodyShape::Multipart { parts } = s else { + panic!() + }; + assert_eq!(parts[0].key, "{typed}"); + assert_eq!( + parts[0].value.markers, + plain.into_iter().collect::>() + ); + } + + #[test] + fn a_json_login_under_a_urlencoded_header_keeps_its_password_hit() { + // jQuery: $.post(url, JSON.stringify({user, pass})) with the default content type. + let m = markers(); + let body = br#"{"user":"kh.throwaway7","pass":"Tr0ub4dor-9x"}"#; + let s = shape_of_body(Some(URLENC), body, &m, None); + let BodyShape::Opaque { len, markers } = s else { + panic!("not opaque") + }; + assert_eq!(usize::try_from(len).unwrap(), body.len()); + assert!(markers.contains(&hit(Role::Password, Encoding::AsTyped))); + assert!(markers.contains(&hit(Role::Username, Encoding::AsTyped))); + } + + #[test] + fn a_big_json_array_is_opaque_quickly_and_small() { + // About 1 MiB of small numbers: the case that made a 33 MB shape. + let mut body = String::from("["); + while body.len() < MAX_BODY_BYTES - 16 { + body.push_str("1,"); + } + body.push_str("1]"); + assert!(body.len() <= MAX_BODY_BYTES); + let started = std::time::Instant::now(); + let s = shape_of_body(Some("application/json"), body.as_bytes(), &markers(), None); + assert!( + started.elapsed() < std::time::Duration::from_secs(1), + "took {:?}", + started.elapsed() + ); + let BodyShape::Opaque { len, .. } = &s else { + panic!("not opaque") + }; + assert_eq!(usize::try_from(*len).unwrap(), body.len()); + assert!(serialized(&s).len() < 1024); + } + + #[test] + fn a_shape_of_exactly_the_cap_is_kept_and_one_more_value_is_opaque() { + // An array is one node and each item another. + let items = |n: usize| format!("[{}]", vec!["1"; n].join(",")); + let ok = shape_of_body( + Some("application/json"), + items(MAX_SHAPE_VALUES - 1).as_bytes(), + &[], + None, + ); + assert!(matches!(ok, BodyShape::Json { .. })); + let over = shape_of_body( + Some("application/json"), + items(MAX_SHAPE_VALUES).as_bytes(), + &[], + None, + ); + assert!(matches!(over, BodyShape::Opaque { .. })); + // Urlencoded fields and multipart parts have the same cap. + let form = |n: usize| { + (0..n) + .map(|i| format!("k{i}=v")) + .collect::>() + .join("&") + }; + assert!(matches!( + shape_of_body(Some(URLENC), form(MAX_SHAPE_VALUES).as_bytes(), &[], None), + BodyShape::Urlencoded { .. } + )); + assert!(matches!( + shape_of_body( + Some(URLENC), + form(MAX_SHAPE_VALUES + 1).as_bytes(), + &[], + None + ), + BodyShape::Opaque { .. } + )); + let multipart = |n: usize| { + let mut b = Vec::new(); + for _ in 0..n { + b.extend_from_slice(b"--b\r\nContent-Disposition: form-data; name=f\r\n\r\nv\r\n"); + } + b.extend_from_slice(b"--b--\r\n"); + b + }; + assert!(matches!( + shape_of_body( + Some("multipart/form-data; boundary=b"), + &multipart(MAX_SHAPE_VALUES + 1), + &[], + None + ), + BodyShape::Opaque { .. } + )); + } + + #[test] + fn a_capped_body_still_reports_whole_body_hits() { + let m = [pw("Tr0ub4dor-9x")]; + let mut body = String::from("[\"Tr0ub4dor-9x\""); + for _ in 0..MAX_SHAPE_VALUES { + body.push_str(",1"); + } + body.push(']'); + let s = shape_of_body(Some("application/json"), body.as_bytes(), &m, None); + let BodyShape::Opaque { markers, .. } = s else { + panic!("not opaque") + }; + assert!(markers.contains(&hit(Role::Password, Encoding::AsTyped))); + } +} diff --git a/crates/kharon-teach/src/browser.rs b/crates/kharon-teach/src/browser.rs new file mode 100644 index 0000000..32a79e7 --- /dev/null +++ b/crates/kharon-teach/src/browser.rs @@ -0,0 +1,997 @@ +//! Find a Chromium-family browser, give it a private throwaway profile, and +//! guarantee the profile is deleted. + +use serde::Serialize; +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +#[derive(Debug, thiserror::Error)] +pub enum BrowserError { + #[error("no Chromium-family browser found; pass --browser ")] + NotFound, + #[error( + "the only browser found is a snap package, which cannot use the recorder's profile folder; install Google Chrome or pass --browser " + )] + SnapOnly, + #[error("browser not found at {0}")] + BadPath(PathBuf), + #[error("the browser did not start: {0}")] + Launch(String), +} + +pub const PROFILE_PREFIX: &str = "kharon-teach-"; +/// The random part of a profile's name: this many ASCII alphanumerics. +const SUFFIX_LEN: usize = 20; +/// Written into every profile: the pid of the recorder that owns it. +pub const OWNER_FILE: &str = "kharon-teach.owner"; + +const SEARCH_ORDER: [&str; 6] = [ + "chromium", + "chromium-browser", + "google-chrome", + "google-chrome-stable", + "microsoft-edge", + "brave-browser", +]; + +/// The browser to launch: `explicit` if given, else the first of the known +/// names found on `$PATH`. +pub fn find_browser(explicit: Option<&Path>) -> Result { + let path_var = std::env::var_os("PATH").unwrap_or_default(); + find_browser_in(explicit, &path_var) +} + +pub fn find_browser_in(explicit: Option<&Path>, path_var: &OsStr) -> Result { + find_browser_with(explicit, path_var, &is_snap_browser) +} + +/// [`find_browser_in`] with the snap test given, so a test can stand in for the file system. +/// An explicit path is used as given, snap or not: the person chose it. +pub(crate) fn find_browser_with( + explicit: Option<&Path>, + path_var: &OsStr, + is_snap: &dyn Fn(&Path) -> bool, +) -> Result { + if let Some(p) = explicit { + return if p.is_file() { + Ok(p.to_owned()) + } else { + Err(BrowserError::BadPath(p.to_owned())) + }; + } + let mut saw_snap = false; + for name in SEARCH_ORDER { + for dir in std::env::split_paths(path_var) { + let cand = dir.join(name); + if !is_executable_file(&cand) { + continue; + } + // A snap browser gets a private /tmp, so it never sees the recorder's profile. + if is_snap(&cand) { + saw_snap = true; + continue; + } + return Ok(cand); + } + } + Err(if saw_snap { + BrowserError::SnapOnly + } else { + BrowserError::NotFound + }) +} + +/// Whether a path is inside a snap package: under `/snap`, or the `snap` launcher that a +/// `/snap/bin/` link resolves to. +pub(crate) fn is_snap_path(path: &Path) -> bool { + path.starts_with("/snap") || path.file_name().is_some_and(|n| n == "snap") +} + +/// Whether the candidate browser is a snap: its path or its canonical path is a snap path, or it +/// is a short wrapper script that starts one (Ubuntu's `chromium-browser`). +fn is_snap_browser(cand: &Path) -> bool { + use std::io::Read; + if is_snap_path(cand) || std::fs::canonicalize(cand).is_ok_and(|c| is_snap_path(&c)) { + return true; + } + let mut head = Vec::new(); + let read = std::fs::File::open(cand).and_then(|f| f.take(4096).read_to_end(&mut head)); + read.is_ok() && head.starts_with(b"#!") && head.windows(6).any(|w| w == b"/snap/") +} + +fn is_executable_file(p: &Path) -> bool { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(p) + .map(|m| m.is_file() && m.permissions().mode() & 0o111 != 0) + .unwrap_or(false) +} + +/// A private throwaway browser profile directory, deleted on drop. +pub struct Profile { + path: PathBuf, +} + +#[derive(Serialize)] +struct Preferences { + credentials_enable_service: bool, + profile: ProfilePrefs, + autofill: AutofillPrefs, +} + +#[derive(Serialize)] +struct ProfilePrefs { + password_manager_enabled: bool, +} + +#[derive(Serialize)] +struct AutofillPrefs { + profile_enabled: bool, + credit_card_enabled: bool, +} + +const PREFERENCES: Preferences = Preferences { + credentials_enable_service: false, + profile: ProfilePrefs { + password_manager_enabled: false, + }, + autofill: AutofillPrefs { + profile_enabled: false, + credit_card_enabled: false, + }, +}; + +fn private_dir(path: &Path) -> std::io::Result<()> { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new().mode(0o700).create(path) +} + +impl Profile { + /// `/kharon-teach-`, mode 0700, holding an owner file with + /// this process's pid. It is built under a name the sweep never matches + /// and renamed into place, so a sweep cannot see it half-made. A crash + /// between the mkdir and the rename leaves a `.kharon-teach-new-*` + /// directory that nothing sweeps; the window is microseconds. + pub fn create() -> std::io::Result { + Self::create_in(&std::env::temp_dir()) + } + + /// [`Profile::create`] in `temp`. + pub(crate) fn create_in(temp: &Path) -> std::io::Result { + use rand::Rng; + use rand::distributions::Alphanumeric; + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + for _ in 0..16 { + let suffix: String = rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(SUFFIX_LEN) + .map(char::from) + .collect(); + let staging = temp.join(format!(".{PROFILE_PREFIX}new-{suffix}")); + match private_dir(&staging) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e), + } + let made = (|| { + let mut f = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(staging.join(OWNER_FILE))?; + f.write_all(std::process::id().to_string().as_bytes()) + })(); + if let Err(e) = made { + let _ = std::fs::remove_dir_all(&staging); + return Err(e); + } + let path = temp.join(format!("{PROFILE_PREFIX}{suffix}")); + // rename onto an existing non-empty directory fails; an existing + // empty one would be replaced, so check first. + if std::fs::symlink_metadata(&path).is_ok() { + let _ = std::fs::remove_dir_all(&staging); + continue; + } + match std::fs::rename(&staging, &path) { + Ok(()) => return Ok(Profile { path }), + Err(e) => { + let _ = std::fs::remove_dir_all(&staging); + if path.exists() { + continue; + } + return Err(e); + } + } + } + Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "could not find an unused profile directory name", + )) + } + + pub fn path(&self) -> &Path { + &self.path + } + + /// Delete the profile now and report a failure. Also runs on drop, + /// where errors are ignored. + pub fn remove(self) -> std::io::Result<()> { + remove_tree(&self.path) + } + + /// Removes profiles an earlier run of this program left in the temp location. A directory is + /// removed only when ALL hold: its name is exactly what [`Profile::create`] makes, it is a + /// real directory owned by this user, its owner file names a pid, that pid is gone, and its + /// browser is gone. Anything else, such as a folder someone made by hand, is left alone. + /// Returns how many were removed. + pub fn sweep_stale() -> std::io::Result { + Self::sweep_stale_in(&std::env::temp_dir()) + } + + /// [`Profile::sweep_stale`] over `dir`. + pub(crate) fn sweep_stale_in(dir: &Path) -> std::io::Result { + use std::os::unix::fs::MetadataExt; + // Not /proc/self: a non-dumpable process's /proc entries belong to root. + let me = rustix::process::geteuid().as_raw(); + let mut n = 0; + for entry in std::fs::read_dir(dir)? { + let Ok(entry) = entry else { continue }; + if !is_profile_name(&entry.file_name().to_string_lossy()) { + continue; + } + let path = entry.path(); + let Ok(meta) = std::fs::symlink_metadata(&path) else { + continue; + }; + if !meta.file_type().is_dir() || meta.uid() != me { + continue; + } + // No owner file, or one that does not parse, means this program did not make it. + let Some(owner) = owner_pid(&path) else { + continue; + }; + if pid_is_alive(owner) || browser_is_alive(&path) { + continue; + } + if remove_tree(&path).is_ok() { + n += 1; + } + } + Ok(n) + } +} + +impl Drop for Profile { + fn drop(&mut self) { + let _ = remove_tree(&self.path); + } +} + +/// `/Default/Preferences`: password manager and autofill off. +fn write_preferences(profile: &Path) -> std::io::Result<()> { + let default = profile.join("Default"); + private_dir(&default)?; + let prefs = serde_json::to_string(&PREFERENCES).map_err(std::io::Error::other)?; + std::fs::write(default.join("Preferences"), prefs) +} + +/// `remove_dir_all` does not follow symlinks. A just-killed browser's helper +/// processes can still be writing, so retry a few times. +fn remove_tree(path: &Path) -> std::io::Result<()> { + let mut last = Ok(()); + for _ in 0..5 { + match std::fs::remove_dir_all(path) { + Ok(()) => return Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => last = Err(e), + } + std::thread::sleep(Duration::from_millis(50)); + } + last +} + +/// `kharon-teach-` and exactly the random suffix `Profile::create` makes. +fn is_profile_name(name: &str) -> bool { + name.strip_prefix(PROFILE_PREFIX) + .is_some_and(|s| s.len() == SUFFIX_LEN && s.bytes().all(|b| b.is_ascii_alphanumeric())) +} + +/// The pid in the owner file. A missing, symlinked, non-regular, over-32-byte or unparseable +/// owner file gives `None`. +fn owner_pid(dir: &Path) -> Option { + let file = dir.join(OWNER_FILE); + let meta = std::fs::symlink_metadata(&file).ok()?; + if !meta.file_type().is_file() || meta.len() > 32 { + return None; + } + std::fs::read_to_string(&file) + .ok()? + .trim() + .parse::() + .ok() +} + +fn pid_is_alive(pid: u32) -> bool { + Path::new(&format!("/proc/{pid}")).exists() +} + +/// Chromium's `SingletonLock` is a symlink whose target text is +/// `-`; the profile is live when that pid exists. +fn browser_is_alive(dir: &Path) -> bool { + let Ok(target) = std::fs::read_link(dir.join("SingletonLock")) else { + return false; + }; + target + .to_string_lossy() + .rsplit('-') + .next() + .and_then(|p| p.parse::().ok()) + .is_some_and(pid_is_alive) +} + +pub struct Launch<'a> { + pub browser: &'a Path, + pub profile: &'a Profile, + pub first_url: &'a str, + pub headless: bool, +} + +/// The browser process and the process group it leads. Killing it kills the whole group, and a +/// drop kills it. +pub struct Browser { + child: tokio::process::Child, + /// The group id (the browser's own pid: it is launched as a group leader). + #[cfg(unix)] + group: Option, +} + +/// How long to wait for a killed group to be empty. +const GROUP_WAIT: Duration = Duration::from_secs(3); +const GROUP_POLL: Duration = Duration::from_millis(20); + +pub struct DevTools { + pub port: u16, + pub ws_path: String, +} + +const START_TIMEOUT: Duration = Duration::from_secs(20); +const POLL: Duration = Duration::from_millis(50); + +impl Browser { + /// Start the browser on the profile and wait up to 20 s for its + /// `DevToolsActivePort` file. + pub async fn launch(l: Launch<'_>) -> Result<(Browser, DevTools), BrowserError> { + use std::process::Stdio; + let launch_err = |e: std::io::Error| BrowserError::Launch(e.to_string()); + if l.first_url.starts_with('-') { + // It would be read as a browser switch. Do not echo the value. + return Err(BrowserError::Launch( + "the first URL must not start with '-'".to_owned(), + )); + } + let profile = l.profile.path(); + write_preferences(profile).map_err(launch_err)?; + + let mut user_data_dir = std::ffi::OsString::from("--user-data-dir="); + user_data_dir.push(profile.as_os_str()); + let mut cmd = tokio::process::Command::new(l.browser); + cmd.arg(user_data_dir) + .arg("--remote-debugging-port=0") + .arg("--no-first-run") + .arg("--no-default-browser-check") + .arg("--disk-cache-size=1") + .arg("--disable-sync") + .arg("--password-store=basic") + .arg("--disable-features=AutofillServerCommunication,PasswordManagerOnboarding"); + if l.headless { + cmd.arg("--headless=new"); + } + cmd.arg(l.first_url) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true); + // Its own process group, so that "kill the browser" means the helpers too. + #[cfg(unix)] + cmd.process_group(0); + let child = cmd.spawn().map_err(launch_err)?; + #[cfg(unix)] + let group = child + .id() + .and_then(|id| i32::try_from(id).ok()) + .and_then(rustix::process::Pid::from_raw); + // From here on a drop (including every early return) kills the group. + let mut browser = Browser { + child, + #[cfg(unix)] + group, + }; + + let file = profile.join("DevToolsActivePort"); + let deadline = tokio::time::Instant::now() + START_TIMEOUT; + loop { + match browser.child.try_wait() { + Ok(Some(status)) => return Err(BrowserError::Launch(status.to_string())), + Ok(None) => {} + Err(e) => return Err(launch_err(e)), + } + if let Some(dt) = std::fs::read_to_string(&file) + .ok() + .and_then(|t| parse_devtools_active_port(&t)) + { + return Ok((browser, dt)); + } + if tokio::time::Instant::now() >= deadline { + return Err(BrowserError::Launch( + "no DevTools port after 20 s".to_owned(), + )); + } + tokio::time::sleep(POLL).await; + } + } + + pub fn pid(&self) -> u32 { + self.child.id().unwrap_or(0) + } + + pub async fn wait(&mut self) -> std::io::Result { + self.child.wait().await + } + + /// SIGKILL to every process in the browser's group. Safe to repeat. + pub fn kill_group(&self) { + #[cfg(unix)] + if let Some(g) = self.group { + let _ = rustix::process::kill_process_group(g, rustix::process::Signal::KILL); + } + } + + /// True once no process is left in the browser's group. A group that was never made counts + /// as empty. Once the group has been seen empty the id is forgotten, so that no later signal + /// can reach a group id the system has given to something else. + pub fn group_is_empty(&mut self) -> bool { + #[cfg(unix)] + if let Some(g) = self.group { + let empty = matches!( + rustix::process::test_kill_process_group(g), + Err(rustix::io::Errno::SRCH) + ); + if empty { + self.group = None; + } + return empty; + } + true + } + + /// Kill the whole group, reap the browser, and wait up to 3 s for the group to be empty. + pub async fn close(mut self) -> std::io::Result<()> { + self.shut().await + } + + /// The body of [`Browser::close`], on a borrow so a test can look at the browser afterwards. + async fn shut(&mut self) -> std::io::Result<()> { + self.kill_group(); + let reaped = self.child.wait().await.map(|_| ()); + let deadline = tokio::time::Instant::now() + GROUP_WAIT; + while !self.group_is_empty() && tokio::time::Instant::now() < deadline { + tokio::time::sleep(GROUP_POLL).await; + } + reaped + } + + /// [`Browser::close`] for a place that cannot await (a `Drop`): it blocks the calling thread + /// for at most 3 s. + pub fn close_blocking(&mut self) { + self.kill_group(); + let deadline = std::time::Instant::now() + GROUP_WAIT; + loop { + // Reaps the leader, which would otherwise stay a zombie and keep the group non-empty. + let _ = self.child.try_wait(); + if self.group_is_empty() || std::time::Instant::now() >= deadline { + return; + } + std::thread::sleep(GROUP_POLL); + } + } +} + +impl Drop for Browser { + fn drop(&mut self) { + self.kill_group(); + } +} + +/// A copy of the rules in `parse_devtools_active_port` in +/// `crates/kharon-connector/src/devtools.rs`: two lines, a port of 1 to 5 +/// digits that is not 0, and a `/devtools/browser/` path. The host is +/// never taken from the file. +pub fn parse_devtools_active_port(text: &str) -> Option { + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + let mut lines = text.lines(); + let port_line = lines.next()?.trim(); + let path = lines.next()?.trim(); + if port_line.is_empty() || port_line.len() > 5 || !port_line.bytes().all(|b| b.is_ascii_digit()) + { + return None; + } + let port: u16 = port_line.parse().ok()?; + if port == 0 { + return None; + } + let id = path.strip_prefix("/devtools/browser/")?; + if id.is_empty() + || id.len() > 128 + || !id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') + { + return None; + } + Some(DevTools { + port, + ws_path: path.to_owned(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + /// A private directory for one test, removed when dropped. Its name does + /// not start with the profile prefix, and nothing else sweeps it. + struct Sandbox(PathBuf); + + impl Sandbox { + fn new(tag: &str) -> Self { + let d = std::env::temp_dir().join(format!( + "kharon-sandbox-{tag}-{}-{:08x}", + std::process::id(), + rand::random::() + )); + std::fs::create_dir(&d).unwrap(); + Sandbox(d) + } + fn path(&self) -> &Path { + &self.0 + } + fn mkdir(&self, name: &str) -> PathBuf { + let d = self.0.join(name); + std::fs::create_dir(&d).unwrap(); + d + } + } + + impl Drop for Sandbox { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + #[test] + fn profile_is_private_and_removed() { + let sb = Sandbox::new("create"); + let p = Profile::create_in(sb.path()).unwrap(); + let path = p.path().to_owned(); + assert_eq!(path.parent().unwrap(), sb.path()); + assert!( + path.file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with(PROFILE_PREFIX) + ); + assert_eq!(std::fs::metadata(&path).unwrap().mode() & 0o777, 0o700); + assert_eq!( + std::fs::read_to_string(path.join(OWNER_FILE)).unwrap(), + std::process::id().to_string() + ); + // No staging directory is left behind. + assert_eq!(std::fs::read_dir(sb.path()).unwrap().count(), 1); + p.remove().unwrap(); + assert!(!path.exists()); + } + + #[test] + fn profile_is_removed_on_drop() { + let sb = Sandbox::new("drop"); + let p = Profile::create_in(sb.path()).unwrap(); + let path = p.path().to_owned(); + std::fs::write(path.join("f"), b"x").unwrap(); + drop(p); + assert!(!path.exists()); + } + + /// A name of the shape `Profile::create` makes. + const A: &str = "kharon-teach-aaaaaaaaaaaaaaaaaaaa"; + const B: &str = "kharon-teach-bbbbbbbbbbbbbbbbbbbb"; + const DEAD: &str = "4294967295"; + + #[test] + fn sweep_removes_a_leftover_and_counts_it() { + let sb = Sandbox::new("stale"); + let d = dir_with_owner(&sb, A, DEAD); + std::fs::write(d.join("f"), b"x").unwrap(); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 1); + assert!(!d.exists()); + } + + #[test] + fn sweep_leaves_a_directory_this_program_did_not_make() { + let sb = Sandbox::new("foreign"); + // Owner-less, whatever the name. + let foo = sb.mkdir("kharon-teach-foo"); + std::fs::write(foo.join("f"), b"x").unwrap(); + let versioned = sb.mkdir("kharon-teach-0.1.0-x86_64-linux"); + // The right name shape, but no owner file. + let shaped = sb.mkdir(A); + // The right shape and a junk owner file. + let junk = dir_with_owner(&sb, B, "hello"); + // An owner file with a dead pid but a name this program never makes. + let odd = dir_with_owner(&sb, "kharon-teach-scratch", DEAD); + let short = dir_with_owner(&sb, "kharon-teach-aaaaaaaaaaaaaaaaaaa", DEAD); + let punct = dir_with_owner(&sb, "kharon-teach-aaaaaaaaaaaaaaaaaa_-", DEAD); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); + for d in [foo, versioned, shaped, junk, odd, short, punct] { + assert!(d.exists(), "{} was removed", d.display()); + } + } + + #[test] + fn profile_name_shape() { + assert!(is_profile_name(A)); + assert!(!is_profile_name("kharon-teach-")); + assert!(!is_profile_name("kharon-teach-aaaaaaaaaaaaaaaaaaaaa")); + assert!(!is_profile_name(".kharon-teach-new-aaaaaaaaaaaaaaaaaaaa")); + assert!(!is_profile_name("other-aaaaaaaaaaaaaaaaaaaa")); + // Whatever `create` makes passes. + let sb = Sandbox::new("shape"); + let p = Profile::create_in(sb.path()).unwrap(); + assert!(is_profile_name( + &p.path().file_name().unwrap().to_string_lossy() + )); + } + + #[test] + fn sweep_leaves_other_directories() { + let sb = Sandbox::new("other"); + let d = sb.mkdir("other-keep"); + let staging = sb.mkdir(".kharon-teach-new-x"); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); + assert!(d.exists() && staging.exists()); + } + + #[test] + fn sweep_leaves_a_symlink_and_its_target() { + let sb = Sandbox::new("link"); + let target = sb.mkdir("other-tgt"); + std::fs::write(target.join("f"), b"x").unwrap(); + let link = sb.path().join(A); + std::os::unix::fs::symlink(&target, &link).unwrap(); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); + assert!(target.join("f").exists()); + assert!(std::fs::symlink_metadata(&link).is_ok()); + } + + // An addition to the brief: a profile whose SingletonLock names a live + // process is a running recorder's profile and must survive a sweep. + #[test] + fn sweep_skips_a_profile_whose_browser_is_alive() { + let sb = Sandbox::new("live"); + let d = dir_with_owner(&sb, A, DEAD); + std::os::unix::fs::symlink( + format!("host-{}", std::process::id()), + d.join("SingletonLock"), + ) + .unwrap(); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); + assert!(d.exists()); + } + + fn dead_pid() -> u32 { + let mut c = std::process::Command::new("true").spawn().unwrap(); + let pid = c.id(); + c.wait().unwrap(); + pid + } + + fn dir_with_owner(sb: &Sandbox, name: &str, owner: &str) -> PathBuf { + let d = sb.mkdir(name); + std::fs::write(d.join(OWNER_FILE), owner).unwrap(); + d + } + + #[test] + fn sweep_skips_a_profile_owned_by_this_process() { + let sb = Sandbox::new("mine"); + let d = dir_with_owner(&sb, A, &std::process::id().to_string()); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); + assert!(d.exists()); + } + + #[test] + fn sweep_removes_a_profile_owned_by_a_dead_process() { + let sb = Sandbox::new("dead"); + let d = dir_with_owner(&sb, A, &dead_pid().to_string()); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 1); + assert!(!d.exists()); + } + + #[test] + fn sweep_treats_an_owner_symlink_as_no_owner() { + let sb = Sandbox::new("ownlink"); + let d = sb.mkdir(A); + let real = d.join("real"); + std::fs::write(&real, DEAD).unwrap(); + std::os::unix::fs::symlink(&real, d.join(OWNER_FILE)).unwrap(); + // No owner is not a stale owner: it is a directory this program cannot vouch for. + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); + assert!(d.exists()); + } + + #[test] + fn sweep_leaves_a_junk_or_oversized_owner_alone() { + let sb = Sandbox::new("junk"); + let a = dir_with_owner(&sb, A, "not a pid"); + let big = format!("{DEAD}{}", " ".repeat(40)); + let b = dir_with_owner(&sb, B, &big); + assert_eq!(Profile::sweep_stale_in(sb.path()).unwrap(), 0); + assert!(a.exists() && b.exists()); + } + + /// A non-dumpable process's `/proc/self` belongs to root, so a sweep that learned its uid + /// from there would compare against 0 and never remove anything. The check runs in a child + /// process (this test binary again) so the parent stays dumpable. On newer kernels + /// `/proc/self` stays owned by the user after that call, so the old code passes this test + /// too: it exercises the sweep under `PR_SET_DUMPABLE 0` and is not proof of the fix. + #[test] + fn sweep_still_removes_a_leftover_when_the_process_is_non_dumpable() { + const MARK: &str = "KHARON_TEACH_SWEEP_NONDUMPABLE_DIR"; + if let Some(dir) = std::env::var_os(MARK) { + use rustix::process::{DumpableBehavior, set_dumpable_behavior}; + set_dumpable_behavior(DumpableBehavior::NotDumpable).unwrap(); + let removed = Profile::sweep_stale_in(Path::new(&dir)).unwrap(); + assert_eq!(removed, 1, "the sweep did not remove the leftover"); + return; + } + let sb = Sandbox::new("nondump"); + let d = dir_with_owner(&sb, A, DEAD); + std::fs::write(d.join("f"), b"x").unwrap(); + let out = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "browser::tests::sweep_still_removes_a_leftover_when_the_process_is_non_dumpable", + "--test-threads=1", + ]) + .env(MARK, sb.path()) + .output() + .unwrap(); + assert!( + out.status.success(), + "child failed: {}", + String::from_utf8_lossy(&out.stdout) + ); + assert!(!d.exists()); + } + + /// A browser stand-in: a `sleep` that leads its own process group. + fn fake_browser() -> Browser { + let mut cmd = tokio::process::Command::new("sleep"); + cmd.arg("30").kill_on_drop(true); + cmd.process_group(0); + let child = cmd.spawn().unwrap(); + let group = child + .id() + .and_then(|id| i32::try_from(id).ok()) + .and_then(rustix::process::Pid::from_raw); + assert!(group.is_some()); + Browser { child, group } + } + + #[tokio::test] + async fn the_group_id_is_forgotten_once_the_group_is_closed_and_empty() { + let mut b = fake_browser(); + assert!(b.group.is_some()); + assert!(!b.group_is_empty(), "a live group is not empty"); + assert!(b.group.is_some(), "a live group's id is kept"); + b.shut().await.unwrap(); + assert!(b.group.is_none(), "the id of an empty group is dropped"); + // Nothing is signalled any more: kill_group has no id to signal. + b.kill_group(); + assert!(b.group_is_empty()); + } + + #[tokio::test] + async fn close_blocking_also_forgets_the_group_id() { + let mut b = fake_browser(); + b.close_blocking(); + assert!(b.group.is_none()); + } + + #[test] + fn preferences_turn_off_the_password_manager_and_autofill() { + let sb = Sandbox::new("prefs"); + write_preferences(sb.path()).unwrap(); + assert_eq!( + std::fs::metadata(sb.path().join("Default")).unwrap().mode() & 0o777, + 0o700 + ); + assert_eq!( + std::fs::read_to_string(sb.path().join("Default/Preferences")).unwrap(), + r#"{"credentials_enable_service":false,"profile":{"password_manager_enabled":false},"autofill":{"profile_enabled":false,"credit_card_enabled":false}}"# + ); + } + + #[tokio::test] + async fn a_first_url_that_looks_like_a_switch_is_refused() { + let sb = Sandbox::new("url"); + let profile = Profile::create_in(sb.path()).unwrap(); + let err = match Browser::launch(Launch { + browser: Path::new("/nonexistent/chrome"), + profile: &profile, + first_url: "--secret-switch", + headless: true, + }) + .await + { + Err(e) => e, + Ok(_) => panic!("launched"), + }; + let BrowserError::Launch(msg) = err else { + panic!("wrong error"); + }; + assert_eq!(msg, "the first URL must not start with '-'"); + } + + #[test] + fn no_browser_on_an_empty_path_is_not_found() { + assert!(matches!( + find_browser_in(None, OsStr::new("")), + Err(BrowserError::NotFound) + )); + } + + #[test] + fn finds_the_first_name_in_search_order() { + let sb = Sandbox::new("bin"); + let dir = sb.path().to_owned(); + for n in ["google-chrome", "chromium"] { + let f = dir.join(n); + std::fs::write(&f, b"#!/bin/sh\n").unwrap(); + std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let found = find_browser_in(None, dir.as_os_str()).unwrap(); + assert_eq!(found, dir.join("chromium")); + } + + #[test] + fn snap_paths_are_recognised() { + for snap in [ + "/snap/bin/chromium", + "/snap/chromium/current/usr/lib/chromium-browser/chrome", + // What `/snap/bin/chromium` resolves to once canonicalised. + "/usr/bin/snap", + ] { + assert!(is_snap_path(Path::new(snap)), "{snap}"); + } + for other in [ + "/usr/bin/chromium", + "/usr/bin/google-chrome", + "/opt/google/chrome/chrome", + "/home/me/snapshots/chrome", + "/usr/bin/snapd", + ] { + assert!(!is_snap_path(Path::new(other)), "{other}"); + } + } + + fn executable(dir: &Path, name: &str, body: &str) -> PathBuf { + let f = dir.join(name); + std::fs::write(&f, body).unwrap(); + std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o755)).unwrap(); + f + } + + #[test] + fn a_snap_candidate_is_skipped_for_the_next_one_in_order() { + let sb = Sandbox::new("snaporder"); + let a = sb.mkdir("a"); + let b = sb.mkdir("b"); + executable(&a, "chromium", "#!/bin/sh\n"); + let chrome = executable(&b, "google-chrome", "#!/bin/sh\n"); + let path = std::env::join_paths([&a, &b]).unwrap(); + // The stub calls only the first candidate a snap. + let is_snap = |p: &Path| p == a.join("chromium"); + assert_eq!(find_browser_with(None, &path, &is_snap).unwrap(), chrome); + // With nothing to fall back on, the error says why. + let only_a = std::env::join_paths([&a]).unwrap(); + assert!(matches!( + find_browser_with(None, &only_a, &is_snap), + Err(BrowserError::SnapOnly) + )); + // No candidate at all is still NotFound. + let empty = std::env::join_paths([sb.mkdir("empty")]).unwrap(); + assert!(matches!( + find_browser_with(None, &empty, &is_snap), + Err(BrowserError::NotFound) + )); + } + + #[test] + fn an_explicit_snap_browser_is_used_as_given() { + let sb = Sandbox::new("snapexplicit"); + let f = executable(sb.path(), "chromium", "#!/bin/sh\n"); + let always = |_: &Path| true; + assert_eq!( + find_browser_with(Some(&f), OsStr::new(""), &always).unwrap(), + f + ); + } + + #[test] + fn the_snap_only_message_says_what_to_do() { + assert_eq!( + BrowserError::SnapOnly.to_string(), + "the only browser found is a snap package, which cannot use the recorder's profile folder; install Google Chrome or pass --browser " + ); + } + + #[test] + fn a_wrapper_script_that_starts_a_snap_is_a_snap_and_a_plain_script_is_not() { + let sb = Sandbox::new("snapwrap"); + let wrapper = executable( + sb.path(), + "chromium-browser", + "#!/bin/sh\nexec /snap/bin/chromium \"$@\"\n", + ); + let plain = executable( + sb.path(), + "chromium", + "#!/bin/sh\nexec /opt/chrome \"$@\"\n", + ); + assert!(is_snap_browser(&wrapper)); + assert!(!is_snap_browser(&plain)); + } + + #[test] + fn explicit_missing_browser_is_bad_path() { + assert!(matches!( + find_browser(Some(Path::new("/nonexistent/chrome"))), + Err(BrowserError::BadPath(_)) + )); + } + + #[test] + fn parses_the_two_line_file() { + let d = parse_devtools_active_port("45123\n/devtools/browser/abc-123\n").unwrap(); + assert_eq!(d.port, 45123); + assert_eq!(d.ws_path, "/devtools/browser/abc-123"); + } + + #[test] + fn accepts_a_bom_and_rejects_a_port_over_u16() { + assert_eq!( + parse_devtools_active_port("\u{feff}45123\n/devtools/browser/abc\n") + .unwrap() + .port, + 45123 + ); + assert!(parse_devtools_active_port("99999\n/devtools/browser/abc\n").is_none()); + } + + #[test] + fn rejects_port_zero_garbage_and_a_foreign_path() { + for bad in [ + "0\n/devtools/browser/abc\n", + "123456\n/devtools/browser/abc\n", + "12x\n/devtools/browser/abc\n", + "", + "45123\n", + "45123\n/evil/browser/abc\n", + "45123\n/devtools/browser/\n", + "45123\n/devtools/browser/a/b\n", + ] { + assert!(parse_devtools_active_port(bad).is_none(), "{bad:?}"); + } + } +} diff --git a/crates/kharon-teach/src/capture/build.rs b/crates/kharon-teach/src/capture/build.rs new file mode 100644 index 0000000..2edf16a --- /dev/null +++ b/crates/kharon-teach/src/capture/build.rs @@ -0,0 +1,369 @@ +//! Turning buffered events into the value-free parts of a `Recording`. + +use kharon_recording::{ + BodyShape, ElementDescriptor, FormRef, FrameRef, MarkerHit, ResponseShape, StoredRequest, + StrippedUrl, +}; + +use super::{Buffered, FrameInfo, RawBody, RawElement, RawForm}; +use crate::body::shape_of_body; +use crate::marker::Marker; +use crate::strip::{TYPED, dedup_hits, hits_in, holds_typed, reduce_authorization, strip_url}; +use zeroize::Zeroizing; + +/// What is known while stripping: every typed value, and the credential pair once chosen. +#[derive(Clone, Copy)] +pub(super) struct Ctx<'a> { + pub markers: &'a [Marker], + pub pair: Option<(&'a Marker, &'a Marker)>, +} + +impl Ctx<'_> { + fn holds_typed(&self, text: &str) -> bool { + holds_typed(text.as_bytes(), self.markers, self.pair) + } +} + +/// Counts of what stripping removed, for URLs that are written to the file. +#[derive(Default, Clone, Copy)] +pub(super) struct Tally { + pub query_values_removed: u64, + pub path_segments_replaced: u64, +} + +impl Tally { + pub fn add(&mut self, other: Tally) { + self.query_values_removed += other.query_values_removed; + self.path_segments_replaced += other.path_segments_replaced; + } +} + +const OTHER_ORIGIN: &str = "{other}:"; + +/// `Some` when the URL parses. The tally is the URL's own counts. +fn strip(url: &str, ctx: Ctx<'_>) -> Option<(StrippedUrl, Tally)> { + strip_url(url, ctx.markers, ctx.pair).map(|s| { + ( + s.value, + Tally { + query_values_removed: s.query_values_removed, + path_segments_replaced: s.path_segments_replaced, + }, + ) + }) +} + +/// A URL for the file; one that does not parse becomes an empty `{other}:` origin. +pub(super) fn stripped_url(url: &str, ctx: Ctx<'_>, tally: &mut Tally) -> StrippedUrl { + match strip(url, ctx) { + Some((u, t)) => { + tally.add(t); + u + } + None => StrippedUrl { + origin: OTHER_ORIGIN.to_owned(), + path: String::new(), + query_keys: Vec::new(), + }, + } +} + +pub(super) fn frame_ref(info: &FrameInfo, ctx: Ctx<'_>) -> FrameRef { + FrameRef { + main: info.main, + origin: strip(&info.url, ctx).map_or_else(|| OTHER_ORIGIN.to_owned(), |(u, _)| u.origin), + cross_origin: info.cross_origin, + } +} + +/// An attribute is kept only when it holds no typed value. +fn clean(attr: Option<&String>, ctx: Ctx<'_>) -> Option { + attr.filter(|a| !ctx.holds_typed(a)).cloned() +} + +/// `s` with CSS escapes undone (CSS Syntax 3, 4.3.7): `\` + 1-6 hex digits (+ one optional +/// whitespace) is that code point (0, a surrogate or past U+10FFFF is U+FFFD); `\` + any other +/// char is that char. A `\` before a newline or at the end is dropped. +fn css_unescape(s: &str) -> Zeroizing { + let mut out = Zeroizing::new(String::with_capacity(s.len())); + let mut it = s.chars().peekable(); + while let Some(c) = it.next() { + if c != '\\' { + out.push(c); + continue; + } + let mut hex = Zeroizing::new(String::new()); + while hex.len() < 6 { + match it.peek() { + Some(h) if h.is_ascii_hexdigit() => { + hex.push(*h); + it.next(); + } + _ => break, + } + } + if hex.is_empty() { + match it.next() { + Some('\n') | None => {} + Some(o) => out.push(o), + } + continue; + } + let cp = u32::from_str_radix(&hex, 16).unwrap_or(0); + out.push(char::from_u32(cp).filter(|_| cp != 0).unwrap_or('\u{FFFD}')); + match it.peek() { + Some('\r') => { + it.next(); + if it.peek() == Some(&'\n') { + it.next(); + } + } + Some(' ' | '\t' | '\n' | '\u{c}') => { + it.next(); + } + _ => {} + } + } + out +} + +/// A selector holding a typed value, also once CSS escapes are undone, is removed. +fn selector_is_clean(selector: &str, ctx: Ctx<'_>) -> bool { + !ctx.holds_typed(selector) && !ctx.holds_typed(&css_unescape(selector)) +} + +const INPUT_TYPES: [&str; 22] = [ + "text", + "password", + "email", + "tel", + "number", + "search", + "url", + "hidden", + "checkbox", + "radio", + "submit", + "button", + "reset", + "file", + "date", + "datetime-local", + "month", + "week", + "time", + "color", + "range", + "image", +]; + +/// An HTML input type keyword, lowercased; anything else is `{other}`. +fn input_type_keyword(raw: &str) -> String { + let lower = raw.to_ascii_lowercase(); + if INPUT_TYPES.contains(&lower.as_str()) { + lower + } else { + "{other}".to_owned() + } +} + +/// A tag name, lowercased, when it is ASCII letters, digits and `-` and holds no typed value. +fn tag_name(raw: &str, ctx: Ctx<'_>) -> String { + let lower = raw.to_ascii_lowercase(); + let plain = !lower.is_empty() + && lower + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-'); + if plain && !ctx.holds_typed(&lower) { + lower + } else { + "{other}".to_owned() + } +} + +fn known_method(method: &str, lowercase: bool) -> String { + let upper = method.to_ascii_uppercase(); + let known = ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]; + if !known.contains(&upper.as_str()) { + return "{other}".to_owned(); + } + if lowercase { + upper.to_lowercase() + } else { + upper + } +} + +pub(super) fn form_ref(form: &RawForm, ctx: Ctx<'_>, tally: &mut Tally) -> FormRef { + FormRef { + index: form.index, + action: form + .action + .as_deref() + .and_then(|a| strip(a, ctx)) + .map(|(u, t)| { + tally.add(t); + u + }), + method: known_method(&form.method, true), + } +} + +pub(super) fn describe( + raw: &RawElement, + was_password: bool, + ctx: Ctx<'_>, + tally: &mut Tally, +) -> ElementDescriptor { + ElementDescriptor { + element_id: raw.element_id, + tag: tag_name(&raw.tag, ctx), + input_type: raw.input_type.as_deref().map(input_type_keyword), + id: clean(raw.id.as_ref(), ctx), + name: clean(raw.name.as_ref(), ctx), + autocomplete: clean(raw.autocomplete.as_ref(), ctx), + placeholder: clean(raw.placeholder.as_ref(), ctx), + aria_label: clean(raw.aria_label.as_ref(), ctx), + was_password, + form: raw.form.as_ref().map(|f| form_ref(f, ctx, tally)), + selectors: raw + .selectors + .iter() + .filter(|s| selector_is_clean(s, ctx)) + .cloned() + .collect(), + in_shadow_root: raw.in_shadow_root, + } +} + +/// The media type alone, lowercased: parameters (a multipart boundary) are not kept. One that +/// holds a typed value is `{typed}`. +fn media_type(content_type: Option<&str>, ctx: Ctx<'_>) -> Option { + let essence = content_type? + .split(';') + .next() + .unwrap_or_default() + .trim() + .to_ascii_lowercase(); + Some(if ctx.holds_typed(&essence) { + TYPED.to_owned() + } else { + essence + }) +} + +pub(super) fn response_shape(r: &super::BufferedResponse, ctx: Ctx<'_>) -> ResponseShape { + ResponseShape { + status: r.status, + content_type: media_type(r.content_type.as_deref(), ctx), + set_cookie: r.set_cookie, + } +} + +/// Every marker hit in the request's raw bytes: its URL, its Authorization header and its whole +/// body, with no shape built. What decides whether the request is kept. +pub(super) fn raw_hits(b: &Buffered, ctx: Ctx<'_>) -> Vec { + let mut out = hits_in(b.url.as_bytes(), ctx.markers, ctx.pair); + if let Some(a) = &b.authorization { + out.extend(reduce_authorization(a, ctx.markers, ctx.pair).markers); + } + if let RawBody::Bytes(bytes) = &b.body { + out.extend(hits_in(bytes, ctx.markers, ctx.pair)); + } + dedup_hits(out) +} + +/// The reduced request, with the counts of what stripping its URL removed and whether its body +/// was reduced to a shape. `None` when the URL does not parse. Whether to keep it is decided by +/// the caller BEFORE this is called, so only kept requests are shaped. +pub(super) fn reduce_request(b: &Buffered, ctx: Ctx<'_>) -> Option<(StoredRequest, Tally, bool)> { + let (url, tally) = strip(&b.url, ctx)?; + let reduced_body = matches!(b.body, RawBody::Bytes(_)); + let body = match &b.body { + RawBody::None => BodyShape::None, + RawBody::Unavailable => BodyShape::Unavailable, + RawBody::Bytes(bytes) => { + shape_of_body(b.content_type.as_deref(), bytes, ctx.markers, ctx.pair) + } + }; + let stored = StoredRequest { + t_ms: b.t_ms, + method: known_method(&b.method, false), + url, + resource: b.resource, + main_frame_navigation: b.main_frame_navigation, + content_type: media_type(b.content_type.as_deref(), ctx), + authorization: b + .authorization + .as_ref() + .map(|a| reduce_authorization(a, ctx.markers, ctx.pair)), + url_markers: hits_in(b.url.as_bytes(), ctx.markers, ctx.pair), + body, + response: b.response.as_ref().map(|r| response_shape(r, ctx)), + }; + Some((stored, tally, reduced_body)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::pw; + + fn un(s: &str) -> String { + css_unescape(s).to_string() + } + + #[test] + fn a_hex_escape_is_its_code_point() { + assert_eq!(un(r"#\39 x"), "#9x"); + assert_eq!(un(r"#\000039x"), "#9x"); + } + + #[test] + fn any_other_escaped_char_is_itself() { + assert_eq!(un(r"#a\.b"), "#a.b"); + assert_eq!(un(r"#a\\b"), r"#a\b"); + } + + #[test] + fn zero_a_surrogate_or_past_the_last_code_point_is_the_replacement_char() { + assert_eq!(un(r"#\0 x"), "#\u{FFFD}x"); + assert_eq!(un(r"#\D800 x"), "#\u{FFFD}x"); + assert_eq!(un(r"#\110000 x"), "#\u{FFFD}x"); + } + + #[test] + fn a_trailing_backslash_is_dropped() { + assert_eq!(un(r"#a\"), "#a"); + } + + #[test] + fn a_selector_with_a_hex_escaped_typed_value_is_left_out() { + let m = [pw("9xTr0ub4dor")]; + let ctx = Ctx { + markers: &m, + pair: None, + }; + assert!(!selector_is_clean(r"#\39 xTr0ub4dor", ctx)); + } + + #[test] + fn a_selector_with_an_escaped_backslash_in_a_typed_value_is_left_out() { + let m = [pw(r"Tr0ub\4dor")]; + let ctx = Ctx { + markers: &m, + pair: None, + }; + assert!(!selector_is_clean(r"#Tr0ub\\4dor", ctx)); + } + + #[test] + fn a_selector_with_no_typed_value_is_kept() { + let m = [pw("9xTr0ub4dor")]; + let ctx = Ctx { + markers: &m, + pair: None, + }; + assert!(selector_is_clean(r"#login\.form > input:nth-child(2)", ctx)); + } +} diff --git a/crates/kharon-teach/src/capture/mod.rs b/crates/kharon-teach/src/capture/mod.rs new file mode 100644 index 0000000..d889945 --- /dev/null +++ b/crates/kharon-teach/src/capture/mod.rs @@ -0,0 +1,1914 @@ +//! The capture state machine: events from a recorded login go in, and `finish` returns a +//! `Recording` that says how the login works and holds nothing the person typed. +//! +//! Pure: no browser, no I/O, no logging. Requests are buffered with their bodies in `Zeroizing` +//! buffers and analysed in `finish`, after the typed values are known. + +mod build; +mod resolve; +mod types; + +use kharon_recording::{ + BrowserInfo, Deciding, Flags, ListenerWorld, Recording, Role, Step, StoredRequest, + StrippedCounts, Viewport, classify, is_otp_descriptor, +}; +pub use types::{CaptureEvent, EndInfo, PageEventKind, RawBody, RawElement, RawForm, RawFrame}; +use url::Url; +use zeroize::Zeroizing; + +use crate::body::MAX_BODY_BYTES; +use crate::site::{is_captcha_host, registrable_domain}; +use build::{Ctx, Tally, describe, form_ref, frame_ref, raw_hits, reduce_request, stripped_url}; +use resolve::{Elem, resolve}; + +/// Once this many body bytes are buffered, every further body is buffered as `Unavailable`. +pub const MAX_BUFFERED_BODY_BYTES: usize = 32 << 20; +/// Steps kept. Later ones are counted in `StrippedCounts::steps_dropped` and not recorded. Typed +/// values are still tracked: only the step list stops growing. +pub const MAX_STEPS: usize = 10_000; +/// Requests buffered. Later ones are counted in `requests_seen` and not buffered. +pub const MAX_BUFFERED_REQUESTS: usize = 5_000; + +/// A frame as it was when an event arrived. +struct FrameInfo { + main: bool, + url: String, + cross_origin: bool, +} + +enum What { + Navigation(String), + Input(usize), + Click(RawElement), + Submit(RawForm), + EnterKey(RawElement), +} + +struct Pending { + t_ms: u64, + frame: FrameInfo, + what: What, +} + +struct BufferedResponse { + status: u16, + content_type: Option, + set_cookie: bool, +} + +/// A request waiting for `finish`, with its bytes in zeroizing buffers. +struct Buffered { + request_id: String, + t_ms: u64, + method: String, + url: String, + resource: kharon_recording::ResourceKind, + main_frame_navigation: bool, + content_type: Option, + authorization: Option>>, + body: RawBody, + /// A non-GET request to the login site made after the first `Input`. + login_site_write: bool, + response: Option, +} + +pub struct Capture { + start_url: String, + tool_version: String, + login_site: String, + started: bool, + browser: Option, + listener_world: ListenerWorld, + main_origin: Option, + final_url: Option, + steps: Vec, + steps_dropped: u64, + max_steps: usize, + max_requests: usize, + /// The first sequence number of the first element known to be a password. + first_password_seq: Option, + elems: Vec, + reqs: Vec, + seq: u64, + inputs_seen: bool, + requests_seen: u64, + popup: bool, + captcha: bool, + /// When the first main-frame navigation to another registrable domain happened. + first_cross_site_ms: Option, + framed_login: bool, + body_unavailable: bool, + body_budget: usize, + buffered_body_bytes: usize, +} + +fn host_of(url: &str) -> Option { + Url::parse(url) + .ok() + .and_then(|u| u.host_str().map(str::to_owned)) +} + +fn raw_origin(url: &str) -> Option { + Url::parse(url) + .ok() + .map(|u| u.origin().ascii_serialization()) +} + +/// A CAPTCHA host, or `www.google.com/recaptcha/…`. +fn is_captcha_url(url: &str) -> bool { + let Ok(parsed) = Url::parse(url) else { + return false; + }; + let Some(host) = parsed.host_str() else { + return false; + }; + is_captcha_host(host) + || (matches!(host, "www.google.com" | "google.com") + && parsed.path().starts_with("/recaptcha/")) +} + +fn same_element(e: &Elem, raw: &RawElement) -> bool { + e.raw.document_id == raw.document_id && e.raw.element_id == raw.element_id +} + +impl Capture { + pub fn new(start_url: &str, tool_version: &str) -> Self { + let login_site = host_of(start_url).map(|h| registrable_domain(&h)); + Self { + start_url: start_url.to_owned(), + tool_version: tool_version.to_owned(), + login_site: login_site.unwrap_or_default(), + started: false, + browser: None, + listener_world: ListenerWorld::Isolated, + main_origin: raw_origin(start_url), + final_url: None, + steps: Vec::new(), + steps_dropped: 0, + max_steps: MAX_STEPS, + max_requests: MAX_BUFFERED_REQUESTS, + first_password_seq: None, + elems: Vec::new(), + reqs: Vec::new(), + seq: 0, + inputs_seen: false, + requests_seen: 0, + popup: false, + captcha: false, + first_cross_site_ms: None, + framed_login: false, + body_unavailable: false, + body_budget: MAX_BUFFERED_BODY_BYTES, + buffered_body_bytes: 0, + } + } + + pub fn started(&self) -> bool { + self.started + } + + /// Takes one event. Everything but `Popup` is ignored until `Started` arrives. + pub fn push(&mut self, ev: CaptureEvent) { + match ev { + CaptureEvent::Popup => self.popup = true, + CaptureEvent::Started { + browser, + listener_world, + } => { + if !self.started { + self.started = true; + self.browser = Some(browser); + self.listener_world = listener_world; + } + } + _ if !self.started => {} + CaptureEvent::Navigation { t_ms, frame } => self.navigation(t_ms, &frame), + CaptureEvent::Page { + t_ms, + kind, + frame, + element, + value, + } => self.page(t_ms, kind, &frame, element, value), + CaptureEvent::Request { + request_id, + t_ms, + method, + url, + resource, + main_frame_navigation, + content_type, + authorization, + body, + frame_url: _, + } => self.request(Buffered { + request_id, + t_ms, + method, + url, + resource, + main_frame_navigation, + content_type, + authorization, + body, + login_site_write: false, + response: None, + }), + CaptureEvent::Response { + request_id, + status, + content_type, + set_cookie, + } => { + if let Some(r) = self + .reqs + .iter_mut() + .rev() + .find(|r| r.request_id == request_id) + { + r.response = Some(BufferedResponse { + status, + content_type, + set_cookie, + }); + } + } + } + } + + fn frame_info(&self, frame: &RawFrame) -> FrameInfo { + let cross_origin = !frame.main && raw_origin(&frame.url) != self.main_origin; + FrameInfo { + main: frame.main, + url: frame.url.clone(), + cross_origin, + } + } + + fn navigation(&mut self, t_ms: u64, frame: &RawFrame) { + if is_captcha_url(&frame.url) { + self.captcha = true; + } + if frame.main { + self.main_origin = raw_origin(&frame.url); + self.final_url = Some(frame.url.clone()); + // Only web pages count: an error page (`chrome-error://`) is not another site. + let other_site = Url::parse(&frame.url).is_ok_and(|u| { + matches!(u.scheme(), "http" | "https") + && u.host_str() + .is_some_and(|h| registrable_domain(h) != self.login_site) + }); + if other_site { + self.first_cross_site_ms.get_or_insert(t_ms); + } + } + let info = self.frame_info(frame); + self.add_step(Pending { + t_ms, + frame: info, + what: What::Navigation(frame.url.clone()), + }); + } + + /// Keeps the step, or counts it when the step list is full. + fn add_step(&mut self, step: Pending) { + if self.steps.len() < self.max_steps { + self.steps.push(step); + } else { + self.steps_dropped += 1; + } + } + + fn page( + &mut self, + t_ms: u64, + kind: PageEventKind, + frame: &RawFrame, + element: RawElement, + value: Option>, + ) { + let info = self.frame_info(frame); + let known = self.elems.iter().position(|e| same_element(e, &element)); + if let Some(i) = known { + self.note_report(i, &element, &info); + } + let what = match kind { + PageEventKind::Input | PageEventKind::Change => { + let i = match known { + Some(i) => i, + None => { + self.elems.push(Elem { + raw: element.clone(), + value: Zeroizing::new(String::new()), + was_password: false, + shadow: false, + has_step: false, + first_seq: self.seq + 1, + last_seq: 0, + last_before_password: None, + }); + let i = self.elems.len() - 1; + self.note_report(i, &element, &info); + i + } + }; + self.seq += 1; + self.elems[i].last_seq = self.seq; + if self.first_password_seq.is_none_or(|s| self.seq < s) { + self.elems[i].last_before_password = Some(self.seq); + } + if let Some(v) = value { + self.elems[i].value = v; + } + if kind == PageEventKind::Change || self.elems[i].has_step { + if kind == PageEventKind::Input { + self.inputs_seen = true; + } + return; + } + self.elems[i].has_step = true; + self.inputs_seen = true; + What::Input(i) + } + PageEventKind::Click => What::Click(element), + PageEventKind::EnterKey => What::EnterKey(element), + PageEventKind::Submit => match element.form { + Some(form) => What::Submit(form), + None => return, + }, + }; + self.add_step(Pending { + t_ms, + frame: info, + what, + }); + } + + fn note_report(&mut self, i: usize, element: &RawElement, info: &FrameInfo) { + let e = &mut self.elems[i]; + e.was_password |= element.is_password_now; + if e.was_password { + let first = e.first_seq; + self.first_password_seq = Some(self.first_password_seq.map_or(first, |s| s.min(first))); + } + let e = &mut self.elems[i]; + e.shadow |= element.in_shadow_root; + if e.was_password && !info.main && info.cross_origin { + self.framed_login = true; + } + } + + fn request(&mut self, mut b: Buffered) { + self.requests_seen += 1; + if is_captcha_url(&b.url) { + self.captcha = true; + } + if self.reqs.len() >= self.max_requests { + return; + } + if let RawBody::Bytes(bytes) = &b.body { + if bytes.len() > MAX_BODY_BYTES + || self.buffered_body_bytes.saturating_add(bytes.len()) > self.body_budget + { + b.body = RawBody::Unavailable; + } else { + self.buffered_body_bytes += bytes.len(); + } + } + let to_login_site = + host_of(&b.url).is_some_and(|h| registrable_domain(&h) == self.login_site); + let write = !b.method.eq_ignore_ascii_case("GET") && to_login_site; + if write && matches!(b.body, RawBody::Unavailable) { + self.body_unavailable = true; + } + b.login_site_write = write && self.inputs_seen; + self.reqs.push(b); + } + + /// Resolves the typed values, searches the buffered requests, strips, sets the flags and + /// classifies. Consumes the capture, so every buffered value is dropped (and zeroed) here. + pub fn finish(self, end: EndInfo) -> Recording { + let resolved = resolve(&self.elems, &self.reqs); + let ctx = Ctx { + markers: &resolved.markers, + pair: resolved.pair(), + }; + let mut tally = Tally::default(); + + let start_url = stripped_url(&self.start_url, ctx, &mut tally); + let final_url = self + .final_url + .as_deref() + .map(|u| stripped_url(u, ctx, &mut tally)); + let steps = self.build_steps(&resolved, ctx, &mut tally); + + // Which requests are kept is decided on the raw bytes, before any body is shaped. A weak + // (short-value) hit counts only when no confident hit of its role exists anywhere. + let hits: Vec> = + self.reqs.iter().map(|b| raw_hits(b, ctx)).collect(); + let deciding = Deciding::from_hits(hits.iter().flatten()); + let mut requests: Vec = Vec::new(); + let mut bodies_reduced = 0; + for (b, hs) in self.reqs.iter().zip(&hits) { + let telling = hs.iter().any(|h| deciding.counts(h)); + if !(b.main_frame_navigation || telling || b.login_site_write) { + continue; + } + let Some((stored, t, reduced)) = reduce_request(b, ctx) else { + continue; + }; + tally.add(t); + bodies_reduced += u64::from(reduced); + requests.push(stored); + } + + // The login is over once the password has been sent: the last request carrying it, or + // when none does, the last typing. A move to another site after that is not part of it. + let credentials_sent = self + .reqs + .iter() + .zip(&hits) + .filter(|(_, hs)| { + hs.iter() + .any(|h| h.role == Role::Password && deciding.counts(h)) + }) + .map(|(b, _)| b.t_ms) + .max() + .or_else(|| { + steps + .iter() + .filter_map(|s| match s { + Step::Input { t_ms, .. } => Some(*t_ms), + _ => None, + }) + .max() + }); + let cross_site = self + .first_cross_site_ms + .is_some_and(|t| credentials_sent.is_none_or(|c| t < c)); + let went_on_after_login = self.first_cross_site_ms.is_some() && !cross_site; + + let flags = + Flags { + captcha: self.captcha, + otp_step: self.otp_step(), + cross_site, + popup: self.popup, + framed_login: self.framed_login, + shadow_dom: self.elems.iter().enumerate().any(|(i, e)| { + e.shadow && (e.was_password || Some(i) == resolved.username_elem) + }), + multiple_passwords: resolved.multiple_passwords, + body_unavailable: self.body_unavailable, + }; + let mut classification = classify(&steps, &requests, &flags); + if went_on_after_login { + classification + .notes + .push("After the login the browser went on to another site.".to_owned()); + } + Recording { + format_version: kharon_recording::FORMAT_VERSION, + tool_version: self.tool_version.clone(), + browser: self.browser.clone().unwrap_or(BrowserInfo { + product: String::new(), + user_agent: String::new(), + viewport: Viewport { + width: 0, + height: 0, + }, + }), + listener_world: self.listener_world, + started_at: end.started_at, + ended_at: end.ended_at, + ended_by: end.ended_by, + success_confirmed: end.success_confirmed, + start_url, + final_url, + steps, + flags, + classification, + stripped: StrippedCounts { + requests_seen: self.requests_seen, + requests_stored: requests.len() as u64, + request_bodies_reduced: bodies_reduced, + query_values_removed: tally.query_values_removed, + path_segments_replaced: tally.path_segments_replaced, + typed_values_removed: resolved.markers.len() as u64, + steps_dropped: self.steps_dropped, + }, + requests, + } + } + + /// An `Input` step after the first password input whose element looks like a one-time code + /// field. Judged on the element as the page described it, before any attribute is dropped. + fn otp_step(&self) -> bool { + let none = Ctx { + markers: &[], + pair: None, + }; + let mut seen_password = false; + for p in &self.steps { + let What::Input(i) = p.what else { continue }; + let e = &self.elems[i]; + if seen_password + && is_otp_descriptor(&describe( + &e.raw, + e.was_password, + none, + &mut Tally::default(), + )) + { + return true; + } + seen_password |= e.was_password; + } + false + } + + fn build_steps( + &self, + resolved: &resolve::Resolved, + ctx: Ctx<'_>, + tally: &mut Tally, + ) -> Vec { + self.steps + .iter() + .map(|p| { + let frame = frame_ref(&p.frame, ctx); + let t_ms = p.t_ms; + match &p.what { + What::Navigation(url) => Step::Navigation { + t_ms, + frame, + url: stripped_url(url, ctx, tally), + }, + What::Input(i) => { + let e = &self.elems[*i]; + let role = if e.was_password { + Role::Password + } else if Some(*i) == resolved.username_elem { + Role::Username + } else { + Role::None + }; + Step::Input { + t_ms, + frame, + element: describe(&e.raw, e.was_password, ctx, tally), + value_len: u32::try_from(e.value.chars().count()).unwrap_or(u32::MAX), + role, + } + } + What::Click(raw) => Step::Click { + t_ms, + frame, + element: self.describe_known(raw, ctx, tally), + }, + What::EnterKey(raw) => Step::EnterKey { + t_ms, + frame, + element: self.describe_known(raw, ctx, tally), + }, + What::Submit(form) => Step::Submit { + t_ms, + frame, + form: form_ref(form, ctx, tally), + }, + } + }) + .collect() + } + + fn describe_known( + &self, + raw: &RawElement, + ctx: Ctx<'_>, + tally: &mut Tally, + ) -> kharon_recording::ElementDescriptor { + let was_password = raw.is_password_now + || self + .elems + .iter() + .any(|e| same_element(e, raw) && e.was_password); + describe(raw, was_password, ctx, tally) + } +} + +#[cfg(test)] +mod tests { + use base64ct::{Base64, Encoding}; + use kharon_recording::{ + BodyShape, BrowserInfo, EndedBy, ListenerWorld, MarkerHit, PasswordLeg, Recording, + ResourceKind, Role, Shape, Step, StoredRequest, Viewport, + }; + use zeroize::Zeroizing; + + use super::*; + use crate::body::MAX_BODY_BYTES; + use crate::testing::assert_holds_no_typed_value; + + const LOGIN: &str = "https://ex.test/login"; + const USER: &str = "kh.throwaway7"; + const PASS: &str = "Tr0ub4dor-9x"; + const PASS2: &str = "Qx7!other-pass"; + const OTP: &str = "482913"; + const TENANT: &str = "acme-corp-77"; + /// Every value typed in any scenario below. + const ALL_VALUES: [&str; 5] = [USER, PASS, PASS2, OTP, TENANT]; + const FORM: &str = "application/x-www-form-urlencoded"; + + fn el(id: u32, name: &str, ty: &str) -> RawElement { + RawElement { + element_id: id, + document_id: "doc1".into(), + tag: "INPUT".into(), + input_type: Some(ty.to_uppercase()), + id: Some(name.into()), + name: Some(name.into()), + autocomplete: None, + placeholder: None, + aria_label: None, + is_password_now: ty == "password", + form: Some(RawForm { + index: 0, + action: Some("https://ex.test/login".into()), + method: "post".into(), + }), + selectors: vec![format!("#{name}")], + in_shadow_root: false, + } + } + + fn user_el() -> RawElement { + el(1, "username", "text") + } + + fn pw_el() -> RawElement { + el(2, "password", "password") + } + + fn no_form(mut e: RawElement) -> RawElement { + e.form = None; + e + } + + fn button() -> RawElement { + let mut b = el(9, "go", "submit"); + b.tag = "button".into(); + b + } + + struct Session { + cap: Capture, + t: u64, + rid: u32, + url: String, + } + + impl Session { + fn unstarted(url: &str) -> Self { + Session { + cap: Capture::new(url, "0.0.0"), + t: 0, + rid: 0, + url: url.into(), + } + } + + fn new(url: &str) -> Self { + let mut s = Self::unstarted(url); + s.start(); + s + } + + fn start(&mut self) { + self.cap.push(CaptureEvent::Started { + browser: BrowserInfo { + product: "Chrome/1".into(), + user_agent: "UA".into(), + viewport: Viewport { + width: 1280, + height: 800, + }, + }, + listener_world: ListenerWorld::Isolated, + }); + let url = self.url.clone(); + self.nav_to(&url); + } + + fn tick(&mut self) -> u64 { + self.t += 10; + self.t + } + + fn frame(&self) -> RawFrame { + RawFrame { + main: true, + url: self.url.clone(), + } + } + + fn nav_to(&mut self, url: &str) { + self.url = url.into(); + let (t_ms, frame) = (self.tick(), self.frame()); + self.cap.push(CaptureEvent::Navigation { t_ms, frame }); + } + + fn nav(mut self, url: &str) -> Self { + self.nav_to(url); + self + } + + fn child_nav(mut self, url: &str) -> Self { + let t_ms = self.tick(); + self.cap.push(CaptureEvent::Navigation { + t_ms, + frame: RawFrame { + main: false, + url: url.into(), + }, + }); + self + } + + fn page(&mut self, kind: PageEventKind, e: &RawElement, value: Option<&str>) { + let (t_ms, frame) = (self.tick(), self.frame()); + self.cap.push(CaptureEvent::Page { + t_ms, + kind, + frame, + element: e.clone(), + value: value.map(|v| Zeroizing::new(v.to_owned())), + }); + } + + fn type_into(mut self, e: &RawElement, text: &str) -> Self { + self.page(PageEventKind::Input, e, Some(text)); + self + } + + /// One Input event per character, each carrying the value so far. + fn typing(mut self, e: &RawElement, text: &str) -> Self { + let ends: Vec = text.char_indices().map(|(i, c)| i + c.len_utf8()).collect(); + for end in ends { + self.page(PageEventKind::Input, e, Some(&text[..end])); + } + self + } + + fn change(mut self, e: &RawElement, text: &str) -> Self { + self.page(PageEventKind::Change, e, Some(text)); + self + } + + fn click(mut self, e: &RawElement) -> Self { + self.page(PageEventKind::Click, e, None); + self + } + + fn submit(mut self, e: &RawElement) -> Self { + self.page(PageEventKind::Submit, e, None); + self + } + + fn enter(mut self, e: &RawElement) -> Self { + self.page(PageEventKind::EnterKey, e, None); + self + } + + fn popup(mut self) -> Self { + self.cap.push(CaptureEvent::Popup); + self + } + + fn request( + mut self, + method: &str, + url: &str, + resource: ResourceKind, + main: bool, + content_type: Option<&str>, + body: RawBody, + ) -> Self { + self.rid += 1; + let (t_ms, request_id) = (self.tick(), format!("r{}", self.rid)); + self.cap.push(CaptureEvent::Request { + request_id: request_id.clone(), + t_ms, + method: method.into(), + url: url.into(), + resource, + main_frame_navigation: main, + content_type: content_type.map(str::to_owned), + authorization: None, + body, + frame_url: self.url.clone(), + }); + self.cap.push(CaptureEvent::Response { + request_id, + status: 200, + content_type: Some("text/html; charset=utf-8".into()), + set_cookie: false, + }); + self + } + + fn post_form(self, url: &str, body: &str) -> Self { + self.request( + "POST", + url, + ResourceKind::Document, + true, + Some(FORM), + RawBody::Bytes(Zeroizing::new(body.as_bytes().to_vec())), + ) + } + + fn post_json(self, url: &str, body: &str) -> Self { + self.request( + "POST", + url, + ResourceKind::Fetch, + false, + Some("application/json"), + RawBody::Bytes(Zeroizing::new(body.as_bytes().to_vec())), + ) + } + + fn get(self, url: &str, resource: ResourceKind) -> Self { + self.request("GET", url, resource, false, None, RawBody::None) + } + + fn finish(self) -> Recording { + self.cap.finish(EndInfo { + ended_by: EndedBy::Confirmed, + success_confirmed: true, + started_at: "2026-11-18T00:00:00Z".into(), + ended_at: "2026-11-18T00:01:00Z".into(), + }) + } + } + + fn inputs(rec: &Recording) -> Vec<(&kharon_recording::ElementDescriptor, u32, Role)> { + rec.steps + .iter() + .filter_map(|s| match s { + Step::Input { + element, + value_len, + role, + .. + } => Some((element, *value_len, *role)), + _ => None, + }) + .collect() + } + + fn role_of(rec: &Recording, element_id: u32) -> Option { + inputs(rec) + .into_iter() + .find(|(e, _, _)| e.element_id == element_id) + .map(|(_, _, r)| r) + } + + // ---- scenarios, each a complete recording; the leak test walks the whole list ---- + + fn sc_form_post() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .click(&button()) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}&csrf=abc"), + ) + .get("https://ex.test/logo.png", ResourceKind::Other) + .nav("https://ex.test/home") + .finish() + } + + fn sc_keystrokes() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .typing(&pw_el(), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish() + } + + fn sc_show_toggle() -> Recording { + let mut shown = pw_el(); + shown.is_password_now = false; + shown.input_type = Some("text".into()); + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), "Tr0ub4") + .type_into(&shown, PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish() + } + + fn sc_confirm_field() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .type_into(&el(3, "confirm", "password"), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}&confirm={PASS}"), + ) + .finish() + } + + fn sc_two_passwords() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .type_into(&el(3, "confirm", "password"), PASS2) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}&confirm={PASS2}"), + ) + .finish() + } + + fn sc_two_step() -> Recording { + let mut pw = pw_el(); + pw.document_id = "doc2".into(); + Session::new(LOGIN) + .type_into(&user_el(), USER) + .post_form("https://ex.test/identify", &format!("username={USER}")) + .nav("https://ex.test/password") + .type_into(&pw, PASS) + .post_form("https://ex.test/login", &format!("password={PASS}")) + .finish() + } + + fn sc_username_fallback() -> Recording { + let tenant = no_form(el(4, "tenant", "text")); + Session::new(LOGIN) + .type_into(&tenant, TENANT) + .type_into(&no_form(user_el()), USER) + .type_into(&no_form(pw_el()), PASS) + .enter(&no_form(pw_el())) + .post_json( + "https://ex.test/api/login", + &format!(r#"{{"tenant":"{TENANT}","user":"{USER}","pass":"{PASS}"}}"#), + ) + .finish() + } + + fn sc_cross_site() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .nav("https://idp.other.test/auth") + .type_into(&pw_el(), PASS) + .post_form("https://idp.other.test/auth", &format!("u={USER}&p={PASS}")) + .finish() + } + + fn sc_subdomain() -> Recording { + Session::new(LOGIN) + .nav("https://login.ex.test/sso") + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .post_form("https://login.ex.test/sso", &format!("u={USER}&p={PASS}")) + .finish() + } + + fn sc_captcha() -> Recording { + Session::new(LOGIN) + .child_nav("https://newassets.hcaptcha.com/captcha/v1/frame.html") + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish() + } + + fn sc_recaptcha_request() -> Recording { + Session::new(LOGIN) + .get( + "https://www.google.com/recaptcha/api2/anchor?k=site", + ResourceKind::Other, + ) + .finish() + } + + fn sc_google_search() -> Recording { + Session::new(LOGIN) + .get("https://www.google.com/search?q=x", ResourceKind::Other) + .child_nav("https://www.google.com/maps") + .finish() + } + + fn sc_otp_after() -> Recording { + let mut otp = el(5, "code", "text"); + otp.autocomplete = Some("one-time-code".into()); + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .nav("https://ex.test/verify") + .type_into(&otp, OTP) + .post_form("https://ex.test/verify", &format!("code={OTP}")) + .finish() + } + + fn sc_otp_before() -> Recording { + let mut otp = el(5, "code", "text"); + otp.autocomplete = Some("one-time-code".into()); + Session::new(LOGIN) + .type_into(&otp, OTP) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}&code={OTP}"), + ) + .finish() + } + + fn sc_oversized() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .request( + "POST", + "https://ex.test/login", + ResourceKind::Fetch, + false, + Some("application/octet-stream"), + RawBody::Bytes(Zeroizing::new(vec![b'a'; MAX_BODY_BYTES + 1])), + ) + .finish() + } + + fn sc_combined_base64() -> Recording { + let combined = Base64::encode_string(format!("{USER}:{PASS}").as_bytes()); + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .post_json( + "https://ex.test/api/login", + &format!(r#"{{"auth":"{combined}","remember":true}}"#), + ) + .finish() + } + + fn sc_which_requests() -> Recording { + Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .get("https://ex.test/logo.png", ResourceKind::Other) + .post_json("https://ads.other.test/collect", r#"{"event":"view"}"#) + .post_json("https://ex.test/api/beacon", r#"{"event":"typed"}"#) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish() + } + + fn sc_values_everywhere() -> Recording { + let tenant = no_form(el(4, "tenant", "text")); + let basic = Base64::encode_string(format!("{USER}:{PASS}").as_bytes()); + let mut s = Session::new(&format!("https://ex.test/login?tenant={TENANT}")) + .type_into(&tenant, TENANT) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .get( + &format!("https://{TENANT}.ex.test/u/{USER}/profile?name={USER}&{PASS}=1"), + ResourceKind::Xhr, + ); + s.rid += 1; + s.cap.push(CaptureEvent::Request { + request_id: "auth".into(), + t_ms: 500, + method: "POST".into(), + url: format!("https://ex.test/api/session/{PASS}"), + resource: ResourceKind::Fetch, + main_frame_navigation: false, + content_type: Some(format!("multipart/form-data; boundary={TENANT}")), + authorization: Some(Zeroizing::new(format!("Basic {basic}").into_bytes())), + body: RawBody::Bytes(Zeroizing::new( + format!("--{TENANT}\r\nContent-Disposition: form-data; name=\"{USER}\"\r\n\r\n{PASS}\r\n--{TENANT}--\r\n") + .into_bytes(), + )), + frame_url: LOGIN.into(), + }); + s.nav(&format!("https://ex.test/home?welcome={USER}")) + .finish() + } + + type Build = fn() -> Recording; + + const SCENARIOS: &[(&str, Build)] = &[ + ("form_post", sc_form_post), + ("keystrokes", sc_keystrokes), + ("show_toggle", sc_show_toggle), + ("confirm_field", sc_confirm_field), + ("two_passwords", sc_two_passwords), + ("two_step", sc_two_step), + ("username_fallback", sc_username_fallback), + ("cross_site", sc_cross_site), + ("subdomain", sc_subdomain), + ("captcha", sc_captcha), + ("recaptcha_request", sc_recaptcha_request), + ("google_search", sc_google_search), + ("otp_after", sc_otp_after), + ("otp_before", sc_otp_before), + ("oversized", sc_oversized), + ("combined_base64", sc_combined_base64), + ("which_requests", sc_which_requests), + ("values_everywhere", sc_values_everywhere), + ]; + + fn scenario(name: &str) -> Recording { + SCENARIOS + .iter() + .find(|(n, _)| *n == name) + .map(|(_, build)| build()) + .unwrap_or_else(|| panic!("no scenario named {name}")) + } + + #[test] + fn the_serialized_recording_holds_no_typed_value() { + for (name, build) in SCENARIOS { + let rec = build(); + let json = rec.to_json(); + std::panic::catch_unwind(|| assert_holds_no_typed_value(&json, &ALL_VALUES)) + .unwrap_or_else(|_| panic!("scenario {name} leaked a typed value")); + } + } + + #[test] + fn form_post_end_to_end() { + let rec = scenario("form_post"); + assert_eq!(rec.classification.shape, Shape::FormPost); + let ins = inputs(&rec); + assert_eq!(ins.len(), 2); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 2), Some(Role::Password)); + assert_eq!(rec.stripped.typed_values_removed, 2); + assert_eq!( + rec.final_url.as_ref().map(|u| u.path.as_str()), + Some("/home") + ); + // Types are lowercased, the descriptor keeps the shape of the element. + assert_eq!(ins[0].0.tag, "input"); + assert_eq!(ins[1].0.input_type.as_deref(), Some("password")); + assert!(ins[1].0.was_password); + assert_eq!(rec.browser.product, "Chrome/1"); + } + + #[test] + fn keystrokes_collapse_to_one_input_step() { + let rec = scenario("keystrokes"); + let pw: Vec<_> = inputs(&rec) + .into_iter() + .filter(|(e, _, _)| e.element_id == 2) + .collect(); + assert_eq!(pw.len(), 1); + assert_eq!(pw[0].1, 12); + } + + #[test] + fn show_password_toggle_keeps_the_password() { + let rec = scenario("show_toggle"); + assert_eq!(role_of(&rec, 2), Some(Role::Password)); + assert_eq!(rec.classification.shape, Shape::FormPost); + assert!(!rec.flags.multiple_passwords); + } + + #[test] + fn confirm_field_with_the_same_value_is_one_password() { + let rec = scenario("confirm_field"); + assert!(!rec.flags.multiple_passwords); + assert_eq!(role_of(&rec, 3), Some(Role::Password)); + assert_eq!(rec.classification.shape, Shape::FormPost); + } + + #[test] + fn two_different_passwords_set_the_flag_and_block() { + let rec = scenario("two_passwords"); + assert!(rec.flags.multiple_passwords); + assert_eq!(rec.classification.shape, Shape::Unrecognized); + } + + #[test] + fn two_step_by_navigation() { + let rec = scenario("two_step"); + assert_eq!( + rec.classification.shape, + Shape::TwoStep { + password_leg: PasswordLeg::FormPost + } + ); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert!(!rec.flags.cross_site); + } + + #[test] + fn username_falls_back_to_the_value_sent_with_the_password() { + let rec = scenario("username_fallback"); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 4), Some(Role::None)); + assert_eq!(rec.classification.shape, Shape::ScriptPost); + // The tenant's hit stays in the file, with no role. + let sent_tenant = rec + .requests + .iter() + .flat_map(StoredRequest::hits) + .any(|h: &MarkerHit| h.role == Role::None); + assert!(sent_tenant); + } + + #[test] + fn cross_site_navigation_sets_the_flag() { + assert!(scenario("cross_site").flags.cross_site); + } + + const WENT_ON: &str = "After the login the browser went on to another site."; + + #[test] + fn a_move_to_another_site_after_the_login_is_a_note_and_no_flag() { + let rec = Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .click(&button()) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .nav("https://app.other.test/home") + .finish(); + assert!(!rec.flags.cross_site); + assert_eq!(rec.classification.shape, Shape::FormPost); + assert!( + rec.classification.notes.iter().any(|n| n == WENT_ON), + "{:?}", + rec.classification.notes + ); + } + + #[test] + fn a_move_to_another_site_before_the_password_is_sent_is_cross_site() { + let rec = scenario("cross_site"); + assert!(rec.flags.cross_site); + assert_eq!(rec.classification.shape, Shape::Unrecognized); + assert!(!rec.classification.notes.iter().any(|n| n == WENT_ON)); + } + + #[test] + fn with_no_request_carrying_the_password_the_last_typing_is_the_mark() { + // Typing, then a move to another site, then nothing sent that holds the password. + let after = Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .nav("https://app.other.test/home") + .finish(); + assert!(!after.flags.cross_site); + let before = Session::new(LOGIN) + .type_into(&user_el(), USER) + .nav("https://app.other.test/home") + .type_into(&pw_el(), PASS) + .finish(); + assert!(before.flags.cross_site); + } + + #[test] + fn a_browser_error_page_is_not_another_site() { + let rec = Session::new(LOGIN) + .type_into(&user_el(), USER) + .nav("chrome-error://chromewebdata/") + .type_into(&pw_el(), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish(); + assert!(!rec.flags.cross_site); + assert!(!rec.classification.notes.iter().any(|n| n == WENT_ON)); + } + + #[test] + fn same_site_subdomain_is_not_cross_site() { + assert!(!scenario("subdomain").flags.cross_site); + } + + #[test] + fn captcha_frame_sets_the_flag() { + assert!(scenario("captcha").flags.captcha); + assert!(scenario("recaptcha_request").flags.captcha); + assert!(!scenario("google_search").flags.captcha); + assert!(!scenario("form_post").flags.captcha); + } + + #[test] + fn otp_input_after_the_password_sets_the_flag() { + assert!(scenario("otp_after").flags.otp_step); + assert!(!scenario("otp_before").flags.otp_step); + assert!(!scenario("form_post").flags.otp_step); + } + + #[test] + fn oversized_body_is_unavailable_and_flagged() { + let rec = scenario("oversized"); + assert!(rec.flags.body_unavailable); + assert_eq!(rec.requests.len(), 1); + assert_eq!(rec.requests[0].body, BodyShape::Unavailable); + assert_eq!(rec.classification.shape, Shape::Unrecognized); + } + + #[test] + fn a_combined_base64_credential_in_a_json_body_is_a_plain_hit() { + let rec = scenario("combined_base64"); + assert_eq!(rec.classification.shape, Shape::ScriptPost); + } + + #[test] + fn a_json_login_under_a_urlencoded_header_is_a_script_post() { + // jQuery: $.post(url, JSON.stringify({user, pass})) sends JSON text under the default + // urlencoded content type. It must not read as a form whose one key is the JSON. + let rec = Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .click(&button()) + .request( + "POST", + "https://ex.test/api/login", + ResourceKind::Xhr, + false, + Some(FORM), + RawBody::Bytes(Zeroizing::new( + format!(r#"{{"user":"{USER}","pass":"{PASS}"}}"#).into_bytes(), + )), + ) + .nav("https://ex.test/home") + .finish(); + assert_eq!(rec.classification.shape, Shape::ScriptPost); + assert_eq!(rec.requests.len(), 1); + let login = &rec.requests[0]; + assert!(matches!(&login.body, BodyShape::Opaque { .. })); + assert!( + login + .hits() + .iter() + .any(|h| h.role == Role::Password && !h.low_confidence) + ); + } + + #[test] + fn a_weak_hit_does_not_store_a_request_or_pick_the_leg_when_a_confident_one_exists() { + // The password is five characters, so on its own it matches by chance. The combined + // credential is long enough to be a confident hit for both roles. + let (user, pass) = ("alice1", "abcde"); + let combined = Base64::encode_string(format!("{user}:{pass}").as_bytes()); + let rec = Session::new(LOGIN) + .type_into(&user_el(), user) + .type_into(&pw_el(), pass) + .post_json("https://ads.other.test/collect", r#"{"note":"abcde"}"#) + .post_json( + "https://ex.test/api/login", + &format!(r#"{{"auth":"{combined}"}}"#), + ) + .nav("https://ex.test/home") + .finish(); + let paths: Vec<&str> = rec.requests.iter().map(|r| r.url.path.as_str()).collect(); + assert_eq!(paths, ["/api/login"], "the chance match was stored"); + assert_eq!(rec.classification.shape, Shape::ScriptPost); + assert!(rec.classification.network_swap_candidate); + // With nothing confident anywhere, a weak hit still keeps its request. + let rec = Session::new(LOGIN) + .type_into(&user_el(), "bob") + .type_into(&pw_el(), "abcde") + .post_json("https://ads.other.test/collect", r#"{"note":"abcde"}"#) + .finish(); + assert_eq!(rec.requests.len(), 1); + } + + #[test] + fn only_the_three_kinds_of_request_are_stored() { + let rec = scenario("which_requests"); + // The image GET and the third-party POST are counted, not stored. + assert_eq!(rec.stripped.requests_seen, 4); + let urls: Vec<&str> = rec.requests.iter().map(|r| r.url.path.as_str()).collect(); + assert!( + urls.contains(&"/login"), + "the navigation, kept as a document: {urls:?}" + ); + assert!( + urls.contains(&"/api/beacon"), + "a POST to the login site after the first input" + ); + assert!(!urls.contains(&"/logo.png")); + assert!(!urls.contains(&"/collect")); + assert_eq!(rec.stripped.requests_stored, rec.requests.len() as u64); + assert_eq!(rec.stripped.request_bodies_reduced, 2); + } + + #[test] + fn requests_with_a_hit_are_stored_even_when_third_party() { + let rec = Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .post_json( + "https://sso.other.test/token", + &format!(r#"{{"p":"{PASS}"}}"#), + ) + .finish(); + assert_eq!(rec.requests.len(), 1); + assert_eq!(rec.requests[0].url.origin, "https://sso.other.test"); + } + + #[test] + fn events_before_started_are_ignored_for_steps() { + let mut s = Session::unstarted(LOGIN); + s = s + .type_into(&user_el(), USER) + .click(&button()) + .post_form("https://ex.test/login", "a=b") + .popup(); + assert!(!s.cap.started()); + s.start(); + assert!(s.cap.started()); + let rec = s + .type_into(&pw_el(), PASS) + .post_form("https://ex.test/login", &format!("password={PASS}")) + .finish(); + assert_eq!(rec.stripped.requests_seen, 1); + assert_eq!(inputs(&rec).len(), 1); + assert!(!rec.steps.iter().any(|s| matches!(s, Step::Click { .. }))); + // A pop-up is remembered whenever it arrives, and the earlier typing never counted. + assert!(rec.flags.popup); + assert_eq!(rec.stripped.typed_values_removed, 1); + } + + #[test] + fn a_second_started_is_ignored_and_unknown_responses_too() { + let mut s = Session::new(LOGIN); + s.start(); + s.cap.push(CaptureEvent::Response { + request_id: "nope".into(), + status: 500, + content_type: None, + set_cookie: true, + }); + let rec = s.finish(); + assert_eq!(rec.browser.product, "Chrome/1"); + assert!(rec.requests.is_empty()); + } + + #[test] + fn finish_on_a_capture_that_never_started_still_returns_a_recording() { + let rec = Session::unstarted(LOGIN).finish(); + assert!(rec.steps.is_empty() && rec.requests.is_empty()); + assert_eq!(rec.browser.product, ""); + assert_eq!(rec.browser.viewport.width, 0); + assert_eq!(rec.listener_world, ListenerWorld::Isolated); + assert_eq!(rec.stripped.requests_seen, 0); + } + + #[test] + fn change_adds_no_step_and_click_adds_one_every_time() { + let rec = Session::new(LOGIN) + .change(&user_el(), USER) + .click(&button()) + .click(&button()) + .submit(&no_form(button())) + .submit(&pw_el()) + .finish(); + assert!(inputs(&rec).is_empty()); + let clicks = rec + .steps + .iter() + .filter(|s| matches!(s, Step::Click { .. })) + .count(); + let submits = rec + .steps + .iter() + .filter(|s| matches!(s, Step::Submit { .. })) + .count(); + assert_eq!((clicks, submits), (2, 1)); + // The changed value still counts as typed. + assert_eq!(rec.stripped.typed_values_removed, 1); + assert_holds_no_typed_value(&rec.to_json(), &[USER]); + } + + #[test] + fn an_element_attribute_holding_a_typed_value_is_dropped() { + let mut e = user_el(); + e.id = Some(format!("user-{USER}")); + e.placeholder = Some(format!("e.g. {USER}")); + e.selectors = vec![ + format!(r"#user-{}", USER.replace('.', r"\.")), + "input[name=username]".into(), + ]; + let rec = Session::new(LOGIN).type_into(&e, USER).finish(); + let (descriptor, _, _) = inputs(&rec)[0]; + assert_eq!(descriptor.id, None); + assert_eq!(descriptor.placeholder, None); + assert_eq!(descriptor.name.as_deref(), Some("username")); + assert_eq!( + descriptor.selectors, + vec!["input[name=username]".to_owned()] + ); + assert_holds_no_typed_value(&rec.to_json(), &[USER]); + } + + #[test] + fn child_frames_are_cross_origin_and_a_framed_login_is_flagged() { + let mut s = Session::new(LOGIN); + let frame = RawFrame { + main: false, + url: "https://pay.other.test/frame".into(), + }; + let (t_ms, element) = (50, pw_el()); + s.cap.push(CaptureEvent::Page { + t_ms, + kind: PageEventKind::Input, + frame, + element, + value: Some(Zeroizing::new(PASS.to_owned())), + }); + let rec = s.finish(); + assert!(rec.flags.framed_login); + match &rec.steps[1] { + Step::Input { frame, .. } => { + assert!(!frame.main && frame.cross_origin); + assert_eq!(frame.origin, "https://pay.other.test"); + } + other => panic!("expected an input step, got {other:?}"), + } + } + + #[test] + fn shadow_dom_and_popup_flags() { + let mut shadowed = pw_el(); + shadowed.in_shadow_root = true; + let rec = Session::new(LOGIN) + .type_into(&shadowed, PASS) + .popup() + .finish(); + assert!(rec.flags.shadow_dom && rec.flags.popup); + } + + #[test] + fn hostile_event_order_does_not_panic() { + let mut cap = Capture::new("not a url", "0.0.0"); + let response = |id: &str| CaptureEvent::Response { + request_id: id.into(), + status: 200, + content_type: None, + set_cookie: false, + }; + cap.push(response("ghost")); + cap.push(CaptureEvent::Popup); + cap.push(CaptureEvent::Started { + browser: BrowserInfo { + product: String::new(), + user_agent: String::new(), + viewport: Viewport { + width: 0, + height: 0, + }, + }, + listener_world: ListenerWorld::Main, + }); + cap.push(response("ghost")); + cap.push(CaptureEvent::Navigation { + t_ms: 0, + frame: RawFrame { + main: false, + url: "::::".into(), + }, + }); + let mut empty_doc = pw_el(); + empty_doc.document_id = String::new(); + empty_doc.form = Some(RawForm { + index: u32::MAX, + action: Some("%%%".into()), + method: "\u{0}".into(), + }); + for kind in [ + PageEventKind::Input, + PageEventKind::Submit, + PageEventKind::Click, + ] { + cap.push(CaptureEvent::Page { + t_ms: u64::MAX, + kind, + frame: RawFrame { + main: false, + url: String::new(), + }, + element: empty_doc.clone(), + value: Some(Zeroizing::new(String::new())), + }); + } + for url in ["", "http://", "data:text/html,x", "http://[::1", "\u{0}"] { + cap.push(CaptureEvent::Request { + request_id: "r".into(), + t_ms: 0, + method: "".into(), + url: url.into(), + resource: ResourceKind::Other, + main_frame_navigation: true, + content_type: Some("multipart/form-data".into()), + authorization: Some(Zeroizing::new(vec![0xff, 0x00])), + body: RawBody::Bytes(Zeroizing::new(vec![0xff, 0xfe])), + frame_url: "????".into(), + }); + } + cap.push(response("r")); + let rec = cap.finish(EndInfo { + ended_by: EndedBy::WindowClosed, + success_confirmed: false, + started_at: String::new(), + ended_at: String::new(), + }); + assert!(rec.flags.popup); + assert_eq!(rec.stripped.requests_seen, 5); + let _ = rec.to_json(); + } + + impl Capture { + /// A capture with a small body budget, for the test of that budget. + fn with_body_budget(start_url: &str, tool_version: &str, budget: usize) -> Self { + Self { + body_budget: budget, + ..Self::new(start_url, tool_version) + } + } + + /// A capture with small step and request limits, for the tests of those limits. + fn with_limits(start_url: &str, max_steps: usize, max_requests: usize) -> Self { + Self { + max_steps, + max_requests, + ..Self::new(start_url, "0.0.0") + } + } + } + + fn session_with_limits(max_steps: usize, max_requests: usize) -> Session { + let mut s = Session::unstarted(LOGIN); + s.cap = Capture::with_limits(LOGIN, max_steps, max_requests); + s.start(); + s + } + + #[test] + fn steps_past_the_limit_are_counted_but_their_values_still_count_as_typed() { + // Step 1 is the first navigation, step 2 the user name; the password input and the + // clicks after them do not fit. + let rec = session_with_limits(2, MAX_BUFFERED_REQUESTS) + .type_into(&user_el(), USER) + .type_into(&pw_el(), PASS) + .click(&button()) + .click(&button()) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish(); + assert_eq!(rec.steps.len(), 2); + assert_eq!(rec.stripped.steps_dropped, 3); + // The dropped password input is still a typed value: it is found and never written. + assert_eq!(rec.stripped.typed_values_removed, 2); + assert!( + rec.requests + .iter() + .flat_map(StoredRequest::hits) + .any(|h| h.role == Role::Password) + ); + assert_holds_no_typed_value(&rec.to_json(), &[USER, PASS]); + // Below the limit nothing is dropped. + assert_eq!(scenario("form_post").stripped.steps_dropped, 0); + assert_eq!(MAX_STEPS, 10_000); + } + + #[test] + fn requests_past_the_limit_are_counted_and_not_buffered() { + let mut s = session_with_limits(MAX_STEPS, 3).type_into(&user_el(), USER); + for i in 0..6 { + s = s.post_form(&format!("https://ex.test/p{i}"), "a=1"); + } + let rec = s.finish(); + assert_eq!(rec.stripped.requests_seen, 6); + // Main-frame navigations are always stored, so what is stored is what was buffered. + let paths: Vec<&str> = rec.requests.iter().map(|r| r.url.path.as_str()).collect(); + assert_eq!(paths, ["/p0", "/p1", "/p2"]); + assert_eq!(MAX_BUFFERED_REQUESTS, 5_000); + } + + #[test] + fn an_element_typed_into_endlessly_keeps_its_place_in_the_username_choice() { + // Thousands of events on one field, then the password: the user name is still found + // from the last thing typed before the password, with no per-event record kept. + let mut s = Session::new(LOGIN); + for _ in 0..5_000 { + s = s.type_into(&user_el(), USER); + } + let rec = s + .type_into(&pw_el(), PASS) + .post_form( + "https://ex.test/login", + &format!("username={USER}&password={PASS}"), + ) + .finish(); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(rec.classification.shape, Shape::FormPost); + } + + fn session_with_budget(budget: usize) -> Session { + let mut s = Session::unstarted(LOGIN); + s.cap = Capture::with_body_budget(LOGIN, "0.0.0", budget); + s.start(); + s + } + + #[test] + fn bodies_past_the_buffer_limit_are_unavailable() { + let big = |n: usize| RawBody::Bytes(Zeroizing::new(vec![b'a'; n])); + let rec = session_with_budget(100) + .type_into(&user_el(), USER) + .request( + "POST", + "https://ex.test/a", + ResourceKind::Fetch, + false, + Some("text/plain"), + big(80), + ) + .request( + "POST", + "https://ex.test/b", + ResourceKind::Fetch, + false, + Some("text/plain"), + big(80), + ) + .finish(); + assert_eq!(rec.requests.len(), 2); + assert!(matches!( + rec.requests[0].body, + BodyShape::Opaque { len: 80, .. } + )); + assert_eq!(rec.requests[1].body, BodyShape::Unavailable); + assert!(rec.flags.body_unavailable); + assert_eq!(rec.stripped.request_bodies_reduced, 1); + assert_eq!(MAX_BUFFERED_BODY_BYTES, 32 << 20); + } + + #[test] + fn a_combined_credential_alone_picks_the_username() { + let combined = Base64::encode_string(format!("{USER}:{PASS}").as_bytes()); + let tenant = no_form(el(4, "tenant", "text")); + let rec = Session::new(LOGIN) + .type_into(&no_form(user_el()), USER) + .type_into(&tenant, TENANT) + .type_into(&no_form(pw_el()), PASS) + .post_json( + "https://ex.test/api/login", + &format!(r#"{{"auth":"{combined}"}}"#), + ) + .finish(); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 4), Some(Role::None)); + assert_holds_no_typed_value(&rec.to_json(), &ALL_VALUES); + } + + #[test] + fn the_same_form_beats_a_later_typed_candidate() { + let tenant = no_form(el(4, "tenant", "text")); + let rec = Session::new(LOGIN) + .type_into(&user_el(), USER) + .type_into(&tenant, TENANT) + .type_into(&pw_el(), PASS) + .post_form("https://ex.test/login", &format!("t={TENANT}&p={PASS}")) + .finish(); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 4), Some(Role::None)); + } + + #[test] + fn a_value_found_in_any_request_beats_a_later_typed_one() { + let tenant = no_form(el(4, "tenant", "text")); + let rec = Session::new(LOGIN) + .type_into(&no_form(user_el()), USER) + .type_into(&tenant, TENANT) + .type_into(&no_form(pw_el()), PASS) + .post_form("https://ex.test/identify", &format!("u={USER}")) + .post_form("https://ex.test/login", &format!("p={PASS}")) + .finish(); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 4), Some(Role::None)); + } + + #[test] + fn with_no_other_evidence_the_last_typed_before_the_password_wins() { + let tenant = no_form(el(4, "tenant", "text")); + let rec = Session::new(LOGIN) + .type_into(&tenant, TENANT) + .type_into(&no_form(user_el()), USER) + .type_into(&no_form(pw_el()), PASS) + .finish(); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 4), Some(Role::None)); + } + + #[test] + fn an_edit_after_the_password_does_not_beat_the_last_field_typed_before_it() { + let first = no_form(el(4, "tenant", "text")); + let rec = Session::new(LOGIN) + .type_into(&first, TENANT) + .type_into(&no_form(user_el()), USER) + .type_into(&no_form(pw_el()), PASS) + .type_into(&first, TENANT) + .finish(); + assert_eq!(role_of(&rec, 1), Some(Role::Username)); + assert_eq!(role_of(&rec, 4), Some(Role::None)); + } + + #[test] + fn typed_values_in_every_kept_attribute_are_dropped() { + let mut e = user_el(); + e.name = Some(format!("n-{USER}")); + e.autocomplete = Some(format!("x-{USER}")); + e.aria_label = Some(format!("for {USER}")); + e.form = Some(RawForm { + index: 0, + action: Some(format!("https://ex.test/login?who={USER}&{USER}=1")), + method: "post".into(), + }); + let mut clicked = button(); + clicked.id = Some(format!("go-{USER}")); + clicked.selectors = vec![format!("#go-{USER}"), "button[type=submit]".into()]; + let rec = Session::new(LOGIN) + .type_into(&e, USER) + .click(&clicked) + .finish(); + let (d, _, _) = inputs(&rec)[0]; + assert_eq!( + (&d.name, &d.autocomplete, &d.aria_label), + (&None, &None, &None) + ); + let action = d.form.as_ref().and_then(|f| f.action.as_ref()); + assert!(action.is_some_and(|a| a.query_keys.iter().all(|k| k != USER))); + match &rec.steps[2] { + Step::Click { element, .. } => { + assert_eq!(element.id, None); + assert_eq!(element.selectors, vec!["button[type=submit]".to_owned()]); + } + other => panic!("expected a click, got {other:?}"), + } + assert_holds_no_typed_value(&rec.to_json(), &[USER]); + } + + #[test] + fn page_derived_tag_and_type_are_checked() { + let mut odd = user_el(); + odd.tag = "x.weird".into(); + odd.input_type = Some("weird".into()); + let mut custom = el(6, "c", "TEL"); + custom.tag = "My-Widget".into(); + let mut typed = el(7, "d", "text"); + typed.tag = format!("x-{USER}"); + let rec = Session::new(LOGIN) + .type_into(&odd, USER) + .type_into(&custom, "1") + .type_into(&typed, "2") + .finish(); + let ins = inputs(&rec); + assert_eq!(ins[0].0.tag, "{other}"); + assert_eq!(ins[0].0.input_type.as_deref(), Some("{other}")); + assert_eq!(ins[1].0.tag, "my-widget"); + assert_eq!(ins[1].0.input_type.as_deref(), Some("tel")); + assert_eq!(ins[2].0.tag, "{other}"); + } + + #[test] + fn the_show_password_scenario_keeps_both_lengths() { + let rec = scenario("show_toggle"); + let lens: Vec<(u32, u32)> = inputs(&rec) + .into_iter() + .map(|(e, len, _)| (e.element_id, len)) + .collect(); + assert_eq!(lens, vec![(1, 13), (2, 12)]); + } +} diff --git a/crates/kharon-teach/src/capture/resolve.rs b/crates/kharon-teach/src/capture/resolve.rs new file mode 100644 index 0000000..aa5d843 --- /dev/null +++ b/crates/kharon-teach/src/capture/resolve.rs @@ -0,0 +1,182 @@ +//! Choosing which typed value is the username and which are passwords. + +use kharon_recording::Role; +use zeroize::Zeroizing; + +use super::{Buffered, RawBody, RawElement}; +use crate::marker::Marker; +use crate::strip::any_hit; + +/// Everything known about one element: its first report, and the latest value typed into it. +pub(super) struct Elem { + pub raw: RawElement, + pub value: Zeroizing, + pub was_password: bool, + pub shadow: bool, + pub has_step: bool, + /// Sequence numbers of the first and the latest value-bearing event. + pub first_seq: u64, + pub last_seq: u64, + /// The latest value-bearing event seen before any password element was known. Kept as a + /// running value so an element's bookkeeping has a fixed size however much a page types. + /// If an element only turns out to be a password after other fields were typed into, this + /// can be later than the first password input; the username choice then falls back on the + /// element's last input. + pub last_before_password: Option, +} + +impl Elem { + fn form_key(&self) -> Option<(&str, u32)> { + self.raw + .form + .as_ref() + .map(|f| (self.raw.document_id.as_str(), f.index)) + } +} + +/// One marker per element that holds a typed value, with the roles chosen. +pub(super) struct Resolved { + pub markers: Vec, + pub username_elem: Option, + pub username_marker: Option, + pub password_marker: Option, + pub multiple_passwords: bool, +} + +impl Resolved { + pub fn pair(&self) -> Option<(&Marker, &Marker)> { + match (self.username_marker, self.password_marker) { + (Some(u), Some(p)) => Some((self.markers.get(u)?, self.markers.get(p)?)), + _ => None, + } + } +} + +fn plain_marker(value: &str, role: Role) -> Marker { + Marker::new(Zeroizing::new(value.to_owned()), role) +} + +/// Whether `marker` (or, with `pair`, the combined credential) occurs in the request's URL, +/// authorization header or body, under any encoding. +fn carries_with(req: &Buffered, marker: &Marker, pair: Option<(&Marker, &Marker)>) -> bool { + let one = std::slice::from_ref(marker); + any_hit(req.url.as_bytes(), one, pair) + || req + .authorization + .as_ref() + .is_some_and(|a| any_hit(a, one, pair)) + || matches!(&req.body, RawBody::Bytes(b) if any_hit(b, one, pair)) +} + +fn carries(req: &Buffered, marker: &Marker) -> bool { + carries_with(req, marker, None) +} + +/// The candidate whose last input event before the first password input is the latest; failing +/// that (none typed before it), the one with the latest input of any kind. +fn pick(cands: &[usize], elems: &[Elem], first_password_seq: Option) -> Option { + let last_before = |i: usize| -> Option { + match first_password_seq { + None => Some(elems[i].last_seq), + Some(s) => elems[i].last_before_password.filter(|q| *q < s), + } + }; + cands + .iter() + .copied() + .filter_map(|i| last_before(i).map(|q| (q, i))) + .max() + .map(|(_, i)| i) + .or_else(|| cands.iter().copied().max_by_key(|&i| elems[i].last_seq)) +} + +pub(super) fn resolve(elems: &[Elem], reqs: &[Buffered]) -> Resolved { + let password_values: Vec<&str> = { + let mut seen: Vec<&str> = Vec::new(); + for e in elems + .iter() + .filter(|e| e.was_password && !e.value.is_empty()) + { + if !seen.contains(&e.value.as_str()) { + seen.push(e.value.as_str()); + } + } + seen + }; + let multiple_passwords = password_values.len() > 1; + let password_markers: Vec = password_values + .iter() + .map(|v| plain_marker(v, Role::Password)) + .collect(); + let first_password_seq = elems + .iter() + .filter(|e| e.was_password) + .map(|e| e.first_seq) + .min(); + let cands: Vec = (0..elems.len()) + .filter(|&i| !elems[i].was_password && !elems[i].value.is_empty()) + .collect(); + let probes: Vec<(usize, Marker)> = cands + .iter() + .map(|&i| (i, plain_marker(&elems[i].value, Role::None))) + .collect(); + let narrowed = |keep: &dyn Fn(usize, &Marker) -> bool| -> Vec { + probes + .iter() + .filter(|(i, m)| keep(*i, m)) + .map(|(i, _)| *i) + .collect() + }; + + let same_form = narrowed(&|i, _| { + elems[i].form_key().is_some_and(|k| { + elems + .iter() + .any(|p| p.was_password && p.form_key() == Some(k)) + }) + }); + // A login that sends only a combined credential matches when probed with the pair. + let combined = |r: &Buffered, m: &Marker| { + password_markers + .iter() + .any(|p| carries_with(r, m, Some((m, p)))) + }; + let with_password = narrowed(&|_, m| { + reqs.iter().any(|r| { + (carries(r, m) && password_markers.iter().any(|p| carries(r, p))) || combined(r, m) + }) + }); + let in_any_request = narrowed(&|_, m| reqs.iter().any(|r| carries(r, m) || combined(r, m))); + + let username_elem = [same_form, with_password, in_any_request, cands] + .iter() + .find_map(|level| pick(level, elems, first_password_seq)); + + let mut markers = Vec::new(); + let (mut username_marker, mut password_marker) = (None, None); + for (i, e) in elems.iter().enumerate() { + if e.value.is_empty() { + continue; + } + let role = if e.was_password { + Role::Password + } else if Some(i) == username_elem { + Role::Username + } else { + Role::None + }; + match role { + Role::Username => username_marker = Some(markers.len()), + Role::Password if password_marker.is_none() => password_marker = Some(markers.len()), + Role::Password | Role::None => {} + } + markers.push(plain_marker(&e.value, role)); + } + Resolved { + markers, + username_elem, + username_marker, + password_marker, + multiple_passwords, + } +} diff --git a/crates/kharon-teach/src/capture/types.rs b/crates/kharon-teach/src/capture/types.rs new file mode 100644 index 0000000..5efd2cf --- /dev/null +++ b/crates/kharon-teach/src/capture/types.rs @@ -0,0 +1,99 @@ +//! The events a capture takes and the description it ends with. None of these types derives +//! `Debug`: some hold a typed value. + +use kharon_recording::{BrowserInfo, EndedBy, ListenerWorld, ResourceKind}; +use zeroize::Zeroizing; + +/// An element as the page script described it. No value: that travels beside it. +#[derive(Clone)] +pub struct RawElement { + pub element_id: u32, + pub document_id: String, + pub tag: String, + pub input_type: Option, + pub id: Option, + pub name: Option, + pub autocomplete: Option, + pub placeholder: Option, + pub aria_label: Option, + pub is_password_now: bool, + pub form: Option, + pub selectors: Vec, + pub in_shadow_root: bool, +} + +#[derive(Clone)] +pub struct RawForm { + pub index: u32, + /// The resolved absolute `form.action` DOM property, not the attribute text. + pub action: Option, + pub method: String, +} + +#[derive(Clone)] +pub struct RawFrame { + pub main: bool, + pub url: String, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum PageEventKind { + Input, + Change, + Click, + Submit, + EnterKey, +} + +pub enum RawBody { + None, + Unavailable, + Bytes(Zeroizing>), +} + +pub enum CaptureEvent { + /// The main frame left the notice page. + Started { + browser: BrowserInfo, + listener_world: ListenerWorld, + }, + /// Main and child frames alike. + Navigation { + t_ms: u64, + frame: RawFrame, + }, + /// `value` is set only for `Input` and `Change`. + Page { + t_ms: u64, + kind: PageEventKind, + frame: RawFrame, + element: RawElement, + value: Option>, + }, + Request { + request_id: String, + t_ms: u64, + method: String, + url: String, + resource: ResourceKind, + main_frame_navigation: bool, + content_type: Option, + authorization: Option>>, + body: RawBody, + frame_url: String, + }, + Response { + request_id: String, + status: u16, + content_type: Option, + set_cookie: bool, + }, + Popup, +} + +pub struct EndInfo { + pub ended_by: EndedBy, + pub success_confirmed: bool, + pub started_at: String, + pub ended_at: String, +} diff --git a/crates/kharon-teach/src/cdp.rs b/crates/kharon-teach/src/cdp.rs new file mode 100644 index 0000000..f95d6bf --- /dev/null +++ b/crates/kharon-teach/src/cdp.rs @@ -0,0 +1,766 @@ +//! A small Chrome DevTools Protocol client over a loopback WebSocket that makes +//! calls and receives events. +//! +//! Everything arriving on this socket may contain what a person typed into a +//! login page. Nothing from a frame reaches an error message, a log line or a +//! `Debug` output: errors are built from fixed words and numbers only. + +use crate::browser::DevTools; +use futures_util::{SinkExt, StreamExt}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; +use tokio::sync::{oneshot, watch}; +use tokio_tungstenite::connect_async_with_config; +use tokio_tungstenite::tungstenite::protocol::WebSocketConfig; +use tokio_tungstenite::tungstenite::{self, Message}; +use zeroize::Zeroizing; + +const MAX_REMOTE_MESSAGE_CHARS: usize = 200; +/// CDP frames that carry request bodies can be large. +const MAX_FRAME_BYTES: usize = 64 * 1024 * 1024; +/// How long the WebSocket handshake may take. +pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// How long a call may wait for its reply. +pub const CALL_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Debug, thiserror::Error)] +pub enum CdpError { + #[error("cdp connection closed")] + Closed, + #[error("cdp call timed out")] + Timeout, + #[error("cdp error {code}: {message}")] + Remote { code: i64, message: String }, + #[error("cdp transport: {0}")] + Transport(String), + #[error("cdp reply did not parse ({category} at line {line} column {column})")] + Parse { + category: &'static str, + line: usize, + column: usize, + }, +} + +impl From for CdpError { + /// Keeps the error's category and position only: its message can quote + /// the offending value. + fn from(e: serde_json::Error) -> Self { + use serde_json::error::Category; + let category = match e.classify() { + Category::Io => "io", + Category::Syntax => "syntax", + Category::Data => "data", + Category::Eof => "eof", + }; + CdpError::Parse { + category, + line: e.line(), + column: e.column(), + } + } +} + +/// Parse browser-derived JSON without ever letting the serde error text (which +/// can quote the offending value) out. +pub fn parse_quiet(raw: &str) -> Result { + serde_json::from_str(raw).map_err(CdpError::from) +} + +/// An event pushed by the browser. `params` is the params JSON text (`{}` when +/// absent); parse it with [`parse_quiet`]. +/// +/// Held in `Zeroizing` and with no `Debug`, because it can hold page data. +/// Not covered, and not coverable here: tungstenite's own read buffers and +/// serde's scratch buffers while a frame is parsed. +/// +/// ```compile_fail +/// fn needs_debug() {} +/// needs_debug::(); +/// ``` +pub struct Event { + pub method: String, + pub session_id: Option, + pub params: Zeroizing, +} + +/// What we send. Typed, never `json!`. (A call can carry a script, not typed +/// values, so this text is not zeroized.) +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct Outgoing<'a, P: Serialize> { + id: u64, + method: &'a str, + params: &'a P, + #[serde(skip_serializing_if = "Option::is_none")] + session_id: Option<&'a str>, +} + +/// Anything the browser sends: a reply (`id`) or an event (`method`). It +/// borrows from the frame text, which is held in `Zeroizing`. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Incoming<'a> { + #[serde(default)] + id: Option, + #[serde(default)] + method: Option, + #[serde(default)] + session_id: Option, + #[serde(default, borrow)] + params: Option<&'a RawValue>, + #[serde(default, borrow)] + result: Option<&'a RawValue>, + #[serde(default, borrow)] + error: Option<&'a RawValue>, +} + +#[derive(Deserialize)] +struct WireError { + #[serde(default)] + code: i64, + #[serde(default)] + message: String, +} + +type Reply = Result, CdpError>; + +#[derive(Default)] +struct State { + pending: HashMap>, + closed: bool, +} + +struct Shared { + state: Mutex, + done: watch::Sender, +} + +impl Shared { + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Mark closed, drop every waiter (their receivers see `Closed`) and tell + /// both socket tasks to stop. + fn close(&self) { + { + let mut st = self.lock(); + st.closed = true; + st.pending.clear(); + } + self.done.send_replace(true); + } +} + +/// Removes a call's pending entry however the call ends (reply, timeout, or +/// the caller dropping the future). +struct PendingGuard<'a> { + shared: &'a Shared, + id: u64, +} + +impl Drop for PendingGuard<'_> { + fn drop(&mut self) { + self.shared.lock().pending.remove(&self.id); + } +} + +struct Inner { + shared: Arc, + next_id: AtomicU64, + /// When the last clone drops, this closes, the writer ends and closes the + /// socket, and the reader follows. + out: UnboundedSender, +} + +#[derive(Clone)] +pub struct Cdp { + inner: Arc, +} + +fn transport(e: &tungstenite::Error) -> CdpError { + use tungstenite::Error as E; + let words = match e { + E::Io(io) if io.kind() == std::io::ErrorKind::ConnectionRefused => "connection refused", + E::Io(_) => "io error", + E::Url(_) => "bad url", + E::Http(_) | E::HttpFormat(_) => "handshake failed", + E::ConnectionClosed | E::AlreadyClosed => "connection closed", + _ => "protocol error", + }; + CdpError::Transport(words.to_owned()) +} + +impl Cdp { + /// Dials ws://127.0.0.1:. The host is a constant. + pub async fn connect(dev: &DevTools) -> Result<(Cdp, UnboundedReceiver), CdpError> { + if !dev.ws_path.starts_with("/devtools/") + || dev.ws_path.chars().any(|c| c.is_control() || c == ' ') + { + return Err(CdpError::Transport("bad websocket path".to_owned())); + } + let url = format!("ws://127.0.0.1:{}{}", dev.port, dev.ws_path); + let config = WebSocketConfig { + max_message_size: Some(MAX_FRAME_BYTES), + max_frame_size: Some(MAX_FRAME_BYTES), + ..WebSocketConfig::default() + }; + let dialed = tokio::time::timeout( + CONNECT_TIMEOUT, + connect_async_with_config(url, Some(config), false), + ) + .await + .map_err(|_| CdpError::Transport("handshake timed out".to_owned()))?; + let (ws, _resp) = dialed.map_err(|e| transport(&e))?; + let (sink, stream) = ws.split(); + + let (done, _) = watch::channel(false); + let shared = Arc::new(Shared { + state: Mutex::new(State::default()), + done, + }); + let (out, out_rx) = unbounded_channel(); + let (ev_tx, ev_rx) = unbounded_channel(); + tokio::spawn(read_loop(stream, Arc::clone(&shared), ev_tx)); + tokio::spawn(write_loop(sink, out_rx, Arc::clone(&shared))); + let cdp = Cdp { + inner: Arc::new(Inner { + shared, + next_id: AtomicU64::new(1), + out, + }), + }; + Ok((cdp, ev_rx)) + } + + pub async fn call( + &self, + method: &str, + params: &P, + session_id: Option<&str>, + ) -> Result { + let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed); + let frame = serde_json::to_string(&Outgoing { + id, + method, + params, + session_id, + }) + // The serde message could quote a value; say only that it failed. + .map_err(|_| CdpError::Transport("could not encode call".to_owned()))?; + + let (tx, rx) = oneshot::channel(); + { + let mut st = self.inner.shared.lock(); + if st.closed { + return Err(CdpError::Closed); + } + st.pending.insert(id, tx); + } + let _guard = PendingGuard { + shared: &self.inner.shared, + id, + }; + if self.inner.out.send(frame).is_err() { + return Err(CdpError::Closed); + } + let raw = tokio::time::timeout(CALL_TIMEOUT, rx) + .await + .map_err(|_| CdpError::Timeout)? + .map_err(|_| CdpError::Closed)??; + parse_quiet(&raw) + } +} + +type WsStream = + tokio_tungstenite::WebSocketStream>; + +/// Resolves once the connection is marked closed. The watch guard is dropped +/// before this returns, so it never blocks `Shared::close`. +async fn closed(done: &mut watch::Receiver) { + let _ = done.wait_for(|d| *d).await; +} + +/// Reads frames and routes them. The frame text is `Zeroizing` from the moment +/// it leaves tungstenite. +async fn read_loop( + mut stream: futures_util::stream::SplitStream, + shared: Arc, + events: UnboundedSender, +) { + let mut done = shared.done.subscribe(); + loop { + tokio::select! { + msg = stream.next() => match msg { + Some(Ok(Message::Text(t))) => route(&Zeroizing::new(t), &shared, &events), + Some(Ok(Message::Close(_))) | Some(Err(_)) | None => break, + // Binary, ping, pong: dropped. + Some(Ok(_)) => {} + }, + () = closed(&mut done) => break, + } + } + shared.close(); +} + +/// Writes queued calls. A blocked write never stops the reader. +async fn write_loop( + mut sink: futures_util::stream::SplitSink, + mut out: UnboundedReceiver, + shared: Arc, +) { + let mut done = shared.done.subscribe(); + loop { + let queued = tokio::select! { + q = out.recv() => q, + () = closed(&mut done) => None, + }; + let Some(text) = queued else { break }; + let sent = tokio::select! { + r = sink.send(Message::Text(text)) => r.is_ok(), + () = closed(&mut done) => false, + }; + if !sent { + break; + } + } + shared.close(); + let _ = tokio::time::timeout(Duration::from_millis(500), sink.close()).await; +} + +/// A frame that does not parse is dropped silently (never logged). +fn route(text: &Zeroizing, shared: &Shared, events: &UnboundedSender) { + let Ok(m) = serde_json::from_str::>(text) else { + return; + }; + if let Some(id) = m.id { + let Some(waiter) = shared.lock().pending.remove(&id) else { + return; // matches no pending call + }; + let reply = match (m.error, m.result) { + (Some(raw), _) => { + let w = parse_quiet::(raw.get()).unwrap_or(WireError { + code: 0, + message: String::new(), + }); + Err(CdpError::Remote { + code: w.code, + message: w.message.chars().take(MAX_REMOTE_MESSAGE_CHARS).collect(), + }) + } + (None, Some(result)) => Ok(Zeroizing::new(result.get().to_owned())), + (None, None) => Ok(Zeroizing::new("{}".to_owned())), + }; + let _ = waiter.send(reply); + } else if let Some(method) = m.method { + let params = m.params.map_or("{}", |p| p.get()); + let _ = events.send(Event { + method, + session_id: m.session_id, + params: Zeroizing::new(params.to_owned()), + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::{Empty, FrameNavigated}; + use futures_util::{SinkExt, StreamExt}; + use serde::Deserialize; + use std::future::Future; + use std::time::Duration; + use tokio::net::{TcpListener, TcpStream}; + use tokio::task::JoinHandle; + use tokio::time::timeout; + use tokio_tungstenite::{WebSocketStream, accept_async, tungstenite::Message}; + + type Ws = WebSocketStream; + const T: Duration = Duration::from_secs(5); + + /// Serve one connection with `f`; returns the DevTools to dial and the + /// server task's handle. + async fn serve(f: F) -> (DevTools, JoinHandle) + where + F: FnOnce(Ws) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + O: Send + 'static, + { + let l = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = l.local_addr().unwrap().port(); + let h = tokio::spawn(async move { + let (s, _) = l.accept().await.unwrap(); + f(accept_async(s).await.unwrap()).await + }); + ( + DevTools { + port, + ws_path: "/devtools/browser/test".to_owned(), + }, + h, + ) + } + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct Sent { + id: u64, + method: String, + #[serde(default)] + session_id: Option, + } + + async fn next_text(ws: &mut Ws) -> String { + loop { + match timeout(T, ws.next()).await.unwrap() { + Some(Ok(Message::Text(t))) => return t, + Some(Ok(_)) => continue, + other => panic!("server expected a text frame, got end: {}", other.is_none()), + } + } + } + + #[derive(Deserialize)] + struct Reply { + v: String, + } + + #[tokio::test] + async fn a_call_gets_its_own_reply() { + let (dev, srv) = serve(|mut ws| async move { + let a: Sent = serde_json::from_str(&next_text(&mut ws).await).unwrap(); + let b: Sent = serde_json::from_str(&next_text(&mut ws).await).unwrap(); + for s in [b, a] { + let reply = format!(r#"{{"id":{},"result":{{"v":"{}"}}}}"#, s.id, s.method); + ws.send(Message::Text(reply)).await.unwrap(); + } + ws + }) + .await; + let (cdp, _rx) = Cdp::connect(&dev).await.unwrap(); + let (ra, rb) = timeout(T, async { + tokio::join!( + cdp.call::<_, Reply>("A.first", &Empty {}, None), + cdp.call::<_, Reply>("B.second", &Empty {}, None) + ) + }) + .await + .unwrap(); + assert_eq!(ra.unwrap().v, "A.first"); + assert_eq!(rb.unwrap().v, "B.second"); + drop(srv); + } + + #[tokio::test] + async fn a_remote_error_is_returned() { + let (dev, _srv) = serve(|mut ws| async move { + let s: Sent = serde_json::from_str(&next_text(&mut ws).await).unwrap(); + let long = "n".repeat(500); + let r = format!( + r#"{{"id":{},"error":{{"code":-32000,"message":"{long}"}}}}"#, + s.id + ); + ws.send(Message::Text(r)).await.unwrap(); + ws + }) + .await; + let (cdp, _rx) = Cdp::connect(&dev).await.unwrap(); + let r = timeout(T, cdp.call::<_, Empty2>("X.y", &Empty {}, None)) + .await + .unwrap(); + match r { + Err(CdpError::Remote { code, message }) => { + assert_eq!(code, -32000); + assert_eq!(message.chars().count(), 200); + } + _ => panic!("expected Remote"), + } + } + + #[derive(Deserialize)] + struct Empty2 {} + + #[tokio::test] + async fn a_short_remote_message_is_kept_whole() { + let (dev, _srv) = serve(|mut ws| async move { + let s: Sent = serde_json::from_str(&next_text(&mut ws).await).unwrap(); + let r = format!( + r#"{{"id":{},"error":{{"code":-32601,"message":"nope"}}}}"#, + s.id + ); + ws.send(Message::Text(r)).await.unwrap(); + ws + }) + .await; + let (cdp, _rx) = Cdp::connect(&dev).await.unwrap(); + let r = timeout(T, cdp.call::<_, Empty2>("X.y", &Empty {}, None)) + .await + .unwrap(); + assert!( + matches!(r, Err(CdpError::Remote { code: -32601, ref message }) if message == "nope") + ); + } + + #[tokio::test] + async fn events_are_delivered_with_their_session() { + let (dev, _srv) = serve(|mut ws| async move { + let ev = r#"{"method":"Page.frameNavigated","sessionId":"S1","params":{"frame":{"id":"F","url":"https://a.test/","extra":1}}}"#; + ws.send(Message::Text(ev.to_owned())).await.unwrap(); + let ev2 = r#"{"method":"Target.targetDestroyed","params":{"targetId":"T"}}"#; + ws.send(Message::Text(ev2.to_owned())).await.unwrap(); + ws + }) + .await; + let (_cdp, mut rx) = Cdp::connect(&dev).await.unwrap(); + let e = timeout(T, rx.recv()).await.unwrap().unwrap(); + assert_eq!(e.method, "Page.frameNavigated"); + assert_eq!(e.session_id.as_deref(), Some("S1")); + let f: FrameNavigated = parse_quiet(&e.params).unwrap(); + assert_eq!( + (f.frame.id.as_str(), f.frame.url.as_str()), + ("F", "https://a.test/") + ); + assert!(f.frame.parent_id.is_none()); + let e = timeout(T, rx.recv()).await.unwrap().unwrap(); + assert_eq!(e.method, "Target.targetDestroyed"); + assert!(e.session_id.is_none()); + } + + #[tokio::test] + async fn session_id_is_sent_at_top_level() { + let (dev, srv) = serve(|mut ws| async move { + let mut seen = Vec::new(); + for _ in 0..2 { + let t = next_text(&mut ws).await; + let s: Sent = serde_json::from_str(&t).unwrap(); + ws.send(Message::Text(format!(r#"{{"id":{},"result":{{}}}}"#, s.id))) + .await + .unwrap(); + seen.push(t); + } + (seen, ws) + }) + .await; + let (cdp, _rx) = Cdp::connect(&dev).await.unwrap(); + timeout( + T, + cdp.call::<_, Empty2>("Runtime.enable", &Empty {}, Some("S1")), + ) + .await + .unwrap() + .unwrap(); + timeout( + T, + cdp.call::<_, Empty2>("Browser.getVersion", &Empty {}, None), + ) + .await + .unwrap() + .unwrap(); + let (seen, _ws) = timeout(T, srv).await.unwrap().unwrap(); + let first: Sent = serde_json::from_str(&seen[0]).unwrap(); + assert_eq!(first.id, 1); + assert_eq!(first.session_id.as_deref(), Some("S1")); + assert!(seen[0].contains(r#""params":{}"#)); + let second: Sent = serde_json::from_str(&seen[1]).unwrap(); + assert_eq!(second.id, 2); + assert!(second.session_id.is_none()); + assert!(!seen[1].contains("sessionId")); + } + + #[tokio::test] + async fn calls_fail_after_the_socket_closes() { + let (dev, _srv) = serve(|mut ws| async move { + // Read the first call, then close without answering. + let _ = next_text(&mut ws).await; + let _ = ws.close(None).await; + }) + .await; + let (cdp, mut rx) = Cdp::connect(&dev).await.unwrap(); + // The pending call fails. + let r = timeout(T, cdp.call::<_, Empty2>("A.a", &Empty {}, None)) + .await + .unwrap(); + assert!(matches!(r, Err(CdpError::Closed))); + // A later call fails too, without hanging. + let r = timeout(T, cdp.call::<_, Empty2>("A.b", &Empty {}, None)) + .await + .unwrap(); + assert!(matches!(r, Err(CdpError::Closed))); + // The event channel ends. + assert!(timeout(T, rx.recv()).await.unwrap().is_none()); + } + + #[tokio::test] + async fn stray_replies_and_junk_frames_are_dropped_silently() { + let (dev, _srv) = serve(|mut ws| async move { + let s: Sent = serde_json::from_str(&next_text(&mut ws).await).unwrap(); + ws.send(Message::Text( + r#"{"id":999,"result":{"v":"stray"}}"#.to_owned(), + )) + .await + .unwrap(); + ws.send(Message::Text("not json at all {{".to_owned())) + .await + .unwrap(); + ws.send(Message::Binary(vec![1, 2, 3])).await.unwrap(); + ws.send(Message::Text(format!( + r#"{{"id":{},"result":{{"v":"ok"}}}}"#, + s.id + ))) + .await + .unwrap(); + ws + }) + .await; + let (cdp, mut rx) = Cdp::connect(&dev).await.unwrap(); + let r = timeout(T, cdp.call::<_, Reply>("A.a", &Empty {}, None)) + .await + .unwrap() + .unwrap(); + assert_eq!(r.v, "ok"); + assert!(rx.try_recv().is_err(), "junk must not surface as an event"); + } + + #[tokio::test] + async fn dropping_the_last_clone_closes_the_socket() { + let (dev, srv) = serve(|mut ws| async move { + loop { + match ws.next().await { + None | Some(Err(_)) | Some(Ok(Message::Close(_))) => return, + Some(Ok(_)) => {} + } + } + }) + .await; + let (cdp, rx) = Cdp::connect(&dev).await.unwrap(); + let other = cdp.clone(); + drop(cdp); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!srv.is_finished(), "one clone is still alive"); + drop(other); + drop(rx); + timeout(T, srv).await.unwrap().unwrap(); + } + + #[tokio::test] + async fn a_bad_ws_path_is_refused() { + for bad in [ + "", + "/json/version", + "devtools/x", + "/devtools/\r\nHost: evil", + "//evil/devtools/x", + ] { + let dev = DevTools { + port: 1, + ws_path: bad.to_owned(), + }; + match Cdp::connect(&dev).await { + Err(CdpError::Transport(_)) => {} + _ => panic!("expected Transport for {bad:?}"), + } + } + } + + #[tokio::test] + async fn a_refused_connection_is_a_transport_error_in_fixed_words() { + let l = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = l.local_addr().unwrap().port(); + drop(l); + let dev = DevTools { + port, + ws_path: "/devtools/browser/x".to_owned(), + }; + match Cdp::connect(&dev).await { + Err(CdpError::Transport(w)) => assert_eq!(w, "connection refused"), + _ => panic!("expected Transport"), + } + } + + #[test] + fn a_parse_error_never_carries_the_value() { + #[derive(Debug, Deserialize)] + struct Wants { + #[allow(dead_code)] + value: u32, + } + let e = parse_quiet::(r#"{"value": "Tr0ub4dor-9x"}"#) + .map(|_| ()) + .unwrap_err(); + assert!(matches!( + e, + CdpError::Parse { + category: "data", + .. + } + )); + assert!(!e.to_string().contains("Tr0ub4dor-9x")); + assert!(!format!("{e:?}").contains("Tr0ub4dor-9x")); + let e = parse_quiet::(r#"{"value": 1"#) + .map(|_| ()) + .unwrap_err(); + assert!(matches!( + e, + CdpError::Parse { + category: "eof", + .. + } + )); + } + + #[tokio::test(start_paused = true)] + async fn a_call_with_no_reply_times_out_and_leaves_no_entry() { + let (dev, _srv) = serve(|mut ws| async move { + let _ = next_text(&mut ws).await; + tokio::time::sleep(Duration::from_secs(3600)).await; + ws + }) + .await; + let (cdp, _rx) = Cdp::connect(&dev).await.unwrap(); + let r = cdp.call::<_, Empty2>("A.a", &Empty {}, None).await; + assert!(matches!(r, Err(CdpError::Timeout))); + assert!(cdp.inner.shared.lock().pending.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn a_dropped_call_leaves_no_pending_entry() { + let (dev, _srv) = serve(|mut ws| async move { + let _ = next_text(&mut ws).await; + tokio::time::sleep(Duration::from_secs(3600)).await; + ws + }) + .await; + let (cdp, _rx) = Cdp::connect(&dev).await.unwrap(); + let r = tokio::time::timeout( + Duration::from_secs(1), + cdp.call::<_, Empty2>("A.a", &Empty {}, None), + ) + .await; + assert!(r.is_err(), "the caller's own timeout fires first"); + assert!(cdp.inner.shared.lock().pending.is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn a_handshake_that_never_finishes_times_out() { + let l = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = l.local_addr().unwrap().port(); + let _hold = tokio::spawn(async move { + let _conn = l.accept().await; + tokio::time::sleep(Duration::from_secs(3600)).await; + }); + let dev = DevTools { + port, + ws_path: "/devtools/browser/x".to_owned(), + }; + match Cdp::connect(&dev).await { + Err(CdpError::Transport(w)) => assert_eq!(w, "handshake timed out"), + _ => panic!("expected a handshake timeout"), + } + } +} diff --git a/crates/kharon-teach/src/cli.rs b/crates/kharon-teach/src/cli.rs new file mode 100644 index 0000000..a49c89c --- /dev/null +++ b/crates/kharon-teach/src/cli.rs @@ -0,0 +1,542 @@ +//! The command line: arguments, the refusals, the file name, the output path and the plain-words +//! summary of a recording. + +use kharon_recording::{ + EndedBy, ListenerWorld, PasswordLeg, Recording, RungCeiling, Shape, StrippedUrl, +}; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +/// How a recording file's name ends. +pub const FILE_SUFFIX: &str = ".kharon-recording.json"; + +#[derive(clap::Parser)] +#[command( + name = "kharon-teach", + version, + about = "Record how a login works so kharon can learn it", + args_conflicts_with_subcommands = true +)] +pub struct Cli { + #[command(subcommand)] + pub command: Option, + /// The login page to record (https, or http on this computer) + pub login_url: Option, + /// Where to write the recording (must not exist yet) + #[arg(long)] + pub out: Option, + /// The Chromium-family browser to use, instead of searching PATH + #[arg(long)] + pub browser: Option, + /// Skip the question after the notice; the notice is still shown + #[arg(long)] + pub yes: bool, +} + +#[derive(clap::Subcommand)] +pub enum Command { + /// Print what a recording file says, in plain words + Show { file: PathBuf }, +} + +/// Why the program will not start. One line each. +#[derive(Debug, thiserror::Error)] +pub enum Refusal { + #[error("give the login page's address, or `show `")] + NoUrl, + #[error("that is not a web address")] + NotAUrl, + #[error("the address must start with https:// (plain http only for this computer)")] + NotHttps, + #[error("the address must not hold a user name or password")] + Userinfo, + #[error("{0}")] + NoBrowser(String), + #[error("{} already exists; pick another --out", .0.display())] + OutExists(PathBuf), + #[error("cannot write in {}; it must be an existing directory you can write to", .0.display())] + OutDir(PathBuf), + #[error("cannot read {}: {}", .0.display(), .1)] + Unreadable(PathBuf, String), + #[error("{} is larger than 16 MiB, so it is not a recording", .0.display())] + TooLarge(PathBuf), + #[error("{} is not a recording kharon-teach can read: {}", .0.display(), .1)] + NotARecording(PathBuf, String), +} + +/// `https`, or `http` when the host is `localhost` or a loopback address; no user name or +/// password in it. +pub fn validate_login_url(s: &str) -> Result { + let u = url::Url::parse(s).map_err(|_| Refusal::NotAUrl)?; + if !u.username().is_empty() || u.password().is_some() { + return Err(Refusal::Userinfo); + } + let Some(host) = u.host() else { + return Err(Refusal::NotHttps); + }; + match u.scheme() { + "https" => Ok(u), + "http" => { + let local = match host { + url::Host::Domain(d) => d.eq_ignore_ascii_case("localhost"), + url::Host::Ipv4(a) => a.is_loopback(), + url::Host::Ipv6(a) => a.is_loopback(), + }; + if local { Ok(u) } else { Err(Refusal::NotHttps) } + } + _ => Err(Refusal::NotHttps), + } +} + +fn sanitize_host(host: &str) -> String { + host.to_ascii_lowercase() + .chars() + .map(|c| { + if c.is_ascii_lowercase() || c.is_ascii_digit() || c == '.' || c == '-' { + c + } else { + '_' + } + }) + .collect() +} + +/// `-.kharon-recording.json`. Every character of the host outside +/// `[a-z0-9.-]` becomes `_`, so an IPv6 host puts no `[`, `]` or `:` in a file name. +pub fn default_file_name(host: &str, utc: SystemTime) -> String { + let t = time::OffsetDateTime::from(utc); + format!( + "{}-{:04}{:02}{:02}T{:02}{:02}{:02}Z{FILE_SUFFIX}", + sanitize_host(host), + t.year(), + u8::from(t.month()), + t.day(), + t.hour(), + t.minute(), + t.second() + ) +} + +/// [`default_file_name`] for the instant a recording says it ended (`ended_at`, UTC RFC 3339 +/// as `session::utc_rfc3339` writes it). `None` when the text is not in that shape. +pub fn default_file_name_at(host: &str, ended_at: &str) -> Option { + let digits: Vec = ended_at + .chars() + .take(19) + .filter(char::is_ascii_digit) + .collect(); + if digits.len() != 14 || !ended_at.is_ascii() { + return None; + } + let d: String = digits.into_iter().collect(); + Some(format!( + "{}-{}T{}Z{FILE_SUFFIX}", + sanitize_host(host), + &d[..8], + &d[8..] + )) +} + +/// `dir` is an existing directory this user can create files in. Tried, not read from +/// permission bits: a temporary file is created and removed. +pub fn check_dir_writable(dir: &Path) -> Result<(), Refusal> { + use std::os::unix::fs::OpenOptionsExt; + let dir = if dir.as_os_str().is_empty() { + Path::new(".") + } else { + dir + }; + let bad = || Refusal::OutDir(dir.to_owned()); + if !std::fs::metadata(dir).is_ok_and(|m| m.is_dir()) { + return Err(bad()); + } + for _ in 0..4 { + let probe = dir.join(format!( + ".kharon-teach-probe-{:016x}", + rand::random::() + )); + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&probe) + { + Ok(_) => { + let _ = std::fs::remove_file(&probe); + return Ok(()); + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(_) => return Err(bad()), + } + } + Err(bad()) +} + +/// The file must not exist (a dangling symlink counts as existing) and its directory must be +/// writable. +pub fn check_out_path(p: &Path) -> Result<(), Refusal> { + if std::fs::symlink_metadata(p).is_ok() { + return Err(Refusal::OutExists(p.to_owned())); + } + check_dir_writable(p.parent().unwrap_or(Path::new(""))) +} + +/// `text` from a recording file with every character a terminal could act on replaced by +/// U+FFFD: control characters (escape sequences start with one), the line and paragraph +/// separators, and the bidirectional controls that can make text read differently from how it +/// is stored. A recording sent by someone else must not be able to drive the terminal. +fn plain(text: &str) -> String { + text.chars() + .map(|c| { + if c.is_control() + || matches!(c, '\u{2028}' | '\u{2029}' | '\u{202A}'..='\u{202E}' | '\u{2066}'..='\u{2069}') + { + '\u{FFFD}' + } else { + c + } + }) + .collect() +} + +fn site(u: &StrippedUrl) -> String { + plain(&format!("{}{}", u.origin, u.path)) +} + +/// What a recording says, in plain words, one fact per line. It is built from the recording +/// only and holds no JSON. +pub fn summary(r: &Recording) -> String { + let mut out: Vec = Vec::new(); + out.push(format!("Site: {}", site(&r.start_url))); + out.push( + match r.ended_by { + EndedBy::Confirmed => "How it ended: you pressed Enter to say the login worked.", + EndedBy::WindowClosed => "How it ended: the window was closed before you confirmed.", + } + .to_owned(), + ); + out.push(format!( + "How the login works: {}", + match &r.classification.shape { + Shape::FormPost => "One page; the form sends the password to the site as typed or encoded.".to_owned(), + Shape::ScriptPost => + "One page; the page's script sends the password to the site as typed or encoded.".to_owned(), + Shape::TwoStep { password_leg } => format!( + "Two steps; the user name goes first, then the password, which {}.", + match password_leg { + PasswordLeg::FormPost => "a form sends to the site as typed or encoded", + PasswordLeg::ScriptPost => "the page's script sends to the site as typed or encoded", + PasswordLeg::Transformed => "the page changes before it sends it", + } + ), + Shape::Transformed { digest } => match digest { + Some(d) => format!( + "The page changes the password ({} digest) before it sends it, so kharon cannot send it as typed.", + plain(d) + ), + None => "The page changes the password before it sends it, so kharon cannot send it as typed.".to_owned(), + }, + Shape::Unrecognized => + "kharon-teach did not recognize how this login sends the password.".to_owned(), + } + )); + out.push(format!( + "Strongest protection: {}.", + match r.classification.rung_ceiling { + RungCeiling::Milliseconds => + "kharon can fill and submit in one step, so the password is in the page for milliseconds", + RungCeiling::AgentContinues => "the agent has to continue after the fill", + RungCeiling::NeedsPerson => "a person has to look at this recording", + } + )); + out.push( + if r.classification.network_swap_candidate { + "Network swap: this login looks like one where the password can be put in at the network level." + } else { + "Network swap: this login does not look like a candidate." + } + .to_owned(), + ); + let f = &r.flags; + for (set, words) in [ + (f.captcha, "A captcha was on the page."), + (f.otp_step, "There was a one-time-code step."), + (f.cross_site, "The login went through another site."), + (f.popup, "The login opened a pop-up window."), + (f.framed_login, "The login form was inside a frame."), + (f.shadow_dom, "The login form was inside a shadow root."), + ( + f.multiple_passwords, + "The page had more than one password field.", + ), + ( + f.body_unavailable, + "Some request bodies could not be read, so the shape may be incomplete.", + ), + ] { + if set { + out.push(format!("Flag: {words}")); + } + } + for n in &r.classification.notes { + out.push(format!("Note: {}", plain(n))); + } + if r.listener_world == ListenerWorld::Main { + out.push( + "Caution: the page listener ran in the page's own script world, where the site's scripts can see and fake what it reports. Trust this recording less." + .to_owned(), + ); + } + let s = &r.stripped; + out.push(format!( + "Removed before saving: {} typed value(s), {} query value(s), {} path segment(s); {} of {} request(s) kept, {} body(ies) reduced to a shape.", + s.typed_values_removed, + s.query_values_removed, + s.path_segments_replaced, + s.requests_stored, + s.requests_seen, + s.request_bodies_reduced + )); + if s.steps_dropped > 0 { + out.push(format!( + "Not recorded: {} step(s) past the recorder's limit.", + s.steps_dropped + )); + } + let mut text = out.join("\n"); + text.push('\n'); + text +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use clap::CommandFactory; + use kharon_recording::{BrowserInfo, Classification, Flags, StrippedCounts, Viewport}; + use std::time::Duration; + + #[test] + fn accepts_https_and_loopback_http() { + for ok in [ + "https://ex.test/login", + "http://localhost:8080/", + "http://127.0.0.1:1/", + "http://[::1]:1/", + ] { + assert!(validate_login_url(ok).is_ok(), "{ok}"); + } + } + + #[test] + fn refuses_plain_http_userinfo_and_other_schemes() { + for bad in [ + "http://ex.test", + "https://u:p@ex.test", + "https://u@ex.test", + "file:///x", + "ex.test", + "ftp://ex.test/", + ] { + assert!(validate_login_url(bad).is_err(), "{bad}"); + } + assert!(matches!( + validate_login_url("https://u:p@ex.test"), + Err(Refusal::Userinfo) + )); + } + + #[test] + fn a_refusal_never_echoes_the_address() { + let e = validate_login_url("https://user:secret-pass@ex.test/").unwrap_err(); + assert!(!e.to_string().contains("secret-pass")); + } + + #[test] + fn file_name_shape() { + assert_eq!( + default_file_name( + "ex.test", + SystemTime::UNIX_EPOCH + Duration::from_secs(1_794_974_640) + ), + "ex.test-20261118T040400Z.kharon-recording.json" + ); + } + + #[test] + fn file_name_follows_the_recordings_end_time() { + assert_eq!( + default_file_name_at("Ex.test", "2026-11-18T04:04:00.123Z").unwrap(), + "ex.test-20261118T040400Z.kharon-recording.json" + ); + assert!(default_file_name_at("ex.test", "yesterday").is_none()); + } + + #[test] + fn file_name_sanitizes_the_host() { + let n = default_file_name("[::1]", SystemTime::UNIX_EPOCH); + assert!(n.starts_with("___1_-19700101T000000Z"), "{n}"); + assert!(!n.contains(['[', ']', ':', '/'])); + } + + fn private_dir(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!( + "kharon-cli-test-{tag}-{}-{:08x}", + std::process::id(), + rand::random::() + )); + std::fs::create_dir(&d).unwrap(); + d + } + + #[test] + fn refuses_an_existing_out_file() { + let d = private_dir("exists"); + let f = d.join("a.json"); + assert!(check_out_path(&f).is_ok()); + std::fs::write(&f, b"x").unwrap(); + assert!(matches!(check_out_path(&f), Err(Refusal::OutExists(_)))); + std::fs::remove_dir_all(&d).unwrap(); + } + + #[test] + fn refuses_a_missing_or_read_only_directory() { + use std::os::unix::fs::PermissionsExt; + let d = private_dir("dirs"); + assert!(matches!( + check_out_path(&d.join("nope").join("a.json")), + Err(Refusal::OutDir(_)) + )); + let ro = d.join("ro"); + std::fs::create_dir(&ro).unwrap(); + std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o500)).unwrap(); + assert!(matches!( + check_out_path(&ro.join("a.json")), + Err(Refusal::OutDir(_)) + )); + // A file is not a directory. + let file = d.join("f"); + std::fs::write(&file, b"x").unwrap(); + assert!(matches!( + check_out_path(&file.join("a.json")), + Err(Refusal::OutDir(_)) + )); + std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::remove_dir_all(&d).unwrap(); + } + + pub(crate) fn recording() -> Recording { + Recording { + format_version: 1, + tool_version: "t".into(), + browser: BrowserInfo { + product: "Chrome/1".into(), + user_agent: "UA".into(), + viewport: Viewport { + width: 1, + height: 1, + }, + }, + listener_world: ListenerWorld::Isolated, + started_at: "2026-11-18T00:00:00Z".into(), + ended_at: "2026-11-18T00:01:00Z".into(), + ended_by: EndedBy::Confirmed, + success_confirmed: true, + start_url: StrippedUrl { + origin: "https://ex.test".into(), + path: "/login".into(), + query_keys: vec![], + }, + final_url: None, + steps: vec![], + requests: vec![], + flags: Flags::default(), + classification: Classification { + shape: Shape::FormPost, + rung_ceiling: RungCeiling::Milliseconds, + network_swap_candidate: false, + notes: vec![], + }, + stripped: StrippedCounts { + requests_seen: 9, + requests_stored: 4, + request_bodies_reduced: 2, + query_values_removed: 3, + path_segments_replaced: 1, + typed_values_removed: 5, + steps_dropped: 0, + }, + } + } + + #[test] + fn summary_names_shape_ceiling_flags_and_counts() { + let mut r = recording(); + r.flags.captcha = true; + r.classification.network_swap_candidate = true; + r.classification.notes = vec!["a classifier note".into()]; + r.listener_world = ListenerWorld::Main; + let s = summary(&r); + assert!(s.contains("https://ex.test/login")); + assert!( + s.contains("One page; the form sends the password to the site as typed or encoded.") + ); + assert!(s.contains("password is in the page for milliseconds")); + assert!(s.contains("A captcha")); + assert!(s.contains("Network swap: this login looks like")); + assert!(s.contains("Note: a classifier note")); + assert!(s.contains("Caution")); + assert!(s.contains("5 typed value(s)")); + assert!(s.contains("4 of 9 request(s) kept")); + assert!(!s.contains('{') && !s.contains('"')); + let quiet = summary(&recording()); + assert!(!quiet.contains("Caution") && !quiet.contains("Flag:")); + assert!(!quiet.contains("Not recorded")); + let mut capped = recording(); + capped.stripped.steps_dropped = 7; + assert!(summary(&capped).contains("Not recorded: 7 step(s) past the recorder's limit.")); + assert!(quiet.contains("does not look like a candidate")); + } + + #[test] + fn show_never_prints_a_control_character_from_the_file() { + let mut r = recording(); + r.start_url.path = "/login\u{1b}[31mred".into(); + r.start_url.origin = "https://ex.test\u{7}".into(); + r.classification.notes = + vec!["a note \u{1b}[2J with escapes \u{202E}and bidi\u{2028}".into()]; + r.classification.shape = Shape::Transformed { + digest: Some("sha\u{1b}[0m".into()), + }; + let s = summary(&r); + for c in s.chars().filter(|c| *c != '\n') { + assert!(!c.is_control(), "a control character {:?} was printed", c); + } + for bad in ['\u{202E}', '\u{2028}', '\u{2029}', '\u{2066}', '\u{2069}'] { + assert!(!s.contains(bad), "{bad:?}"); + } + assert!(s.contains("/login\u{FFFD}[31mred")); + assert!(s.contains("Note: a note \u{FFFD}[2J with escapes \u{FFFD}and bidi\u{FFFD}")); + // Ordinary text, including non-ASCII, passes through. + let mut r = recording(); + r.classification.notes = vec!["héllo wörld".into()]; + assert!(summary(&r).contains("Note: héllo wörld")); + } + + #[test] + fn cli_has_no_headless_or_attach_option() { + let names: Vec<_> = Cli::command() + .get_arguments() + .map(|a| a.get_id().to_string()) + .collect(); + for banned in [ + "headless", + "attach", + "port", + "profile", + "user-data-dir", + "endpoint", + ] { + assert!(!names.iter().any(|n| n.contains(banned)), "{banned}"); + } + } +} diff --git a/crates/kharon-teach/src/events.rs b/crates/kharon-teach/src/events.rs new file mode 100644 index 0000000..6fe3d53 --- /dev/null +++ b/crates/kharon-teach/src/events.rs @@ -0,0 +1,653 @@ +//! Typed CDP messages: the event parameters the recorder reads, the parameters +//! it sends, and the report the page script passes through the binding. +//! +//! Anything that can hold page data (what a person typed, a request body, a +//! header value) is held in `Zeroizing` and has no `Debug` impl: a derive would +//! print the content. + +use serde::de::{IgnoredAny, MapAccess, Visitor}; +use serde::{Deserialize, Deserializer, Serialize}; +use std::collections::HashMap; +use std::fmt; +use zeroize::Zeroizing; + +// ---- events the recorder reads ---- + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AttachedToTarget { + pub session_id: String, + pub target_info: TargetInfo, + pub waiting_for_debugger: bool, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TargetInfo { + pub target_id: String, + #[serde(rename = "type")] + pub kind: String, + pub url: String, + #[serde(default)] + pub opener_id: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TargetDestroyed { + pub target_id: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetachedFromTarget { + pub session_id: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FrameNavigated { + pub frame: Frame, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Frame { + pub id: String, + #[serde(default)] + pub parent_id: Option, + pub url: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BindingCalled { + pub name: String, + pub payload: Zeroizing, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RequestWillBeSent { + pub request_id: String, + pub loader_id: String, + #[serde(default)] + pub frame_id: Option, + #[serde(rename = "documentURL")] + pub document_url: String, + pub request: Request, + #[serde(rename = "type", default)] + pub resource: Option, + /// Set when this request is the next hop of a redirect: the previous hop's response. + #[serde(default)] + pub redirect_response: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Request { + pub url: String, + pub method: String, + #[serde(default)] + pub headers: Headers, + #[serde(default)] + pub post_data: Option>, + #[serde(default)] + pub has_post_data: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResponseReceived { + pub request_id: String, + pub response: Response, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Response { + #[serde(deserialize_with = "lenient_status")] + pub status: u16, + #[serde(default)] + pub headers: Headers, +} + +/// A status outside `u16` (or fractional) must not fail the whole event. +fn lenient_status<'de, D: Deserializer<'de>>(d: D) -> Result { + let v = f64::deserialize(d)?; + // `as` saturates and maps NaN to 0. + Ok(v.clamp(0.0, f64::from(u16::MAX)) as u16) +} + +/// The raw response headers, which Chromium reports apart from `Network.responseReceived` because +/// some of them (`Set-Cookie`) are only in this event. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResponseReceivedExtraInfo { + pub request_id: String, + #[serde(default, deserialize_with = "lenient_opt_status")] + pub status_code: Option, + #[serde(default)] + pub headers: Headers, +} + +fn lenient_opt_status<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let v = Option::::deserialize(d)?; + Ok(v.map(|v| v.clamp(0.0, f64::from(u16::MAX)) as u16)) +} + +// ---- results ---- + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VersionInfo { + pub product: String, + pub user_agent: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LayoutMetrics { + pub css_visual_viewport: VisualViewport, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VisualViewport { + pub client_width: f64, + pub client_height: f64, +} + +/// The result of `Network.getRequestPostData`. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PostData { + pub post_data: Zeroizing, +} + +// ---- headers ---- + +/// Header map. Values are `Zeroizing` and there is no `Debug`. +/// +/// ```compile_fail +/// fn needs_debug() {} +/// needs_debug::(); +/// ``` +#[derive(Default)] +pub struct Headers(HashMap>); + +impl Headers { + fn get(&self, name: &str) -> Option<&str> { + self.0 + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case(name)) + .map(|(_, v)| v.as_str()) + } + + pub fn content_type(&self) -> Option<&str> { + self.get("content-type") + } + + pub fn authorization(&self) -> Option<&str> { + self.get("authorization") + } + + pub fn has_set_cookie(&self) -> bool { + self.0.keys().any(|k| k.eq_ignore_ascii_case("set-cookie")) + } +} + +/// A header value that is a string, or anything else (skipped). +struct MaybeString(Option>); + +impl<'de> Deserialize<'de> for MaybeString { + fn deserialize>(d: D) -> Result { + struct V; + impl<'de> Visitor<'de> for V { + type Value = MaybeString; + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("a header value") + } + fn visit_str(self, v: &str) -> Result { + Ok(MaybeString(Some(Zeroizing::new(v.to_owned())))) + } + fn visit_string(self, v: String) -> Result { + Ok(MaybeString(Some(Zeroizing::new(v)))) + } + fn visit_bool(self, _: bool) -> Result { + Ok(MaybeString(None)) + } + fn visit_i64(self, _: i64) -> Result { + Ok(MaybeString(None)) + } + fn visit_u64(self, _: u64) -> Result { + Ok(MaybeString(None)) + } + fn visit_f64(self, _: f64) -> Result { + Ok(MaybeString(None)) + } + fn visit_unit(self) -> Result { + Ok(MaybeString(None)) + } + fn visit_none(self) -> Result { + Ok(MaybeString(None)) + } + fn visit_seq>( + self, + mut seq: A, + ) -> Result { + while seq.next_element::()?.is_some() {} + Ok(MaybeString(None)) + } + fn visit_map>(self, mut map: A) -> Result { + while map.next_entry::()?.is_some() {} + Ok(MaybeString(None)) + } + } + d.deserialize_any(V) + } +} + +impl<'de> Deserialize<'de> for Headers { + fn deserialize>(d: D) -> Result { + struct V; + impl<'de> Visitor<'de> for V { + type Value = Headers; + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("a header object") + } + fn visit_unit(self) -> Result { + Ok(Headers::default()) + } + fn visit_none(self) -> Result { + Ok(Headers::default()) + } + fn visit_map>(self, mut map: A) -> Result { + let mut out = HashMap::new(); + while let Some((k, v)) = map.next_entry::()? { + if let Some(v) = v.0 { + out.insert(k, v); + } + } + Ok(Headers(out)) + } + } + d.deserialize_any(V) + } +} + +// ---- parameters the recorder sends ---- + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SetAutoAttach { + pub auto_attach: bool, + pub wait_for_debugger_on_start: bool, + pub flatten: bool, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AddBinding<'a> { + pub name: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + pub execution_context_name: Option<&'a str>, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AddScriptToEvaluateOnNewDocument<'a> { + pub source: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + pub world_name: Option<&'a str>, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GetRequestPostData<'a> { + pub request_id: &'a str, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SetDiscoverTargets { + pub discover: bool, +} + +#[derive(Serialize)] +pub struct Navigate<'a> { + pub url: &'a str, +} + +#[derive(Serialize)] +pub struct RemoveScript<'a> { + pub identifier: &'a str, +} + +/// The result of `Page.addScriptToEvaluateOnNewDocument`. +#[derive(Deserialize)] +pub struct ScriptId { + pub identifier: String, +} + +/// Parameters for a call that takes none: serializes as `{}`. +#[derive(Serialize)] +pub struct Empty {} + +// ---- the page script's report ---- + +/// What the listener sends once when it starts in a top-level document. +#[derive(Deserialize)] +pub struct Hello { + pub hello: bool, + /// The nonce this install of the script was given: a hello from an earlier install is not + /// the current one's. + pub nonce: String, +} + +#[derive(Serialize)] +pub struct RemoveBinding<'a> { + pub name: &'a str, +} + +const MAX_FIELD_CHARS: usize = 4096; +const MAX_SELECTORS: usize = 16; + +#[derive(Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReportKind { + Input, + Change, + Click, + Submit, + EnterKey, +} + +/// What the page script reported. No `Debug`: `value` is what a person typed. +/// +/// ```compile_fail +/// fn needs_debug() {} +/// needs_debug::(); +/// ``` +#[derive(Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct ListenerReport { + pub kind: ReportKind, + pub document_id: String, + pub frame_url: String, + pub is_top: bool, + pub element: ReportElement, + #[serde(default)] + pub value: Option>, +} + +#[derive(Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct ReportElement { + pub element_id: u32, + pub tag: String, + #[serde(default)] + pub input_type: Option, + #[serde(default)] + pub id: Option, + #[serde(default)] + pub name: Option, + #[serde(default)] + pub autocomplete: Option, + #[serde(default)] + pub placeholder: Option, + #[serde(default)] + pub aria_label: Option, + #[serde(default)] + pub is_password_now: bool, + #[serde(default)] + pub form: Option, + #[serde(default)] + pub selectors: Vec, + #[serde(default)] + pub in_shadow_root: bool, +} + +#[derive(Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct ReportForm { + pub index: u32, + #[serde(default)] + pub action: Option, + pub method: String, +} + +fn short(s: &str) -> bool { + s.chars().take(MAX_FIELD_CHARS + 1).count() <= MAX_FIELD_CHARS +} + +fn short_opt(s: &Option) -> bool { + s.as_deref().is_none_or(short) +} + +impl ListenerReport { + /// False when a hostile page sent more than the recorder will hold: any + /// string over 4,096 characters, or more than 16 selectors. + pub fn within_limits(&self) -> bool { + let e = &self.element; + short(&self.document_id) + && short(&self.frame_url) + && self.value.as_ref().is_none_or(|v| short(v)) + && short(&e.tag) + && short_opt(&e.input_type) + && short_opt(&e.id) + && short_opt(&e.name) + && short_opt(&e.autocomplete) + && short_opt(&e.placeholder) + && short_opt(&e.aria_label) + && e.selectors.len() <= MAX_SELECTORS + && e.selectors.iter().all(|s| short(s)) + && e.form + .as_ref() + .is_none_or(|f| short_opt(&f.action) && short(&f.method)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cdp::parse_quiet; + + const REPORT: &str = r##"{ + "kind": "input", "document_id": "d1", "frame_url": "https://a.test/login", + "is_top": true, "value": "Tr0ub4dor-9x", + "element": { + "element_id": 3, "tag": "input", "input_type": "password", "id": "pw", + "name": "password", "autocomplete": "current-password", + "is_password_now": true, + "form": {"index": 0, "action": "/session", "method": "post"}, + "selectors": ["#pw", "input[name=password]"], "in_shadow_root": false + } + }"##; + + #[test] + fn listener_report_parses_and_value_is_zeroizing() { + let r: ListenerReport = parse_quiet(REPORT).unwrap(); + assert!(matches!(r.kind, ReportKind::Input)); + let v: &Zeroizing = r.value.as_ref().unwrap(); + assert_eq!(v.as_str(), "Tr0ub4dor-9x"); + assert_eq!(r.element.element_id, 3); + assert_eq!(r.element.input_type.as_deref(), Some("password")); + assert!(r.element.is_password_now); + let f = r.element.form.as_ref().unwrap(); + assert_eq!( + (f.index, f.action.as_deref(), f.method.as_str()), + (0, Some("/session"), "post") + ); + assert_eq!(r.element.selectors.len(), 2); + assert!(r.within_limits()); + } + + #[test] + fn a_sparse_report_parses() { + let raw = r#"{"kind":"enter_key","document_id":"d","frame_url":"u","is_top":false, + "element":{"element_id":1,"tag":"input"}}"#; + let r: ListenerReport = parse_quiet(raw).unwrap(); + assert!(matches!(r.kind, ReportKind::EnterKey)); + assert!(r.value.is_none() && r.element.selectors.is_empty() && r.element.form.is_none()); + assert!(!r.element.is_password_now && !r.element.in_shadow_root); + } + + #[test] + fn oversized_reports_are_out_of_limits() { + let mut r: ListenerReport = parse_quiet(REPORT).unwrap(); + r.value = Some(Zeroizing::new("x".repeat(4097))); + assert!(!r.within_limits()); + r.value = Some(Zeroizing::new("x".repeat(4096))); + assert!(r.within_limits()); + + let mut r: ListenerReport = parse_quiet(REPORT).unwrap(); + r.element.selectors = vec!["a".to_owned(); 17]; + assert!(!r.within_limits()); + r.element.selectors = vec!["a".to_owned(); 16]; + assert!(r.within_limits()); + + let mut r: ListenerReport = parse_quiet(REPORT).unwrap(); + r.element.name = Some("n".repeat(5000)); + assert!(!r.within_limits()); + + let mut r: ListenerReport = parse_quiet(REPORT).unwrap(); + r.frame_url = "u".repeat(4097); + assert!(!r.within_limits()); + + let mut r: ListenerReport = parse_quiet(REPORT).unwrap(); + r.element.form.as_mut().unwrap().action = Some("a".repeat(4097)); + assert!(!r.within_limits()); + } + + #[test] + fn headers_are_read_case_insensitively() { + let h: Headers = parse_quiet( + r#"{"Content-Type":"application/json","AUTHORIZATION":"Bearer x","Set-Cookie":"a=b"}"#, + ) + .unwrap(); + assert_eq!(h.content_type(), Some("application/json")); + assert_eq!(h.authorization(), Some("Bearer x")); + assert!(h.has_set_cookie()); + + let h: Headers = parse_quiet(r#"{"content-type":"text/html"}"#).unwrap(); + assert_eq!(h.content_type(), Some("text/html")); + assert_eq!(h.authorization(), None); + assert!(!h.has_set_cookie()); + } + + #[test] + fn null_headers_are_empty_not_fatal() { + let r: ResponseReceived = + parse_quiet(r#"{"requestId":"1","response":{"status":200,"headers":null}}"#).unwrap(); + assert!(r.response.headers.content_type().is_none()); + assert!(!r.response.headers.has_set_cookie()); + } + + #[test] + fn a_non_string_header_is_skipped_not_fatal() { + let h: Headers = + parse_quiet(r#"{"x-n":5,"x-a":["a"],"x-o":{"k":1},"x-z":null,"content-type":"t"}"#) + .unwrap(); + assert_eq!(h.content_type(), Some("t")); + } + + #[test] + fn a_wild_status_does_not_fail_the_event() { + let r: ResponseReceived = + parse_quiet(r#"{"requestId":"1","response":{"status":99999,"headers":{}}}"#).unwrap(); + assert_eq!(r.response.status, u16::MAX); + let r: ResponseReceived = + parse_quiet(r#"{"requestId":"1","response":{"status":-3,"headers":{}}}"#).unwrap(); + assert_eq!(r.response.status, 0); + let r: ResponseReceived = parse_quiet( + r#"{"requestId":"1","response":{"status":302,"headers":{"Set-Cookie":"s"}}}"#, + ) + .unwrap(); + assert_eq!(r.response.status, 302); + assert!(r.response.headers.has_set_cookie()); + } + + #[test] + fn a_request_event_parses_with_unknown_fields_ignored() { + let raw = r#"{"requestId":"r","loaderId":"l","frameId":"f","documentURL":"https://a.test/", + "request":{"url":"https://a.test/s","method":"POST","headers":{"Content-Type":"x"}, + "postData":"user=a&pass=b","hasPostData":true,"initialPriority":"High"}, + "type":"Document","timestamp":1.5}"#; + let e: RequestWillBeSent = parse_quiet(raw).unwrap(); + assert_eq!(e.request.method, "POST"); + assert_eq!( + e.request.post_data.as_ref().map(|p| p.as_str()), + Some("user=a&pass=b") + ); + assert_eq!(e.resource.as_deref(), Some("Document")); + assert_eq!(e.request.headers.content_type(), Some("x")); + } + + #[test] + fn sent_params_serialize_in_camel_case_and_skip_none() { + let s = serde_json::to_string(&AddBinding { + name: "b", + execution_context_name: None, + }) + .unwrap(); + assert_eq!(s, r#"{"name":"b"}"#); + let s = serde_json::to_string(&SetAutoAttach { + auto_attach: true, + wait_for_debugger_on_start: true, + flatten: true, + }) + .unwrap(); + assert_eq!( + s, + r#"{"autoAttach":true,"waitForDebuggerOnStart":true,"flatten":true}"# + ); + assert_eq!(serde_json::to_string(&Empty {}).unwrap(), "{}"); + let s = serde_json::to_string(&AddScriptToEvaluateOnNewDocument { + source: "1", + world_name: Some("w"), + }) + .unwrap(); + assert_eq!(s, r#"{"source":"1","worldName":"w"}"#); + } + + #[test] + fn a_redirect_event_carries_the_previous_hops_response() { + let raw = r#"{"requestId":"r","loaderId":"r","documentURL":"https://a.test/", + "request":{"url":"https://a.test/home","method":"GET"}, + "redirectResponse":{"status":302,"headers":{"Location":"/home"}},"type":"Document"}"#; + let e: RequestWillBeSent = parse_quiet(raw).unwrap(); + assert_eq!(e.redirect_response.map(|r| r.status), Some(302)); + let raw = r#"{"requestId":"r","loaderId":"r","documentURL":"https://a.test/", + "request":{"url":"https://a.test/","method":"GET"}}"#; + assert!( + parse_quiet::(raw) + .unwrap() + .redirect_response + .is_none() + ); + } + + #[test] + fn raw_response_headers_carry_set_cookie() { + let e: ResponseReceivedExtraInfo = parse_quiet( + r#"{"requestId":"r","statusCode":302,"headers":{"set-cookie":"sid=x"},"blockedCookies":[]}"#, + ) + .unwrap(); + assert_eq!(e.status_code, Some(302)); + assert!(e.headers.has_set_cookie()); + let e: ResponseReceivedExtraInfo = parse_quiet(r#"{"requestId":"r"}"#).unwrap(); + assert!(e.status_code.is_none() && !e.headers.has_set_cookie()); + } + + #[test] + fn hello_is_not_a_report() { + assert!( + parse_quiet::(r#"{"hello":true,"nonce":"n1"}"#) + .is_ok_and(|h| h.hello && h.nonce == "n1") + ); + assert!(parse_quiet::(r#"{"hello":true}"#).is_err()); + assert!(parse_quiet::(REPORT).is_err()); + assert!(parse_quiet::(r#"{"hello":true}"#).is_err()); + } +} diff --git a/crates/kharon-teach/src/lib.rs b/crates/kharon-teach/src/lib.rs new file mode 100644 index 0000000..a74cbdb --- /dev/null +++ b/crates/kharon-teach/src/lib.rs @@ -0,0 +1,616 @@ +//! Teach-mode recorder for kharon (#361). + +pub mod body; +pub mod browser; +pub mod capture; +pub mod cdp; +pub mod cli; +pub mod events; +pub mod marker; +pub mod notice; +pub mod session; +pub mod site; +pub mod strip; +#[doc(hidden)] +pub mod testing; + +use cli::{ + Cli, Command, Refusal, check_dir_writable, check_out_path, default_file_name, + default_file_name_at, summary, validate_login_url, +}; +use kharon_recording::Recording; +use session::{End, Outcome, Ready, RecordOptions, record}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use tokio::io::{AsyncBufRead, AsyncBufReadExt}; +use tokio::sync::oneshot; + +/// How a run ended; the process exit code is its value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Exit { + /// A file was written, or the recording was discarded, or nothing was asked for. + Done = 0, + /// Anything else. + Failed = 1, + /// A refusal: nothing was started. + Refused = 2, +} + +impl From for std::process::ExitCode { + fn from(e: Exit) -> Self { + std::process::ExitCode::from(e as u8) + } +} + +/// What tests may change. The shipped program never does: there is no flag and no environment +/// variable for it. +#[doc(hidden)] +pub struct Hooks { + pub headless: bool, + /// Receives a copy of the session's `Ready` once the notice page is showing. + pub ready: Option>, +} + +/// The whole program. Prompts and results go to `stdout`; refusals and errors to `stderr`, one +/// line each, starting `kharon-teach: `. +pub async fn run( + cli: Cli, + stdin: impl AsyncBufRead + Unpin, + stdout: impl Write, + stderr: impl Write, +) -> Exit { + run_with( + cli, + stdin, + stdout, + stderr, + Hooks { + headless: false, + ready: None, + }, + ) + .await +} + +fn fail(stderr: &mut impl Write, what: &dyn std::fmt::Display, exit: Exit) -> Exit { + let _ = writeln!(stderr, "kharon-teach: {what}"); + exit +} + +/// Where the recording goes. +enum Target { + /// `--out`: exactly this path. + Explicit(PathBuf), + /// No `--out`: a generated name in the current directory. + Here, +} + +#[doc(hidden)] +#[derive(Clone, Copy)] +pub enum Naming { + Default, + Explicit, +} + +/// Creates `first` (mode 0600, never replacing a file). If it exists, the same name with `.1` +/// to `.9` added (before the `.kharon-recording.json` ending for a generated name, at the end +/// for an `--out` name) is tried, so a demonstration is not lost to a name clash. +#[doc(hidden)] +pub fn write_new( + first: &Path, + naming: Naming, + text: &str, +) -> Result { + use std::os::unix::fs::OpenOptionsExt; + let name = first + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(); + for n in 0..10 { + let path = if n == 0 { + first.to_owned() + } else { + let numbered = match (naming, name.strip_suffix(cli::FILE_SUFFIX)) { + (Naming::Default, Some(stem)) => format!("{stem}.{n}{}", cli::FILE_SUFFIX), + _ => format!("{name}.{n}"), + }; + first.with_file_name(numbered) + }; + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + { + Ok(mut f) => { + return match f.write_all(text.as_bytes()).and_then(|()| f.flush()) { + Ok(()) => Ok(path), + Err(e) => { + // Never leave a truncated file where the next run would refuse to write. + drop(f); + let _ = std::fs::remove_file(&path); + Err((path, e)) + } + }; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err((path, e)), + } + } + Err(( + first.to_owned(), + std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "ten files of that name already exist", + ), + )) +} + +/// `kharon-teach show `. +fn show(file: &Path, stdout: &mut impl Write, stderr: &mut impl Write) -> Exit { + use std::io::Read; + const CAP: u64 = 16 * 1024 * 1024; + let read = || -> Result { + let unreadable = |e: std::io::Error| Refusal::Unreadable(file.to_owned(), e.to_string()); + let f = std::fs::File::open(file).map_err(unreadable)?; + let mut bytes = Vec::new(); + f.take(CAP + 1) + .read_to_end(&mut bytes) + .map_err(unreadable)?; + if bytes.len() as u64 > CAP { + return Err(Refusal::TooLarge(file.to_owned())); + } + let text = String::from_utf8(bytes) + .map_err(|_| Refusal::NotARecording(file.to_owned(), "not text".to_owned()))?; + Recording::from_json(&text) + .map_err(|e| Refusal::NotARecording(file.to_owned(), e.to_string())) + }; + match read() { + Ok(r) => { + let _ = stdout.write_all(summary(&r).as_bytes()); + Exit::Done + } + Err(e) => fail(stderr, &e, Exit::Refused), + } +} + +/// Everything checked before the notice. +struct Prepared { + url: url::Url, + browser: PathBuf, + target: Target, +} + +fn prepare(cli: &Cli) -> Result { + let url = validate_login_url(cli.login_url.as_deref().ok_or(Refusal::NoUrl)?)?; + let browser = browser::find_browser(cli.browser.as_deref()) + .map_err(|e| Refusal::NoBrowser(e.to_string()))?; + let target = match &cli.out { + Some(p) => { + check_out_path(p)?; + Target::Explicit(p.clone()) + } + None => { + check_dir_writable(Path::new("."))?; + Target::Here + } + }; + Ok(Prepared { + url, + browser, + target, + }) +} + +/// Linux: no core dump and no ptrace by another process, as `kharon` does (REQ-MEM-001). +#[cfg(target_os = "linux")] +fn harden_process() -> Result<(), String> { + use rustix::process::{DumpableBehavior, Resource, Rlimit, set_dumpable_behavior, setrlimit}; + set_dumpable_behavior(DumpableBehavior::NotDumpable).map_err(|e| e.to_string())?; + setrlimit( + Resource::Core, + Rlimit { + current: Some(0), + maximum: Some(0), + }, + ) + .map_err(|e| e.to_string()) +} + +#[cfg(not(target_os = "linux"))] +fn harden_process() -> Result<(), String> { + Ok(()) +} + +/// One line from the person, with the reader handed back so the caller can ask again. +async fn read_answer(mut stdin: R) -> (String, std::io::Result, R) { + let mut line = String::new(); + let read = stdin.read_line(&mut line).await; + (line, read, stdin) +} + +/// The answer a line at the end prompt gives: an empty line confirms, `q` or `quit` (any case) +/// discards, anything else is no answer and the prompt is repeated. +fn end_choice(line: &str) -> Option { + let answer = line.trim(); + if answer.is_empty() { + Some(End::Confirmed) + } else if answer.eq_ignore_ascii_case("q") || answer.eq_ignore_ascii_case("quit") { + Some(End::Discard) + } else { + None + } +} + +enum Ended { + Finished(Result), + Interrupted, +} + +#[doc(hidden)] +pub async fn run_with( + cli: Cli, + mut stdin: impl AsyncBufRead + Unpin, + mut stdout: impl Write, + mut stderr: impl Write, + hooks: Hooks, +) -> Exit { + use tokio::signal::unix::{SignalKind, signal}; + if let Some(Command::Show { file }) = &cli.command { + return show(file, &mut stdout, &mut stderr); + } + let prep = match prepare(&cli) { + Ok(p) => p, + Err(r) => return fail(&mut stderr, &r, Exit::Refused), + }; + let _ = stdout.write_all(notice::NOTICE.as_bytes()); + if !cli.yes { + let _ = stdout.write_all(notice::CONSENT_PROMPT.as_bytes()); + let _ = stdout.flush(); + let mut line = String::new(); + let n = stdin.read_line(&mut line).await.unwrap_or(0); + if n == 0 || line.trim() != "yes" { + let _ = writeln!(stdout, "{}", notice::NOTHING_RECORDED); + return Exit::Done; + } + } + if let Err(e) = harden_process() { + return fail( + &mut stderr, + &format_args!("cannot make the process non-dumpable: {e}"), + Exit::Failed, + ); + } + // Ctrl-C, a stop request, the terminal going away, and Ctrl-\: every way the program can be + // told to stop. The browser is in its own process group, so none of them reaches it by + // itself. + let (Ok(mut sigint), Ok(mut sigterm), Ok(mut sighup), Ok(mut sigquit)) = ( + signal(SignalKind::interrupt()), + signal(SignalKind::terminate()), + signal(SignalKind::hangup()), + signal(SignalKind::quit()), + ) else { + return fail(&mut stderr, &"cannot watch for Ctrl-C", Exit::Failed); + }; + + let (ready_tx, mut ready_rx) = oneshot::channel(); + let (end_tx, end_rx) = oneshot::channel(); + let mut end_tx = Some(end_tx); + let mut rec = Box::pin(record( + RecordOptions { + login_url: &prep.url, + browser: &prep.browser, + headless: hooks.headless, + tool_version: env!("CARGO_PKG_VERSION"), + }, + ready_tx, + end_rx, + )); + let mut forward = hooks.ready; + let (mut got_ready, mut stdin_done) = (false, false); + // The reader goes into the future and comes back with the line, so another line can be + // asked for without a second read ever being dropped half way (a read is not cancel safe). + let stdin_line = read_answer(stdin); + tokio::pin!(stdin_line); + let ended = loop { + tokio::select! { + r = &mut rec => break Ended::Finished(r), + r = &mut ready_rx, if !got_ready => { + got_ready = true; + if let Ok(ready) = r { + if ready.stale_profiles_removed > 0 { + let _ = writeln!( + stdout, + "Removed {} leftover browser profile(s) from an earlier run.", + ready.stale_profiles_removed + ); + } + let _ = writeln!(stdout, "{}", notice::END_PROMPT); + let _ = stdout.flush(); + if let Some(tx) = forward.take() { + let _ = tx.send(Ready { + devtools_port: ready.devtools_port, + profile_dir: ready.profile_dir, + stale_profiles_removed: ready.stale_profiles_removed, + }); + } + } + } + (line, read, back) = &mut stdin_line, if got_ready && !stdin_done => { + // End of input is neither answer: keep recording until the window closes. + if matches!(read, Ok(n) if n > 0) { + match end_choice(&line) { + Some(how) => { + stdin_done = true; + if let Some(tx) = end_tx.take() { + let _ = tx.send(how); + } + } + None => { + let _ = writeln!(stdout, "{}", notice::END_RETRY); + let _ = stdout.flush(); + stdin_line.set(read_answer(back)); + } + } + } else { + stdin_done = true; + } + } + _ = sigint.recv() => break Ended::Interrupted, + _ = sigterm.recv() => break Ended::Interrupted, + _ = sighup.recv() => break Ended::Interrupted, + _ = sigquit.recv() => break Ended::Interrupted, + } + }; + let outcome = match ended { + Ended::Finished(r) => r, + Ended::Interrupted => { + // Dropping the session closes the browser and removes its profile. + drop(rec); + return fail( + &mut stderr, + &"Interrupted. Nothing was saved.", + Exit::Failed, + ); + } + }; + drop(rec); + match outcome { + Ok(Outcome::Recorded(r)) => { + let first = + match &prep.target { + Target::Explicit(p) => p.clone(), + Target::Here => { + let host = prep.url.host_str().unwrap_or("login"); + // The name carries the instant the recording ended, not the time of the write. + PathBuf::from(default_file_name_at(host, &r.ended_at).unwrap_or_else( + || default_file_name(host, std::time::SystemTime::now()), + )) + } + }; + let naming = match prep.target { + Target::Explicit(_) => Naming::Explicit, + Target::Here => Naming::Default, + }; + match write_new(&first, naming, &r.to_json()) { + Ok(path) => { + let _ = writeln!(stdout, "\n{}", summary(&r)); + let _ = writeln!(stdout, "Saved: {}", path.display()); + Exit::Done + } + Err((at, e)) => fail( + &mut stderr, + &format_args!("cannot write {}: {e}", at.display()), + Exit::Failed, + ), + } + } + Ok(Outcome::Discarded) => { + let _ = writeln!(stdout, "Discarded. Nothing was saved."); + Exit::Done + } + Ok(Outcome::NeverStarted) => { + let _ = writeln!(stdout, "{}", notice::NEVER_STARTED); + Exit::Done + } + Err(e) => fail(&mut stderr, &e, Exit::Failed), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::Parser; + + fn cli(args: &[&str]) -> Cli { + Cli::try_parse_from(std::iter::once("kharon-teach").chain(args.iter().copied())).unwrap() + } + + /// Profile directories in the temp directory that name THIS process as their owner. Other + /// tests and other processes make profiles too; only ours can come from `run_with` here. + fn my_profiles() -> Vec { + let me = std::process::id().to_string(); + std::fs::read_dir(std::env::temp_dir()) + .unwrap() + .flatten() + .filter(|e| e.file_name().to_string_lossy().starts_with("kharon-teach-")) + .map(|e| e.path()) + .filter(|p| { + std::fs::read_to_string(p.join(browser::OWNER_FILE)).is_ok_and(|t| t.trim() == me) + }) + .collect() + } + + struct Ran { + exit: Exit, + out: String, + err: String, + } + + async fn go(args: &[&str], stdin: &str) -> Ran { + let (mut out, mut err) = (Vec::new(), Vec::new()); + let exit = run(cli(args), stdin.as_bytes(), &mut out, &mut err).await; + Ran { + exit, + out: String::from_utf8(out).unwrap(), + err: String::from_utf8(err).unwrap(), + } + } + + fn private_dir(tag: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!( + "kharon-run-test-{tag}-{}-{:08x}", + std::process::id(), + rand::random::() + )); + std::fs::create_dir(&d).unwrap(); + d + } + + /// An executable that exists, so browser discovery passes. It is never started in these tests. + fn fake_browser(dir: &std::path::Path) -> std::path::PathBuf { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fake-browser"); + std::fs::write(&p, "#!/bin/sh\nexit 1\n").unwrap(); + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap(); + p + } + + #[tokio::test] + async fn refusals_exit_2_before_the_notice() { + let before = my_profiles(); + let d = private_dir("refuse"); + let browser = fake_browser(&d); + let existing = d.join("taken.json"); + std::fs::write(&existing, b"x").unwrap(); + let b = browser.to_str().unwrap(); + let missing = d.join("no-such-browser"); + let cases: Vec> = vec![ + vec!["http://ex.test/", "--browser", b], + vec!["https://u:p@ex.test/", "--browser", b], + vec!["https://ex.test/", "--browser", missing.to_str().unwrap()], + vec![ + "https://ex.test/", + "--browser", + b, + "--out", + existing.to_str().unwrap(), + ], + vec![ + "https://ex.test/", + "--browser", + b, + "--out", + "/no/such/dir/a.json", + ], + vec![], + ]; + for args in cases { + let r = go(&args, "yes\n").await; + assert_eq!(r.exit, Exit::Refused, "{args:?}"); + assert!(!r.out.contains("THROWAWAY"), "{args:?}"); + assert!(r.out.is_empty(), "{args:?}: {}", r.out); + assert!(r.err.starts_with("kharon-teach: "), "{}", r.err); + assert_eq!(r.err.lines().count(), 1, "{}", r.err); + } + let after = my_profiles(); + assert!( + after.iter().all(|p| before.contains(p)), + "a refusal created a profile directory" + ); + std::fs::remove_dir_all(&d).unwrap(); + } + + #[tokio::test] + async fn anything_but_yes_records_nothing() { + let d = private_dir("consent"); + let browser = fake_browser(&d); + let b = browser.to_str().unwrap(); + for input in ["no\n", "y\n", "YES\n", "\n", ""] { + let r = go(&["https://ex.test/", "--browser", b], input).await; + assert_eq!(r.exit, Exit::Done, "{input:?}"); + assert!(r.out.contains("THROWAWAY")); + assert!(r.out.contains("Nothing was recorded."), "{input:?}"); + assert!(r.err.is_empty()); + } + std::fs::remove_dir_all(&d).unwrap(); + } + + #[tokio::test] + async fn show_prints_the_summary_and_refuses_a_bad_file() { + let d = private_dir("show"); + let bad = d.join("bad.json"); + std::fs::write(&bad, b"{}").unwrap(); + let r = go(&["show", bad.to_str().unwrap()], "").await; + assert_eq!(r.exit, Exit::Refused); + assert!(r.out.is_empty()); + assert!(r.err.starts_with("kharon-teach: ") && r.err.lines().count() == 1); + let r = go(&["show", d.join("missing").to_str().unwrap()], "").await; + assert_eq!(r.exit, Exit::Refused); + let big = d.join("big.json"); + std::fs::write(&big, vec![b' '; 16 * 1024 * 1024 + 1]).unwrap(); + let r = go(&["show", big.to_str().unwrap()], "").await; + assert_eq!(r.exit, Exit::Refused); + assert!(r.err.contains("16 MiB")); + std::fs::remove_dir_all(&d).unwrap(); + } + + #[tokio::test] + async fn show_prints_the_summary_of_a_good_file() { + let d = private_dir("show-ok"); + let good = d.join("good.json"); + std::fs::write(&good, cli::tests::recording().to_json()).unwrap(); + let r = go(&["show", good.to_str().unwrap()], "").await; + assert_eq!(r.exit, Exit::Done, "{}", r.err); + assert!(r.out.contains("Site: https://ex.test/login")); + std::fs::remove_dir_all(&d).unwrap(); + } + + #[test] + fn the_end_prompt_reads_enter_q_and_quit_and_asks_again_otherwise() { + assert!(matches!(end_choice("\n"), Some(End::Confirmed))); + assert!(matches!(end_choice(" \r\n"), Some(End::Confirmed))); + for discard in ["q\n", "Q", " quit ", "QUIT\n", "Quit"] { + assert!( + matches!(end_choice(discard), Some(End::Discard)), + "{discard:?}" + ); + } + for again in ["no\n", "n", "discard", "qq", "y", "quit now", "exit"] { + assert!(end_choice(again).is_none(), "{again:?}"); + } + } + + #[test] + fn the_never_started_message_says_what_happened() { + assert_eq!( + notice::NEVER_STARTED, + "Recording never started: the login page was not opened in the first tab. Nothing was saved." + ); + } + + #[test] + fn the_write_picks_a_numbered_name_when_the_file_appears() { + let d = private_dir("numbered"); + let first = d.join(default_file_name("ex.test", std::time::UNIX_EPOCH)); + let a = write_new(&first, Naming::Default, "a").unwrap(); + let b = write_new(&first, Naming::Default, "b").unwrap(); + assert_eq!(a, first); + assert!( + b.to_string_lossy().ends_with(".1.kharon-recording.json"), + "{b:?}" + ); + assert_eq!(std::fs::read_to_string(&a).unwrap(), "a"); + let ex = d.join("mine.json"); + write_new(&ex, Naming::Explicit, "x").unwrap(); + let y = write_new(&ex, Naming::Explicit, "y").unwrap(); + assert_eq!(y, d.join("mine.json.1")); + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(&y).unwrap().permissions().mode() & 0o777, + 0o600 + ); + std::fs::remove_dir_all(&d).unwrap(); + } +} diff --git a/crates/kharon-teach/src/listener.js b/crates/kharon-teach/src/listener.js new file mode 100644 index 0000000..7a7bedb --- /dev/null +++ b/crates/kharon-teach/src/listener.js @@ -0,0 +1,274 @@ +// kharon-teach page listener. Runs at document start in every frame. +// +// It reports what a person did (which element, what kind of event, and the +// text of a text-entry control) through the one binding the recorder added. +// It keeps no state except an element counter, and leaves no global. +(() => { + "use strict"; + try { + const BINDING = "__kharonTeach"; + // Replaced by the recorder with a fresh value for each install of this script. + const NONCE = "__NONCE__"; + const GUARD = Symbol.for("kharon-teach.listener"); + if (Object.prototype.hasOwnProperty.call(document, GUARD)) return; + Object.defineProperty(document, GUARD, { value: true, enumerable: false }); + + const MAX_STR = 2048; + const MAX_VALUE = 4096; + const MAX_SELECTORS = 5; + const TEXTISH = ["text", "email", "password", "tel", "number", "search", "url"]; + + const cut = (s) => (s === null || s === undefined ? null : String(s).slice(0, MAX_STR)); + const ids = new WeakMap(); + let counter = 0; + let documentId = ""; + try { + const bytes = new Uint8Array(8); + crypto.getRandomValues(bytes); + documentId = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join(""); + } catch (_) { + documentId = String(Math.random()).slice(2) + String(Date.now()); + } + + // Reads go through the prototypes, never through the element: a form's child named + // "action", "method", "id", "tagName" or "getAttribute" replaces that property on the form. + const proto = Element.prototype; + const getAttr = proto.getAttribute; + const matchesFn = proto.matches; + const closestFn = proto.closest; + const getRoot = Node.prototype.getRootNode; + const getter = (obj, name) => { + const d = Object.getOwnPropertyDescriptor(obj, name); + return d && d.get ? d.get : () => undefined; + }; + const tagGetter = getter(proto, "tagName"); + const formActionGetter = getter(HTMLFormElement.prototype, "action"); + const formMethodGetter = getter(HTMLFormElement.prototype, "method"); + const docFormsGetter = getter(Document.prototype, "forms"); + const attr = (el, name) => getAttr.call(el, name); + const tagOf = (el) => String(tagGetter.call(el)).toLowerCase(); + + const send = (report) => { + try { + const bind = window[BINDING]; + if (typeof bind === "function") bind(JSON.stringify(report)); + } catch (_) { + /* never throw into the page */ + } + }; + + const isTop = () => { + try { + return window.top === window; + } catch (_) { + return false; + } + }; + + const elementId = (el) => { + let n = ids.get(el); + if (n === undefined) { + n = ++counter; + ids.set(el, n); + } + return n; + }; + + const attrEscape = (s) => String(s).replace(/\\/g, "\\\\").replace(/"/g, '\\"'); + + const structural = (el, scope) => { + const parts = []; + let node = el; + for (let depth = 0; node && node.nodeType === 1 && depth < 6; depth++) { + let part = tagOf(node); + const parent = node.parentElement; + if (parent) { + const same = Array.from(parent.children).filter((c) => tagOf(c) === part); + if (same.length > 1) part += ":nth-of-type(" + (same.indexOf(node) + 1) + ")"; + } + parts.unshift(part); + const candidate = parts.join(" > "); + try { + if (scope.querySelector(candidate) === el) return candidate; + } catch (_) { + /* try a longer path */ + } + node = parent; + } + return null; + }; + + const textEntry = (el) => { + const tag = tagOf(el); + if (tag === "textarea") return true; + if (tag !== "input") return false; + return TEXTISH.includes(String(el.type || "text").toLowerCase()); + }; + + // What the person has typed into the element now: used to keep it out of the descriptor + // and the selectors, whether or not this report sends it. + const typedValue = (el) => { + try { + return textEntry(el) ? String(el.value === undefined ? "" : el.value) : ""; + } catch (_) { + return ""; + } + }; + + // An attribute that holds the typed text (a page that mirrors the field into its own id or + // name) is not reported and builds no selector. + const clean = (typed, v) => { + if (v === null || v === undefined) return null; + const t = String(v); + return typed.length > 0 && t.includes(typed) ? null : t; + }; + + const selectorsFor = (el, typed) => { + const out = []; + let root = null; + try { + root = getRoot.call(el); + } catch (_) { + root = document; + } + const inShadow = typeof ShadowRoot !== "undefined" && root instanceof ShadowRoot; + const scope = inShadow ? root : document; + const add = (s, unique) => { + if (!s || s.length > MAX_STR || out.includes(s)) return; + try { + if (scope.querySelector(s) !== el) return; + if (unique && scope.querySelectorAll(s).length !== 1) return; + out.push(s); + } catch (_) { + /* an invalid candidate is dropped */ + } + }; + const tag = tagOf(el); + const id = clean(typed, attr(el, "id")); + if (id) add("#" + CSS.escape(id)); + const name = clean(typed, attr(el, "name")); + if (name) add(tag + '[name="' + attrEscape(name) + '"]'); + const auto = clean(typed, attr(el, "autocomplete")); + if (auto) add(tag + '[autocomplete="' + attrEscape(auto) + '"]'); + const type = clean(typed, attr(el, "type")); + if (type) add(tag + '[type="' + attrEscape(type) + '"]', true); + add(structural(el, scope)); + return { selectors: out.slice(0, MAX_SELECTORS), inShadow }; + }; + + const formInfo = (form) => { + if (!form) return null; + let index = 0; + try { + index = Math.max(0, Array.prototype.indexOf.call(docFormsGetter.call(document), form)); + } catch (_) { + index = 0; + } + let action = null; + let method = "get"; + try { + action = cut(formActionGetter.call(form)); + method = String(formMethodGetter.call(form) || "get").toLowerCase(); + } catch (_) { + /* leave the defaults */ + } + return { index, action, method }; + }; + + const describe = (el) => { + const tag = tagOf(el); + const isInput = tag === "input"; + const typed = typedValue(el); + const sel = selectorsFor(el, typed); + const form = tag === "form" ? el : el.form || null; + return { + element_id: elementId(el), + tag, + input_type: isInput ? String(el.type || "text").toLowerCase() : null, + id: cut(clean(typed, attr(el, "id"))), + name: cut(clean(typed, attr(el, "name"))), + autocomplete: cut(clean(typed, attr(el, "autocomplete"))), + placeholder: cut(clean(typed, attr(el, "placeholder"))), + aria_label: cut(clean(typed, attr(el, "aria-label"))), + is_password_now: isInput && String(el.type).toLowerCase() === "password", + form: formInfo(form), + selectors: sel.selectors, + in_shadow_root: sel.inShadow, + }; + }; + + const originOf = (e) => { + let t = null; + try { + const path = e.composedPath ? e.composedPath() : []; + t = path.length > 0 ? path[0] : e.target; + } catch (_) { + t = e.target; + } + return t && t.nodeType === 1 ? t : null; + }; + + const report = (kind, el, withValue) => { + let value = null; + if (withValue && textEntry(el)) { + const v = String(el.value === undefined ? "" : el.value); + if (v.length <= MAX_VALUE) value = v; + } + const body = { + kind, + document_id: documentId, + frame_url: cut(location.href) || "", + is_top: isTop(), + element: describe(el), + }; + if (value !== null) body.value = value; + send(body); + }; + + const CLICKABLE = 'button, a, input[type="submit"], input[type="button"], [role="button"]'; + + const handlers = { + input: (e) => { + const el = originOf(e); + if (el && matchesFn.call(el, "input, textarea, select")) report("input", el, true); + }, + change: (e) => { + const el = originOf(e); + if (el && matchesFn.call(el, "input, textarea, select")) report("change", el, true); + }, + click: (e) => { + const el = originOf(e); + const hit = el ? closestFn.call(el, CLICKABLE) : null; + if (hit) report("click", hit, false); + }, + submit: (e) => { + const el = originOf(e); + if (el && tagOf(el) === "form") report("submit", el, false); + }, + keydown: (e) => { + if (e.key !== "Enter") return; + const el = originOf(e); + if (el && tagOf(el) === "input") report("enter_key", el, false); + }, + }; + + for (const type of Object.keys(handlers)) { + const fn = handlers[type]; + document.addEventListener( + type, + (e) => { + try { + fn(e); + } catch (_) { + /* never throw into the page */ + } + }, + true, + ); + } + + if (isTop()) send({ hello: true, nonce: NONCE }); + } catch (_) { + /* never throw into the page */ + } +})(); diff --git a/crates/kharon-teach/src/main.rs b/crates/kharon-teach/src/main.rs new file mode 100644 index 0000000..6e59609 --- /dev/null +++ b/crates/kharon-teach/src/main.rs @@ -0,0 +1,58 @@ +use clap::Parser; +use kharon_teach::cli::Cli; +use std::process::ExitCode; + +/// The one line a panic prints: where it happened and nothing from its message. The message of +/// a panic can hold a slice of whatever text the program was handling. +fn panic_line(file: &str, line: u32) -> String { + format!("kharon-teach: internal error at {file}:{line}") +} + +fn main() -> ExitCode { + // First of all, so that no panic ever reaches the default hook, which prints its payload. + std::panic::set_hook(Box::new(|info| match info.location() { + Some(l) => eprintln!("{}", panic_line(l.file(), l.line())), + None => eprintln!("kharon-teach: internal error"), + })); + let cli = Cli::parse(); + let runtime = match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + { + Ok(r) => r, + Err(e) => { + eprintln!("kharon-teach: cannot start: {e}"); + return ExitCode::from(1); + } + }; + let exit = runtime.block_on(kharon_teach::run( + cli, + tokio::io::BufReader::new(tokio::io::stdin()), + std::io::stdout(), + std::io::stderr(), + )); + // `run` has already closed the browser and removed its profile, whatever ended the session + // (the window, a confirm, Ctrl-C or SIGTERM). What is left is the blocking thread of the + // end-prompt read, which cannot be cancelled and would hold the process until Enter, so + // the runtime is shut down without waiting for it. + { + use std::io::Write; + let _ = std::io::stdout().flush(); + let _ = std::io::stderr().flush(); + } + runtime.shutdown_background(); + exit.into() +} + +#[cfg(test)] +mod tests { + use super::panic_line; + + #[test] + fn the_panic_line_names_the_place_and_nothing_else() { + assert_eq!( + panic_line("src/strip.rs", 134), + "kharon-teach: internal error at src/strip.rs:134" + ); + } +} diff --git a/crates/kharon-teach/src/marker.rs b/crates/kharon-teach/src/marker.rs new file mode 100644 index 0000000..58cb43a --- /dev/null +++ b/crates/kharon-teach/src/marker.rs @@ -0,0 +1,852 @@ +//! Typed markers: a value the user typed, held only in memory, searched for in request bytes +//! under the encodings a site might apply. A `Marker` can never be printed. + +use base64ct::{Base64, Base64Unpadded, Base64Url, Base64UrlUnpadded, Encoding as B64}; +use kharon_recording::{Encoding, MarkerHit, Role}; +use md5::Md5; +use memchr::memmem; +use sha1::Sha1; +use sha2::{Digest, Sha256, Sha512}; +use std::sync::OnceLock; + +use zeroize::Zeroizing; + +/// Markers shorter than this many characters produce low-confidence hits. +pub const LOW_CONFIDENCE_BELOW: u32 = 6; + +/// A typed value and its role. It implements none of `Debug`, `Display`, `Serialize` or `Clone`. +/// +/// ```compile_fail +/// let m = kharon_teach::marker::Marker::new(zeroize::Zeroizing::new(String::from("x")), kharon_recording::Role::Password); +/// println!("{:?}", m); +/// ``` +/// +/// ```compile_fail +/// let m = kharon_teach::marker::Marker::new(zeroize::Zeroizing::new(String::from("x")), kharon_recording::Role::Password); +/// println!("{}", m); +/// ``` +/// +/// ```compile_fail +/// let m = kharon_teach::marker::Marker::new(zeroize::Zeroizing::new(String::from("x")), kharon_recording::Role::Password); +/// let _ = m.clone(); +/// ``` +pub struct Marker { + value: Zeroizing, + role: Role, + /// The encoded forms searched for, built on first use and then reused by every search. + /// Zeroized buffers; dropped with the marker. + forms: OnceLock)>>, +} + +impl Marker { + pub fn new(value: Zeroizing, role: Role) -> Self { + Self { + value, + role, + forms: OnceLock::new(), + } + } + + /// The forms to search for, in `Encoding` order. Built once. + fn forms(&self) -> &[(Encoding, Vec)] { + self.forms.get_or_init(|| build_forms(self)) + } + + pub fn role(&self) -> Role { + self.role + } + + /// Length in characters. + pub fn len(&self) -> u32 { + u32::try_from(self.value.chars().count()).unwrap_or(u32::MAX) + } + + pub fn is_empty(&self) -> bool { + self.value.is_empty() + } + + /// Whether both markers hold the same value (role is ignored). Not constant time. + pub fn same_value(&self, other: &Marker) -> bool { + self.value.as_str() == other.value.as_str() + } + + /// The same value under another role. The forms are rebuilt: only a password has digests. + pub fn with_role(self, role: Role) -> Self { + Self::new(self.value, role) + } + + /// Byte range of the first place this marker's value occurs in `haystack`, comparing ASCII letters + /// without regard to case. `None` for an empty marker. Used for host names, which arrive lowercased. + /// + /// Compares byte windows in place, so no lowercase copy of the value is ever made. Non-ASCII + /// bytes must match exactly. The range starts and ends on character boundaries: a match + /// begins with an ASCII byte or a UTF-8 lead byte, and ends with a complete character. + pub fn find_ignore_ascii_case(&self, haystack: &str) -> Option> { + let needle = self.value.as_bytes(); + if needle.is_empty() { + return None; + } + haystack + .as_bytes() + .windows(needle.len()) + .position(|w| w.eq_ignore_ascii_case(needle)) + .map(|start| start..start + needle.len()) + } + + /// Byte ranges of every place this marker's value, lowercased (Unicode lowercasing), occurs + /// in `lowered`, which the caller has already lowercased. Empty for an empty or all-ASCII + /// value: `find_ignore_ascii_case` covers those without copying the value. Used for an + /// internationalised host name in its Unicode form. The lowercase copy is zeroized. + pub(crate) fn find_lowercased_in(&self, lowered: &str) -> Vec> { + if self.value.is_empty() || self.value.is_ascii() { + return Vec::new(); + } + let needle = Zeroizing::new(self.value.to_lowercase()); + lowered + .match_indices(needle.as_str()) + .map(|(start, found)| start..start + found.len()) + .collect() + } +} + +type Needle = Zeroizing>; + +/// The request bytes as sent, plus up to two percent-decoded copies: `%XX` only, and `+` as a +/// space then `%XX`. A copy identical to the raw bytes is left out. The copies are zeroized. +struct Haystacks<'a> { + raw: &'a [u8], + decoded: Vec, +} + +impl<'a> Haystacks<'a> { + fn new(raw: &'a [u8]) -> Self { + let mut decoded: Vec = Vec::new(); + for plus_is_space in [false, true] { + let copy = percent_decode(raw, plus_is_space); + if copy.as_slice() != raw && !decoded.iter().any(|d| d.as_slice() == copy.as_slice()) { + decoded.push(copy); + } + } + Self { raw, decoded } + } + + fn in_raw(&self, forms: &[Needle]) -> bool { + forms.iter().any(|n| contains(self.raw, n)) + } + + fn in_decoded(&self, forms: &[Needle]) -> bool { + forms + .iter() + .any(|n| self.decoded.iter().any(|d| contains(d, n))) + } +} + +fn hex_value(b: u8) -> Option { + char::from(b) + .to_digit(16) + .and_then(|d| u8::try_from(d).ok()) +} + +/// Decodes `%XX` (either hex case); a `%` not followed by two hex digits is copied unchanged. +/// With `plus_is_space`, `+` becomes a space first. The one percent-decoder of this crate; the +/// copy is zeroized. +pub(crate) fn percent_decode(raw: &[u8], plus_is_space: bool) -> Needle { + let mut out = Zeroizing::new(Vec::with_capacity(raw.len())); + let mut i = 0; + while i < raw.len() { + let b = raw[i]; + if b == b'%' + && let (Some(hi), Some(lo)) = ( + raw.get(i + 1).copied().and_then(hex_value), + raw.get(i + 2).copied().and_then(hex_value), + ) + { + out.push(hi * 16 + lo); + i += 3; + continue; + } + out.push(if plus_is_space && b == b'+' { b' ' } else { b }); + i += 1; + } + out +} + +/// Every way `marker` occurs in `haystack`. Digests are searched only for Role::Password. +/// +/// At most one hit per `Encoding`, in the order of the enum's variants. An encoded form that is +/// byte-identical to the as-typed form is not searched, so a plain value reports only `AsTyped`. +/// +/// Percent encoding is found by decoding the haystack (see `Haystacks`) and searching the raw +/// bytes and both decoded copies. `Percent` is reported when the typed value is found only in a +/// decoded copy; every other encoding is reported when its form is found in any of the three. +pub fn search(haystack: &[u8], marker: &Marker) -> Vec { + if marker.is_empty() { + return Vec::new(); + } + let low_confidence = marker.len() < LOW_CONFIDENCE_BELOW; + let haystacks = Haystacks::new(haystack); + let mut hits = Vec::new(); + let mut hit = |encoding| { + hits.push(MarkerHit { + role: marker.role, + encoding, + low_confidence, + }); + }; + for (encoding, forms) in marker.forms() { + let encoding = *encoding; + if encoding == Encoding::AsTyped { + if haystacks.in_raw(forms) { + hit(Encoding::AsTyped); + } else if haystacks.in_decoded(forms) { + hit(Encoding::Percent); + } + } else if haystacks.in_raw(forms) || haystacks.in_decoded(forms) { + hit(encoding); + } + } + hits +} + +/// The base64 and base64url (padded or not) of `username:password`. A match returns two hits, +/// one per role, both `Encoding::Base64` (or `Base64Url`). +/// +/// Searches the same three haystacks as `search`. Nothing is returned when either marker is empty. `low_confidence` is set when the combined +/// length of both values is under `LOW_CONFIDENCE_BELOW` characters. +pub fn search_pair(haystack: &[u8], username: &Marker, password: &Marker) -> Vec { + if username.is_empty() || password.is_empty() { + return Vec::new(); + } + let low_confidence = username.len().saturating_add(password.len()) < LOW_CONFIDENCE_BELOW; + let (standard, url_safe) = pair_forms(username, password); + let haystacks = Haystacks::new(haystack); + let mut hits = Vec::new(); + for (encoding, forms) in [ + (Encoding::Base64, standard), + (Encoding::Base64Url, url_safe), + ] { + if haystacks.in_raw(&forms) || haystacks.in_decoded(&forms) { + for role in [username.role, password.role] { + hits.push(MarkerHit { + role, + encoding, + low_confidence, + }); + } + } + } + hits +} + +/// Whether any of `forms` occurs in the raw bytes or a decoded copy as a whole token: the byte +/// before it and the byte after it are not ASCII alphanumeric, or the text ends there. +fn contains_whole_token(haystacks: &Haystacks, forms: &[Needle]) -> bool { + let bounded = |hay: &[u8], needle: &[u8]| { + let mut from = 0; + while let Some(i) = memmem::find(&hay[from..], needle) { + let (start, end) = (from + i, from + i + needle.len()); + let before = start.checked_sub(1).map(|j| hay[j]); + let after = hay.get(end).copied(); + if !before.is_some_and(|b| b.is_ascii_alphanumeric()) + && !after.is_some_and(|b| b.is_ascii_alphanumeric()) + { + return true; + } + from = start + 1; + } + false + }; + forms + .iter() + .filter(|n| !n.is_empty()) + .any(|n| bounded(haystacks.raw, n) || haystacks.decoded.iter().any(|d| bounded(d, n))) +} + +/// Whether `marker` occurs in `haystack` in ANY of its forms (as typed, escaped, base64, hex, and +/// for a password the digests) as a whole token. For kept text, where a short marker's forms +/// would otherwise match inside ordinary words. +pub(crate) fn found_as_whole_token(haystack: &[u8], marker: &Marker) -> bool { + if marker.is_empty() { + return false; + } + let haystacks = Haystacks::new(haystack); + marker + .forms() + .iter() + .any(|(_, forms)| contains_whole_token(&haystacks, forms)) +} + +/// [`found_as_whole_token`] for the base64 forms of `username:password`. +pub(crate) fn pair_found_as_whole_token( + haystack: &[u8], + username: &Marker, + password: &Marker, +) -> bool { + if username.is_empty() || password.is_empty() { + return false; + } + let (standard, url_safe) = pair_forms(username, password); + let haystacks = Haystacks::new(haystack); + contains_whole_token(&haystacks, &standard) || contains_whole_token(&haystacks, &url_safe) +} + +/// The base64 forms of `username:password`: (standard, URL-safe), each padded then unpadded, +/// the URL-safe ones without any that equal a standard one. +fn pair_forms(username: &Marker, password: &Marker) -> (Vec, Vec) { + let mut joined = Zeroizing::new(String::with_capacity( + username.value.len() + password.value.len() + 1, + )); + joined.push_str(&username.value); + joined.push(':'); + joined.push_str(&password.value); + let (standard, url_safe) = base64_forms(joined.as_bytes()); + let url_safe = { + let seen: Vec<&[u8]> = standard.iter().map(|n| n.as_slice()).collect(); + dedup(url_safe, &seen) + }; + (standard, url_safe) +} + +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() && memmem::find(haystack, needle).is_some() +} + +/// The search forms per encoding, in `Encoding` order. +fn build_forms(marker: &Marker) -> Vec<(Encoding, Vec)> { + let value = marker.value.as_bytes(); + let (standard, url_safe) = base64_forms(value); + let standard = dedup(standard, &[value]); + let url_safe = { + let mut seen: Vec<&[u8]> = vec![value]; + seen.extend(standard.iter().map(|n| n.as_slice())); + dedup(url_safe, &seen) + }; + let mut out = vec![ + (Encoding::AsTyped, vec![Zeroizing::new(value.to_vec())]), + ( + Encoding::JsonEscaped, + dedup(json_forms(&marker.value), &[value]), + ), + (Encoding::Base64, standard), + (Encoding::Base64Url, url_safe), + (Encoding::Hex, hex_forms(value)), + ]; + if marker.role == Role::Password { + let digests: [(Encoding, Encoding, Vec); 4] = [ + ( + Encoding::Md5Hex, + Encoding::Md5Base64, + Md5::digest(value).to_vec(), + ), + ( + Encoding::Sha1Hex, + Encoding::Sha1Base64, + Sha1::digest(value).to_vec(), + ), + ( + Encoding::Sha256Hex, + Encoding::Sha256Base64, + Sha256::digest(value).to_vec(), + ), + ( + Encoding::Sha512Hex, + Encoding::Sha512Base64, + Sha512::digest(value).to_vec(), + ), + ]; + for (hex_encoding, b64_encoding, digest) in digests { + out.push((hex_encoding, hex_forms(&digest))); + out.push((b64_encoding, base64_forms(&digest).0)); + } + } + out +} + +/// Drops empty forms, forms equal to one in `seen`, and repeats. +fn dedup(forms: Vec, seen: &[&[u8]]) -> Vec { + let mut kept: Vec = Vec::new(); + for form in forms { + let known = form.is_empty() + || seen.iter().any(|s| *s == form.as_slice()) + || kept.iter().any(|k| k.as_slice() == form.as_slice()); + if !known { + kept.push(form); + } + } + kept +} + +fn encode_b64(bytes: &[u8]) -> Needle { + let mut buf = Zeroizing::new(vec![0u8; E::encoded_len(bytes)]); + if E::encode(bytes, &mut buf).is_err() { + // The buffer is sized by `encoded_len`, so this cannot happen; match nothing if it does. + buf.clear(); + } + buf +} + +/// (standard, URL-safe), each padded then unpadded. +fn base64_forms(bytes: &[u8]) -> (Vec, Vec) { + ( + vec![ + encode_b64::(bytes), + encode_b64::(bytes), + ], + vec![ + encode_b64::(bytes), + encode_b64::(bytes), + ], + ) +} + +/// Lowercase then uppercase. +fn hex_forms(bytes: &[u8]) -> Vec { + let mut lower = Zeroizing::new(vec![0u8; bytes.len() * 2]); + if hex::encode_to_slice(bytes, &mut lower).is_err() { + lower.clear(); + } + let mut upper = lower.clone(); + upper.make_ascii_uppercase(); + dedup(vec![lower, upper], &[]) +} + +#[derive(Clone, Copy)] +enum JsonMode { + /// Non-ASCII characters stay as UTF-8. + Raw, + /// Non-ASCII characters become backslash-u escapes, with hex digits in the given case. + Ascii { upper: bool }, +} + +/// The content between the quotes as `serde_json` writes it (raw non-ASCII), and the form with +/// every non-ASCII character as a backslash-u escape in each hex case. +fn json_forms(value: &str) -> Vec { + [ + JsonMode::Raw, + JsonMode::Ascii { upper: false }, + JsonMode::Ascii { upper: true }, + ] + .into_iter() + .map(|mode| json_escape(value, mode)) + .collect() +} + +fn push_u_escape(out: &mut Vec, unit: u16, upper: bool) { + let digits: &[u8; 16] = if upper { + b"0123456789ABCDEF" + } else { + b"0123456789abcdef" + }; + out.extend_from_slice(b"\\u"); + for shift in [12, 8, 4, 0] { + out.push(digits[usize::from((unit >> shift) & 0xf)]); + } +} + +fn json_escape(value: &str, mode: JsonMode) -> Needle { + let upper = matches!(mode, JsonMode::Ascii { upper: true }); + let mut out = Zeroizing::new(Vec::with_capacity(value.len() * 6)); + for c in value.chars() { + match c { + '"' => out.extend_from_slice(b"\\\""), + '\\' => out.extend_from_slice(b"\\\\"), + '\n' => out.extend_from_slice(b"\\n"), + '\r' => out.extend_from_slice(b"\\r"), + '\t' => out.extend_from_slice(b"\\t"), + '\u{8}' => out.extend_from_slice(b"\\b"), + '\u{c}' => out.extend_from_slice(b"\\f"), + c if u32::from(c) < 0x20 => push_u_escape(&mut out, u16::from(c as u8), upper), + c if c.is_ascii() => out.push(c as u8), + c => match mode { + JsonMode::Raw => { + let mut buf = [0u8; 4]; + out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes()); + } + JsonMode::Ascii { .. } => { + let mut units = [0u16; 2]; + for unit in c.encode_utf16(&mut units) { + push_u_escape(&mut out, *unit, upper); + } + } + }, + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pw(s: &str) -> Marker { + Marker::new(Zeroizing::new(s.to_owned()), Role::Password) + } + fn user(s: &str) -> Marker { + Marker::new(Zeroizing::new(s.to_owned()), Role::Username) + } + /// `@` stands for a backslash, so escape sequences stay literal in the source. + fn bs(s: &str) -> String { + s.replace('@', "\\") + } + fn encodings(h: &[u8], m: &Marker) -> Vec { + search(h, m).into_iter().map(|x| x.encoding).collect() + } + + #[test] + fn many_searches_with_one_marker_are_fast() { + // The encoded forms (four digests, base64, hex, JSON) are built once per marker, not once + // per search. Measured in a debug build: rebuilding them for each search took about 4.2 s + // for this many searches, reusing them about 0.8 s. + let m = pw("Tr0ub4dor-9x"); + let hay = b"user=kh.throwaway7&csrf=abc123&password=Tr0ub4dor-9x"; + let started = std::time::Instant::now(); + let mut found = 0; + for _ in 0..40_000 { + found += search(hay, &m).len(); + } + assert_eq!(found, 40_000); + assert!( + started.elapsed() < std::time::Duration::from_secs(2), + "40,000 searches took {:?}", + started.elapsed() + ); + } + + #[test] + fn as_typed() { + assert_eq!( + encodings(b"user=a&password=Tr0ub4dor-9x&x=1", &pw("Tr0ub4dor-9x")), + [Encoding::AsTyped] + ); + } + + #[test] + fn percent_encoded() { + assert_eq!( + encodings(b"p=a%20b%26c%2Bd12", &pw("a b&c+d12")), + [Encoding::Percent] + ); + } + + #[test] + fn percent_encoded_lowercase_hex() { + assert_eq!( + encodings(b"p=a%2fb%26c%2bd12", &pw("a/b&c+d12")), + [Encoding::Percent] + ); + } + + #[test] + fn percent_encoded_plus_for_space() { + assert_eq!( + encodings(b"p=a+b%26c%2Bd12", &pw("a b&c+d12")), + [Encoding::Percent] + ); + } + + #[test] + fn json_escaped() { + assert_eq!( + encodings(br#"{"p":"a\"b\\c12"}"#, &pw(r#"a"b\c12"#)), + [Encoding::JsonEscaped] + ); + } + + #[test] + fn json_escaped_control_characters() { + assert_eq!( + encodings(br#"{"p":"a\nb\u0001c12"}"#, &pw("a\nb\u{1}c12")), + [Encoding::JsonEscaped] + ); + } + + #[test] + fn base64_and_base64url() { + assert_eq!( + encodings(b"x=Pj4+Pz8/MTI=", &pw(">>>???12")), + [Encoding::Base64] + ); + // unpadded is found too + assert_eq!( + encodings(b"x=Pj4-Pz8_MTI", &pw(">>>???12")), + [Encoding::Base64Url] + ); + } + + #[test] + fn hex_either_case() { + let lower = hex::encode("Tr0ub4dor-9x"); + let upper = lower.to_uppercase(); + assert_eq!( + encodings(lower.as_bytes(), &pw("Tr0ub4dor-9x")), + [Encoding::Hex] + ); + assert_eq!( + encodings(upper.as_bytes(), &pw("Tr0ub4dor-9x")), + [Encoding::Hex] + ); + } + + #[test] + fn sha256_hex_of_a_password() { + let digest = hex::encode(Sha256::digest(b"Tr0ub4dor-9x")); + let haystack = format!("h={digest}"); + assert_eq!( + encodings(haystack.as_bytes(), &pw("Tr0ub4dor-9x")), + [Encoding::Sha256Hex] + ); + } + + #[test] + fn digests_are_not_searched_for_a_username() { + let digest = hex::encode(Sha256::digest(b"Tr0ub4dor-9x")); + let haystack = format!("h={digest}"); + assert!(search(haystack.as_bytes(), &user("Tr0ub4dor-9x")).is_empty()); + } + + #[test] + fn short_marker_is_low_confidence() { + assert!(search(b"x=abc12", &pw("abc12"))[0].low_confidence); + assert!(!search(b"x=abc123", &pw("abc123"))[0].low_confidence); + } + + #[test] + fn empty_marker_matches_nothing() { + assert!(search(b"anything", &pw("")).is_empty()); + } + + #[test] + fn hit_carries_the_marker_role() { + let hits = search(b"u=kh.throwaway7", &user("kh.throwaway7")); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].role, Role::Username); + } + + #[test] + fn non_ascii_percent_encoded() { + assert_eq!( + encodings( + b"p=p%C3%A4ssw%C3%B6rd%E2%9C%9312", + &pw("p\u{e4}ssw\u{f6}rd\u{2713}12") + ), + [Encoding::Percent] + ); + } + + #[test] + fn non_ascii_json_escaped_both_forms() { + let m = pw("p\u{e4}ssw\u{f6}rd\u{2713}12"); + let raw = "{\"p\":\"p\u{e4}ssw\u{f6}rd\u{2713}12\"}"; + assert_eq!(encodings(raw.as_bytes(), &m), [Encoding::AsTyped]); + assert_eq!( + encodings(bs("{\"p\":\"p@u00e4ssw@u00f6rd@u271312\"}").as_bytes(), &m), + [Encoding::JsonEscaped] + ); + assert_eq!( + encodings(bs("{\"p\":\"p@u00E4ssw@u00F6rd@u271312\"}").as_bytes(), &m), + [Encoding::JsonEscaped] + ); + } + + #[test] + fn json_surrogate_pair() { + assert_eq!( + encodings( + bs("{\"p\":\"pw@ud83d@ude00xyz\"}").as_bytes(), + &pw("pw\u{1f600}xyz") + ), + [Encoding::JsonEscaped] + ); + } + + #[test] + fn two_hundred_characters() { + let value = "Zq9".repeat(66) + "ab"; + assert_eq!(value.chars().count(), 200); + let haystack = format!("a=1&p={value}&b=2"); + assert_eq!( + encodings(haystack.as_bytes(), &pw(&value)), + [Encoding::AsTyped] + ); + } + + #[test] + fn pair_is_found_where_neither_value_alone_is() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let haystack = format!(r#"{{"auth":"{b64}"}}"#); + let u = user("kh.throwaway7"); + let p = pw("Tr0ub4dor-9x"); + assert!(search(haystack.as_bytes(), &u).is_empty()); + assert!(search(haystack.as_bytes(), &p).is_empty()); + let hits = search_pair(haystack.as_bytes(), &u, &p); + assert_eq!( + hits.iter() + .map(|h| (h.role, h.encoding)) + .collect::>(), + [ + (Role::Username, Encoding::Base64), + (Role::Password, Encoding::Base64) + ] + ); + } + + #[test] + fn pair_absent_gives_nothing() { + assert!( + search_pair(b"nothing here", &user("kh.throwaway7"), &pw("Tr0ub4dor-9x")).is_empty() + ); + } + + #[test] + fn component_encoding_leaves_punctuation_literal() { + let m = pw("my pass!x*9"); + assert_eq!(encodings(b"p=my%20pass!x*9", &m), [Encoding::Percent]); + assert_eq!(encodings(b"p=my+pass%21x*9", &m), [Encoding::Percent]); + } + + #[test] + fn percent_hex_digits_in_mixed_case() { + assert_eq!( + encodings(b"p=p%c3%A4ssw%C3%b6rd12", &pw("p\u{e4}ssw\u{f6}rd12")), + [Encoding::Percent] + ); + } + + #[test] + fn stray_percent_does_not_panic_or_match() { + let m = pw("abcdef12"); + assert!(search(b"p=abc%", &m).is_empty()); + assert!(search(b"p=abc%zzdef12", &m).is_empty()); + assert!(search(b"%", &m).is_empty()); + assert!(search(b"%4", &m).is_empty()); + assert_eq!(encodings(b"p=%zz%61bcdef12", &m), [Encoding::Percent]); + } + + #[test] + fn base64_with_percent_encoded_symbols() { + assert_eq!( + encodings(b"x=Pj4%2BPz8%2FMTI%3D", &pw(">>>???12")), + [Encoding::Base64] + ); + } + + #[test] + fn raw_match_reports_only_as_typed() { + assert_eq!( + encodings(b"a=my%20pass!x*9&b=my pass!x*9", &pw("my pass!x*9")), + [Encoding::AsTyped] + ); + } + + #[test] + fn every_digest_in_hex_and_base64() { + let value = "Tr0ub4dor-9x"; + let digests: [(Encoding, Encoding, Vec); 4] = [ + ( + Encoding::Md5Hex, + Encoding::Md5Base64, + Md5::digest(value).to_vec(), + ), + ( + Encoding::Sha1Hex, + Encoding::Sha1Base64, + Sha1::digest(value).to_vec(), + ), + ( + Encoding::Sha256Hex, + Encoding::Sha256Base64, + Sha256::digest(value).to_vec(), + ), + ( + Encoding::Sha512Hex, + Encoding::Sha512Base64, + Sha512::digest(value).to_vec(), + ), + ]; + for (hex_enc, b64_enc, digest) in digests { + for (expected, text) in [ + (hex_enc, hex::encode(&digest)), + (hex_enc, hex::encode(&digest).to_uppercase()), + (b64_enc, Base64::encode_string(&digest)), + (b64_enc, Base64Unpadded::encode_string(&digest)), + ] { + let haystack = format!("h={text}&z=1"); + assert_eq!(encodings(haystack.as_bytes(), &pw(value)), [expected]); + assert!(search(haystack.as_bytes(), &user(value)).is_empty()); + } + } + } + + #[test] + fn pair_in_url_safe_base64() { + let joined = b"kh>>>user:pa??w0rd-9"; + let std = Base64::encode_string(joined); + assert!(std.contains('+')); + let url = Base64UrlUnpadded::encode_string(joined); + let haystack = format!("auth={url}"); + let hits = search_pair(haystack.as_bytes(), &user("kh>>>user"), &pw("pa??w0rd-9")); + assert_eq!( + hits.iter() + .map(|h| (h.role, h.encoding)) + .collect::>(), + [ + (Role::Username, Encoding::Base64Url), + (Role::Password, Encoding::Base64Url) + ] + ); + } + + #[test] + fn pair_with_percent_encoded_base64() { + let joined = b"kh>>>user:pa??w0rd-9"; + let haystack = format!( + "auth={}", + Base64::encode_string(joined) + .replace('+', "%2B") + .replace('/', "%2F") + .replace('=', "%3D") + ); + assert_eq!( + search_pair(haystack.as_bytes(), &user("kh>>>user"), &pw("pa??w0rd-9")).len(), + 2 + ); + } + + #[test] + fn same_value_and_with_role() { + let a = pw("abc"); + let b = user("abc"); + assert!(a.same_value(&b)); + assert_eq!(a.with_role(Role::None).role(), Role::None); + assert_eq!(pw("h\u{e4}").len(), 2); + } + + #[test] + fn find_ignore_ascii_case_in_a_lowercased_host() { + let m = user("Kh.Throwaway7"); + assert_eq!( + m.find_ignore_ascii_case("a.kh.throwaway7.ex.test"), + Some(2..15) + ); + } + + #[test] + fn find_ignore_ascii_case_no_match_and_empty() { + assert_eq!( + user("kh.throwaway7").find_ignore_ascii_case("ex.test"), + None + ); + assert_eq!(user("longer-than-host").find_ignore_ascii_case("a"), None); + assert_eq!(user("").find_ignore_ascii_case("ex.test"), None); + } + + #[test] + fn find_ignore_ascii_case_non_ascii_is_byte_exact() { + let m = user("M\u{fc}ller1"); + assert_eq!(m.find_ignore_ascii_case("x-m\u{fc}ller1"), Some(2..10)); + assert_eq!( + user("\u{c9}t\u{e9}12").find_ignore_ascii_case("\u{e9}t\u{e9}12"), + None + ); + } +} diff --git a/crates/kharon-teach/src/notice.rs b/crates/kharon-teach/src/notice.rs new file mode 100644 index 0000000..e0409c4 --- /dev/null +++ b/crates/kharon-teach/src/notice.rs @@ -0,0 +1,117 @@ +//! The page the browser opens first: what the recorder does with what a person types, and one +//! link that starts the recording. + +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode}; + +pub const NOTICE: &str = "kharon-teach records how a login works so kharon can learn it.\n\ +Use a THROWAWAY account. Everything you type in the window that\n\ +opens is read by this program. Nothing you type is saved, and\n\ +nothing is sent anywhere. The result is one file on this computer.\n"; + +/// Asked on the terminal before anything starts. +pub const CONSENT_PROMPT: &str = "Type yes to open the browser window and start: "; +/// Printed once the browser window is showing the notice. +pub const END_PROMPT: &str = "Press Enter once you are logged in. Type q to discard."; +/// Printed when a line at the end prompt is neither Enter, `q` nor `quit`. +pub const END_RETRY: &str = "Press Enter to save, or type q to discard."; +/// Printed when the session ended before the recording ever started: the login page was not +/// opened in the first tab (the window was closed, or the link was opened in another tab). +pub const NEVER_STARTED: &str = + "Recording never started: the login page was not opened in the first tab. Nothing was saved."; +/// Under the link on the notice page. +pub const OPEN_IN_THIS_TAB: &str = "Open the link in this tab."; +/// Printed when consent was not given. +pub const NOTHING_RECORDED: &str = "Nothing was recorded."; + +const KEEP: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~'); + +fn escape_html(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + c => out.push(c), + } + } + out +} + +/// The notice page as HTML text. It holds no script. +pub fn notice_page_html(login_url: &url::Url) -> String { + let host = login_url.host_str().unwrap_or("the login page"); + format!( + "kharon-teach\ +
{notice}

Start recording and open {host}
{tab}

", + notice = escape_html(NOTICE), + tab = escape_html(OPEN_IN_THIS_TAB), + href = escape_html(login_url.as_str()), + host = escape_html(host), + ) +} + +/// A `data:` URL page that shows [`NOTICE`] and one link, "Start recording and open ", to +/// `login_url`. +pub fn notice_page_url(login_url: &url::Url) -> String { + let html = notice_page_html(login_url); + format!( + "data:text/html;charset=utf-8,{}", + utf8_percent_encode(&html, KEEP) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_page_shows_the_notice_and_one_link_to_the_login_url() { + let url = url::Url::parse("https://login.example.test/sign-in?next=a").unwrap(); + let html = notice_page_html(&url); + assert!(html.contains("THROWAWAY")); + assert!(html.contains("Start recording and open login.example.test")); + assert_eq!(html.matches("
Open the link in this tab.

")); + } + + #[test] + fn hostile_characters_are_escaped_and_the_href_round_trips() { + let url = url::Url::parse("https://a.test/p?x=1&y='2'&z=\"q\"&w=%3Cscript%3E").unwrap(); + let html = notice_page_html(&url); + let start = html.find("href=\"").unwrap() + 6; + let end = start + html[start..].find('"').unwrap(); + let attr = &html[start..end]; + assert!(!attr.contains('<') && !attr.contains('\'') && !attr.contains('>')); + let back = attr + .replace("<", "<") + .replace(">", ">") + .replace("'", "'") + .replace(""", "\"") + .replace("&", "&"); + assert_eq!(back, url.as_str()); + assert_eq!(html.matches("
")); + } + + #[test] + fn the_url_is_a_data_url_with_no_raw_markup() { + let url = url::Url::parse("https://a.test/").unwrap(); + let page = notice_page_url(&url); + let body = page.strip_prefix("data:text/html;charset=utf-8,").unwrap(); + assert!(!body.contains('<') && !body.contains(' ') && !body.contains('"')); + let decoded = percent_encoding::percent_decode_str(body) + .decode_utf8() + .unwrap(); + assert_eq!(decoded, notice_page_html(&url)); + } +} diff --git a/crates/kharon-teach/src/session.rs b/crates/kharon-teach/src/session.rs new file mode 100644 index 0000000..5696d37 --- /dev/null +++ b/crates/kharon-teach/src/session.rs @@ -0,0 +1,889 @@ +//! One recording session: a launched browser, a CDP connection and the capture state machine, +//! wired so that what a person does in the window becomes a [`Recording`]. +//! +//! Nothing in this file writes to stdout or stderr, and nothing from a frame reaches an error. + +use crate::browser::{Browser, BrowserError, Launch, Profile}; +use crate::capture::{ + Capture, CaptureEvent, EndInfo, PageEventKind, RawBody, RawElement, RawForm, RawFrame, +}; +use crate::cdp::{Cdp, CdpError, Event, parse_quiet}; +use crate::events::{ + AddBinding, AddScriptToEvaluateOnNewDocument, AttachedToTarget, BindingCalled, Empty, + FrameNavigated, GetRequestPostData, Hello, LayoutMetrics, ListenerReport, Navigate, PostData, + RemoveBinding, RemoveScript, ReportKind, RequestWillBeSent, ResponseReceived, + ResponseReceivedExtraInfo, ScriptId, SetAutoAttach, SetDiscoverTargets, TargetDestroyed, + VersionInfo, +}; +use crate::notice::notice_page_url; +use kharon_recording::{BrowserInfo, EndedBy, ListenerWorld, Recording, ResourceKind, Viewport}; +use serde::Serialize; +use serde::de::IgnoredAny; +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant, SystemTime}; +use tokio::sync::{mpsc::UnboundedReceiver, oneshot}; +use zeroize::Zeroizing; + +/// The page script, run in every frame at document start. +pub const LISTENER_JS: &str = include_str!("listener.js"); +const BINDING: &str = "__kharonTeach"; +const WORLD: &str = "kharon-teach"; +/// How long the listener may take to announce itself on the notice page. +const HELLO_WAIT: Duration = Duration::from_secs(5); +/// The same, when a test has made the isolated world unable to answer. +const HELLO_WAIT_SKIPPED: Duration = Duration::from_millis(300); +/// How long to keep taking events after the person confirms. +const LATE_WAIT: Duration = Duration::from_millis(300); +/// How long `Network.getRequestPostData` may take. +const POST_DATA_WAIT: Duration = Duration::from_secs(5); +/// How many times the recorder had to ask the browser for a request body that was not inline. +#[doc(hidden)] +pub static POST_DATA_FETCHES: AtomicUsize = AtomicUsize::new(0); +/// How long the browser may take to show its first page. +const FIRST_PAGE_WAIT: Duration = Duration::from_secs(10); + +/// Which world the listener is tried in first. `SkipIsolated` is for tests of the fallback. +#[doc(hidden)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum WorldProbe { + Normal, + /// The isolated world gets its binding but never the script, so the fallback runs. + SkipIsolated, +} + +pub enum End { + Confirmed, + Discard, +} + +pub struct RecordOptions<'a> { + pub login_url: &'a url::Url, + pub browser: &'a std::path::Path, + pub headless: bool, + pub tool_version: &'a str, +} + +pub struct Ready { + pub devtools_port: u16, + pub profile_dir: PathBuf, + pub stale_profiles_removed: usize, +} + +#[derive(Debug, thiserror::Error)] +pub enum SessionError { + #[error(transparent)] + Browser(#[from] BrowserError), + #[error(transparent)] + Cdp(#[from] CdpError), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("the browser showed no page")] + NoPage, + #[error("the page script did not start in the browser")] + ListenerNotRunning, +} + +pub enum Outcome { + Recorded(Box), + Discarded, + NeverStarted, +} + +/// RFC 3339, UTC. +pub fn utc_rfc3339(t: SystemTime) -> String { + use time::format_description::well_known::Rfc3339; + time::OffsetDateTime::from(t) + .format(&Rfc3339) + .unwrap_or_default() +} + +struct Sess { + target_id: String, + /// The id of the injected script, so the world can be changed once. + script_id: Option, +} + +/// One hop of a request. Chromium reuses a request id across redirects, and the raw headers of a +/// hop (`Set-Cookie`) can arrive after the next hop's `requestWillBeSent`. +#[derive(Default)] +struct Hop { + /// The raw-header event for this hop has been taken. + extra_seen: bool, + status: Option, + content_type: Option, + set_cookie: bool, +} + +enum Flow { + Continue, + WindowClosed, +} + +struct Rec { + cdp: Cdp, + notice: String, + world: ListenerWorld, + sessions: HashMap, + page_targets: HashSet, + first_session: Option, + first_main_frame: Option, + capture: Capture, + clock: Option<(Instant, SystemTime)>, + hops: HashMap>, + hello: bool, + /// This attempt's value in the script; only a hello carrying it counts. + nonce: String, + probe: WorldProbe, +} + +fn fresh_nonce() -> String { + use rand::Rng; + use rand::distributions::Alphanumeric; + rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(24) + .map(char::from) + .collect() +} + +async fn send( + cdp: &Cdp, + method: &str, + params: &P, + session: Option<&str>, +) -> Result<(), CdpError> { + cdp.call::(method, params, session) + .await + .map(|_| ()) +} + +/// Sweeps stale profiles, launches, records until `end` fires or the window closes, cleans up. +/// `ready` fires once the browser is up and the notice page is showing. Dropping the future +/// (Ctrl-C) still removes the profile. +pub async fn record( + opts: RecordOptions<'_>, + ready: oneshot::Sender, + end: oneshot::Receiver, +) -> Result { + record_with(opts, ready, end, WorldProbe::Normal).await +} + +/// [`record`], with a way for a test to make the isolated world fail. +#[doc(hidden)] +pub async fn record_with( + opts: RecordOptions<'_>, + ready: oneshot::Sender, + end: oneshot::Receiver, + probe: WorldProbe, +) -> Result { + let stale = Profile::sweep_stale().unwrap_or(0); + let profile = Profile::create()?; + let profile_dir = profile.path().to_owned(); + let notice = notice_page_url(opts.login_url); + let (browser, dev) = Browser::launch(Launch { + browser: opts.browser, + profile: &profile, + first_url: "about:blank", + headless: opts.headless, + }) + .await?; + let mut guard = Cleanup { + browser: Some(browser), + profile: Some(profile), + }; + let Some(browser) = guard.browser.as_mut() else { + return Err(SessionError::NoPage); + }; + let result = run( + browser, + &dev, + &opts, + ¬ice, + ( + Ready { + devtools_port: dev.port, + profile_dir, + stale_profiles_removed: stale, + }, + ready, + ), + end, + probe, + ) + .await; + guard.finish().await; + result +} + +/// Owns the browser and its profile. However the session ends, including the future being +/// dropped, the browser is killed and the profile removed. +struct Cleanup { + browser: Option, + profile: Option, +} + +/// Removal retries: the group is empty by now, but a filesystem can still be busy for a moment. +const REMOVE_TRIES: u32 = 60; +const REMOVE_PAUSE: Duration = Duration::from_millis(50); + +/// Removes the profile; true once it is gone. +fn try_remove(path: &std::path::Path) -> bool { + let _ = std::fs::remove_dir_all(path); + !path.exists() +} + +impl Cleanup { + /// The normal end: kill the browser's whole group, wait for it to be empty (at most 3 s), + /// then remove the profile. + async fn finish(&mut self) { + if let Some(b) = self.browser.take() { + let _ = b.close().await; + } + if let Some(p) = self.profile.take() { + let path = p.path().to_owned(); + for _ in 0..REMOVE_TRIES { + if try_remove(&path) { + break; + } + tokio::time::sleep(REMOVE_PAUSE).await; + } + } + } +} + +impl Drop for Cleanup { + /// A drop cannot await, so this blocks its thread: up to 3 s for the browser's group to be + /// empty, then up to 3 s for the removal. It is the Ctrl-C path, and it cannot hang. + fn drop(&mut self) { + if let Some(mut b) = self.browser.take() { + b.close_blocking(); + } + if let Some(p) = self.profile.take() { + let path = p.path().to_owned(); + for _ in 0..REMOVE_TRIES { + if try_remove(&path) { + break; + } + std::thread::sleep(REMOVE_PAUSE); + } + } + } +} + +async fn run( + browser: &mut Browser, + dev: &crate::browser::DevTools, + opts: &RecordOptions<'_>, + notice: &str, + (info, ready): (Ready, oneshot::Sender), + mut end: oneshot::Receiver, + probe: WorldProbe, +) -> Result { + let (cdp, mut rx) = Cdp::connect(dev).await?; + let mut rec = Rec { + cdp: cdp.clone(), + notice: notice.to_owned(), + world: ListenerWorld::Isolated, + sessions: HashMap::new(), + page_targets: HashSet::new(), + first_session: None, + first_main_frame: None, + capture: Capture::new(opts.login_url.as_str(), opts.tool_version), + clock: None, + hops: HashMap::new(), + hello: false, + nonce: fresh_nonce(), + probe, + }; + + send( + &cdp, + "Target.setDiscoverTargets", + &SetDiscoverTargets { discover: true }, + None, + ) + .await?; + send(&cdp, "Target.setAutoAttach", &auto_attach(), None).await?; + + // The browser's first page. + let deadline = tokio::time::Instant::now() + FIRST_PAGE_WAIT; + while rec.first_session.is_none() { + match tokio::time::timeout_at(deadline, rx.recv()).await { + Ok(Some(ev)) => { + if let Flow::WindowClosed = rec.on_event(ev).await { + return Ok(Outcome::NeverStarted); + } + } + Ok(None) | Err(_) => return Err(SessionError::NoPage), + } + } + + match rec.choose_world(&mut rx).await? { + Choice::Done => {} + Choice::Closed => return Ok(Outcome::NeverStarted), + } + let _ = ready.send(info); + + let mut closed = false; + let confirmed = loop { + tokio::select! { + biased; + e = &mut end => break matches!(e, Ok(End::Confirmed)), + ev = rx.recv() => match ev { + Some(ev) => { + if let Flow::WindowClosed = rec.on_event(ev).await { + closed = true; + break false; + } + } + None => { closed = true; break false; } + }, + _ = browser.wait() => { closed = true; break false; } + } + }; + + if !confirmed && !closed { + return Ok(Outcome::Discarded); + } + // Everything already queued, then a short wait for the last navigation and its response. + while let Ok(ev) = rx.try_recv() { + if closed { + rec.on_event(ev).await; + } else if let Flow::WindowClosed = rec.on_event(ev).await { + closed = true; + } + } + if !closed { + let deadline = tokio::time::Instant::now() + LATE_WAIT; + while let Ok(Some(ev)) = tokio::time::timeout_at(deadline, rx.recv()).await { + rec.on_event(ev).await; + } + } + Ok(rec.finish(confirmed)) +} + +fn auto_attach() -> SetAutoAttach { + SetAutoAttach { + auto_attach: true, + wait_for_debugger_on_start: true, + flatten: true, + } +} + +enum Choice { + Done, + Closed, +} + +enum Wait { + Hello, + Timeout, + Closed, +} + +impl Rec { + fn t_ms(&self) -> u64 { + self.clock.map_or(0, |(t0, _)| { + u64::try_from(t0.elapsed().as_millis()).unwrap_or(u64::MAX) + }) + } + + /// Show the notice page and find out which world the listener runs in. The world is never + /// switched after this. + async fn choose_world( + &mut self, + rx: &mut UnboundedReceiver, + ) -> Result { + let Some(first) = self.first_session.clone() else { + return Err(SessionError::NoPage); + }; + let notice = self.notice.clone(); + send( + &self.cdp, + "Page.navigate", + &Navigate { url: ¬ice }, + Some(&first), + ) + .await?; + match self.wait_hello(rx).await { + Wait::Hello => return Ok(Choice::Done), + Wait::Closed => return Ok(Choice::Closed), + Wait::Timeout => {} + } + // The isolated world did not deliver: install in the main world instead, under a new + // nonce, so that a late hello from the isolated attempt cannot pass for the new one. + if let Some(id) = self + .sessions + .get_mut(&first) + .and_then(|s| s.script_id.take()) + { + send( + &self.cdp, + "Page.removeScriptToEvaluateOnNewDocument", + &RemoveScript { identifier: &id }, + Some(&first), + ) + .await?; + } + self.world = ListenerWorld::Main; + self.nonce = fresh_nonce(); + self.install(&first).await?; + send( + &self.cdp, + "Page.navigate", + &Navigate { url: ¬ice }, + Some(&first), + ) + .await?; + match self.wait_hello(rx).await { + Wait::Hello => Ok(Choice::Done), + Wait::Closed => Ok(Choice::Closed), + Wait::Timeout => Err(SessionError::ListenerNotRunning), + } + } + + async fn wait_hello(&mut self, rx: &mut UnboundedReceiver) -> Wait { + self.hello = false; + let wait = + if self.world == ListenerWorld::Isolated && self.probe == WorldProbe::SkipIsolated { + HELLO_WAIT_SKIPPED + } else { + HELLO_WAIT + }; + let deadline = tokio::time::Instant::now() + wait; + while !self.hello { + match tokio::time::timeout_at(deadline, rx.recv()).await { + Ok(Some(ev)) => { + if let Flow::WindowClosed = self.on_event(ev).await { + return Wait::Closed; + } + } + Ok(None) => return Wait::Closed, + Err(_) => return Wait::Timeout, + } + } + Wait::Hello + } + + /// The binding and the script, in the world chosen now. + async fn install(&mut self, session: &str) -> Result<(), CdpError> { + let isolated = self.world == ListenerWorld::Isolated; + if !isolated { + // The isolated attempt's binding may still exist under the same name. + let _ = send( + &self.cdp, + "Runtime.removeBinding", + &RemoveBinding { name: BINDING }, + Some(session), + ) + .await; + } + send( + &self.cdp, + "Runtime.addBinding", + &AddBinding { + name: BINDING, + execution_context_name: isolated.then_some(WORLD), + }, + Some(session), + ) + .await?; + if isolated && self.probe == WorldProbe::SkipIsolated { + return Ok(()); + } + let source = LISTENER_JS.replacen("__NONCE__", &self.nonce, 1); + let id: ScriptId = self + .cdp + .call( + "Page.addScriptToEvaluateOnNewDocument", + &AddScriptToEvaluateOnNewDocument { + source: &source, + world_name: isolated.then_some(WORLD), + }, + Some(session), + ) + .await?; + if let Some(s) = self.sessions.get_mut(session) { + s.script_id = Some(id.identifier); + } + Ok(()) + } + + async fn setup(&mut self, session: &str) -> Result<(), CdpError> { + send( + &self.cdp, + "Target.setAutoAttach", + &auto_attach(), + Some(session), + ) + .await?; + send(&self.cdp, "Page.enable", &Empty {}, Some(session)).await?; + send(&self.cdp, "Network.enable", &Empty {}, Some(session)).await?; + send(&self.cdp, "Runtime.enable", &Empty {}, Some(session)).await?; + self.install(session).await?; + Ok(()) + } + + async fn on_event(&mut self, ev: Event) -> Flow { + let session = ev.session_id.as_deref(); + match ev.method.as_str() { + "Target.attachedToTarget" => { + if let Ok(p) = parse_quiet::(&ev.params) { + self.attached(p).await; + } + } + "Target.targetDestroyed" => { + if let Ok(p) = parse_quiet::(&ev.params) { + self.sessions.retain(|_, s| s.target_id != p.target_id); + if self.page_targets.remove(&p.target_id) && self.page_targets.is_empty() { + return Flow::WindowClosed; + } + } + } + _ => { + let Some(session) = session else { + return Flow::Continue; + }; + if !self.sessions.contains_key(session) { + return Flow::Continue; + } + match ev.method.as_str() { + "Page.frameNavigated" => { + if let Ok(p) = parse_quiet::(&ev.params) { + self.navigated(session, p).await; + } + } + "Runtime.bindingCalled" => { + if let Ok(p) = parse_quiet::(&ev.params) { + self.binding(session, &p); + } + } + "Network.requestWillBeSent" => { + if let Ok(p) = parse_quiet::(&ev.params) { + self.request(session, p).await; + } + } + "Network.responseReceived" => { + if let Ok(p) = parse_quiet::(&ev.params) { + let hops = self.hops.entry(p.request_id.clone()).or_default(); + if hops.is_empty() { + hops.push(Hop::default()); + } + let n = hops.len() - 1; + let hop = &mut hops[n]; + hop.status = Some(p.response.status); + hop.content_type = p.response.headers.content_type().map(str::to_owned); + hop.set_cookie |= p.response.headers.has_set_cookie(); + self.push_response(&p.request_id, n); + } + } + "Network.responseReceivedExtraInfo" => { + if let Ok(p) = parse_quiet::(&ev.params) { + let hops = self.hops.entry(p.request_id.clone()).or_default(); + if hops.is_empty() { + hops.push(Hop::default()); + } + // One per hop, in order: the first hop that has not had its own and + // does not contradict this one's status. + let n = hops + .iter() + .position(|h| { + !h.extra_seen + && (h.status.is_none() + || p.status_code.is_none() + || h.status == p.status_code) + }) + .unwrap_or(hops.len() - 1); + let hop = &mut hops[n]; + hop.extra_seen = true; + hop.status = hop.status.or(p.status_code); + if hop.content_type.is_none() { + hop.content_type = p.headers.content_type().map(str::to_owned); + } + hop.set_cookie |= p.headers.has_set_cookie(); + self.push_response(&p.request_id, n); + } + } + _ => {} + } + } + } + Flow::Continue + } + + async fn attached(&mut self, p: AttachedToTarget) { + let is_page = match p.target_info.kind.as_str() { + "page" => true, + "iframe" => false, + _ => { + // Workers and the rest: let them run, record nothing from them. + let _ = send( + &self.cdp, + "Runtime.runIfWaitingForDebugger", + &Empty {}, + Some(&p.session_id), + ) + .await; + return; + } + }; + let is_first = is_page && self.first_session.is_none(); + if is_page { + self.page_targets.insert(p.target_info.target_id.clone()); + if is_first { + self.first_session = Some(p.session_id.clone()); + } else { + self.capture.push(CaptureEvent::Popup); + } + } + self.sessions.insert( + p.session_id.clone(), + Sess { + target_id: p.target_info.target_id, + script_id: None, + }, + ); + // A page that vanishes while it is being set up is not an error for the session. + let _ = self.setup(&p.session_id).await; + let _ = send( + &self.cdp, + "Runtime.runIfWaitingForDebugger", + &Empty {}, + Some(&p.session_id), + ) + .await; + } + + fn is_first_main(&self, session: &str, parent: Option<&str>) -> bool { + self.first_session.as_deref() == Some(session) && parent.is_none() + } + + async fn navigated(&mut self, session: &str, p: FrameNavigated) { + let main = self.is_first_main(session, p.frame.parent_id.as_deref()); + if main { + self.first_main_frame = Some(p.frame.id.clone()); + if self.clock.is_none() { + let url = p.frame.url.as_str(); + if url == self.notice || url.starts_with("data:") || url == "about:blank" { + return; + } + self.start(session).await; + } + } + if self.clock.is_some() { + self.capture.push(CaptureEvent::Navigation { + t_ms: self.t_ms(), + frame: RawFrame { + main, + url: p.frame.url, + }, + }); + } + } + + /// The main frame left the notice page. + async fn start(&mut self, session: &str) { + self.clock = Some((Instant::now(), SystemTime::now())); + let version: Result = + self.cdp.call("Browser.getVersion", &Empty {}, None).await; + let metrics: Result = self + .cdp + .call("Page.getLayoutMetrics", &Empty {}, Some(session)) + .await; + let (product, user_agent) = version + .map(|v| (v.product, v.user_agent)) + .unwrap_or_default(); + let (width, height) = metrics.map_or((0, 0), |m| { + ( + m.css_visual_viewport.client_width.round() as u32, + m.css_visual_viewport.client_height.round() as u32, + ) + }); + self.capture.push(CaptureEvent::Started { + browser: BrowserInfo { + product, + user_agent, + viewport: Viewport { width, height }, + }, + listener_world: self.world, + }); + } + + fn binding(&mut self, session: &str, p: &BindingCalled) { + if p.name != BINDING { + return; + } + if let Ok(h) = parse_quiet::(&p.payload) { + if h.hello && h.nonce == self.nonce { + self.hello = true; + } + return; + } + let Ok(r) = parse_quiet::(&p.payload) else { + return; + }; + if !r.within_limits() || self.clock.is_none() || !self.sessions.contains_key(session) { + return; + } + let kind = match r.kind { + ReportKind::Input => PageEventKind::Input, + ReportKind::Change => PageEventKind::Change, + ReportKind::Click => PageEventKind::Click, + ReportKind::Submit => PageEventKind::Submit, + ReportKind::EnterKey => PageEventKind::EnterKey, + }; + let e = r.element; + let element = RawElement { + element_id: e.element_id, + document_id: r.document_id, + tag: e.tag, + input_type: e.input_type, + id: e.id, + name: e.name, + autocomplete: e.autocomplete, + placeholder: e.placeholder, + aria_label: e.aria_label, + is_password_now: e.is_password_now, + form: e.form.map(|f| RawForm { + index: f.index, + action: f.action, + method: f.method, + }), + selectors: e.selectors, + in_shadow_root: e.in_shadow_root, + }; + let value = match kind { + PageEventKind::Input | PageEventKind::Change => r.value, + _ => None, + }; + self.capture.push(CaptureEvent::Page { + t_ms: self.t_ms(), + kind, + frame: RawFrame { + main: r.is_top, + url: r.frame_url, + }, + element, + value, + }); + } + + fn push_response(&mut self, id: &str, n: usize) { + let Some(hop) = self.hops.get(id).and_then(|h| h.get(n)) else { + return; + }; + let Some(status) = hop.status else { return }; + self.capture.push(CaptureEvent::Response { + request_id: format!("{id}#{n}"), + status, + content_type: hop.content_type.clone(), + set_cookie: hop.set_cookie, + }); + } + + async fn request(&mut self, session: &str, p: RequestWillBeSent) { + let id = p.request_id.clone(); + if let Some(prev) = p.redirect_response { + let hops = self.hops.entry(id.clone()).or_default(); + if hops.is_empty() { + hops.push(Hop::default()); + } + let last = hops.len() - 1; + let hop = &mut hops[last]; + hop.status = Some(prev.status); + if hop.content_type.is_none() { + hop.content_type = prev.headers.content_type().map(str::to_owned); + } + hop.set_cookie |= prev.headers.has_set_cookie(); + hops.push(Hop::default()); + self.push_response(&id, last); + } else { + self.hops.insert(id.clone(), vec![Hop::default()]); + } + let n = self.hops.get(&id).map_or(0, |h| h.len() - 1); + if self.clock.is_none() { + return; + } + + let resource = match p.resource.as_deref() { + Some("Document") => ResourceKind::Document, + Some("XHR") => ResourceKind::Xhr, + Some("Fetch") => ResourceKind::Fetch, + _ => ResourceKind::Other, + }; + let main_frame_navigation = resource == ResourceKind::Document + && p.frame_id.is_some() + && p.frame_id == self.first_main_frame + && p.request_id == p.loader_id; + let body = match (&p.request.post_data, p.request.has_post_data) { + (Some(d), _) => RawBody::Bytes(Zeroizing::new(d.as_bytes().to_vec())), + (None, Some(true)) => { + POST_DATA_FETCHES.fetch_add(1, Ordering::Relaxed); + // A target that dies mid-call must not stall event handling for CALL_TIMEOUT. + let params = GetRequestPostData { request_id: &id }; + let call = self.cdp.call::<_, PostData>( + "Network.getRequestPostData", + ¶ms, + Some(session), + ); + match tokio::time::timeout(POST_DATA_WAIT, call).await { + Ok(Ok(d)) => RawBody::Bytes(Zeroizing::new(d.post_data.as_bytes().to_vec())), + Ok(Err(_)) | Err(_) => RawBody::Unavailable, + } + } + _ => RawBody::None, + }; + self.capture.push(CaptureEvent::Request { + request_id: format!("{id}#{n}"), + t_ms: self.t_ms(), + method: p.request.method.clone(), + url: p.request.url.clone(), + resource, + main_frame_navigation, + content_type: p.request.headers.content_type().map(str::to_owned), + authorization: p + .request + .headers + .authorization() + .map(|a| Zeroizing::new(a.as_bytes().to_vec())), + body, + frame_url: p.document_url.clone(), + }); + } + + fn finish(self, confirmed: bool) -> Outcome { + let Some((_, started)) = self.clock else { + return Outcome::NeverStarted; + }; + let info = EndInfo { + ended_by: if confirmed { + EndedBy::Confirmed + } else { + EndedBy::WindowClosed + }, + success_confirmed: confirmed, + started_at: utc_rfc3339(started), + ended_at: utc_rfc3339(SystemTime::now()), + }; + Outcome::Recorded(Box::new(self.capture.finish(info))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn times_are_rfc3339_utc() { + assert_eq!(utc_rfc3339(SystemTime::UNIX_EPOCH), "1970-01-01T00:00:00Z"); + let now = utc_rfc3339(SystemTime::now()); + assert!(now.ends_with('Z') && now.len() >= 20, "{now}"); + } + + #[test] + fn the_listener_script_is_embedded_and_names_the_binding() { + assert!(LISTENER_JS.contains(BINDING)); + } +} diff --git a/crates/kharon-teach/src/site.rs b/crates/kharon-teach/src/site.rs new file mode 100644 index 0000000..ded3e60 --- /dev/null +++ b/crates/kharon-teach/src/site.rs @@ -0,0 +1,81 @@ +//! Which site a host belongs to, by the Public Suffix List, and which hosts serve CAPTCHAs. + +use std::net::IpAddr; +use std::sync::OnceLock; + +use publicsuffix::{List, Psl}; + +/// The one vendored snapshot, shared with `kharond` as a data file (not as a crate). +const PSL_SNAPSHOT: &[u8] = include_bytes!("../../kharond/data/public_suffix_list.dat"); + +/// Hosts that serve a CAPTCHA. `google.com/recaptcha` is a path, so `capture` checks that case. +const CAPTCHA_HOSTS: [&str; 5] = [ + "recaptcha.net", + "hcaptcha.com", + "challenges.cloudflare.com", + "arkoselabs.com", + "funcaptcha.com", +]; + +fn psl() -> Option<&'static List> { + static PSL: OnceLock> = OnceLock::new(); + PSL.get_or_init(|| List::from_bytes(PSL_SNAPSHOT).ok()) + .as_ref() +} + +/// The registrable domain of `host` (its eTLD+1), lowercased, without a trailing dot. An IP +/// address, a single-label host such as `localhost`, or a host the list has no answer for is +/// returned lowercased and otherwise unchanged. +pub fn registrable_domain(host: &str) -> String { + let host = host.strip_suffix('.').unwrap_or(host).to_ascii_lowercase(); + let bare = host + .strip_prefix('[') + .and_then(|h| h.strip_suffix(']')) + .unwrap_or(&host); + if bare.parse::().is_ok() { + return host; + } + psl() + .and_then(|list| list.domain(host.as_bytes())) + .map_or_else( + || host.clone(), + |d| String::from_utf8_lossy(d.as_bytes()).into_owned(), + ) +} + +/// Whether `host`, or any parent domain of it, is a known CAPTCHA host. +pub fn is_captcha_host(host: &str) -> bool { + let host = host.trim_end_matches('.').to_ascii_lowercase(); + CAPTCHA_HOSTS + .iter() + .any(|h| host == *h || host.ends_with(&format!(".{h}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn registrable_domains() { + assert_eq!(registrable_domain("login.ex.co.uk"), "ex.co.uk"); + assert_eq!(registrable_domain("login.ex.test"), "ex.test"); + assert_eq!(registrable_domain("ex.test"), "ex.test"); + assert_eq!(registrable_domain("127.0.0.1"), "127.0.0.1"); + assert_eq!(registrable_domain("localhost"), "localhost"); + assert_eq!(registrable_domain("[::1]"), "[::1]"); + assert_eq!(registrable_domain("Login.Ex.Co.UK."), "ex.co.uk"); + assert_eq!(registrable_domain("ex.test."), "ex.test"); + assert_eq!(registrable_domain("LOCALHOST."), "localhost"); + } + + #[test] + fn captcha_hosts_match_the_host_and_its_subdomains() { + assert!(is_captcha_host("hcaptcha.com")); + assert!(is_captcha_host("newassets.hcaptcha.com")); + assert!(is_captcha_host("challenges.cloudflare.com")); + assert!(is_captcha_host("www.recaptcha.net")); + assert!(!is_captcha_host("cloudflare.com")); + assert!(!is_captcha_host("nothcaptcha.com")); + assert!(!is_captcha_host("www.google.com")); + } +} diff --git a/crates/kharon-teach/src/strip.rs b/crates/kharon-teach/src/strip.rs new file mode 100644 index 0000000..41cc115 --- /dev/null +++ b/crates/kharon-teach/src/strip.rs @@ -0,0 +1,1184 @@ +//! Stripping: a URL or an `Authorization` header reduced to structure only. +//! +//! Everything this module KEEPS (origin, path, query keys, scheme) is tested for typed values as a +//! whole string, never piece by piece, and with ASCII letter case ignored, because a site may +//! lowercase what was typed and a typed value may itself hold a separator. + +use std::ops::Range; + +use base64ct::{Base64, Base64Unpadded, Base64Url, Base64UrlUnpadded, Encoding as B64}; +use kharon_recording::{AuthShape, Encoding, MarkerHit, StrippedUrl}; +use url::{Position, Url}; +use zeroize::Zeroizing; + +use crate::marker::{ + LOW_CONFIDENCE_BELOW, Marker, found_as_whole_token, pair_found_as_whole_token, percent_decode, + search, search_pair, +}; + +/// Written in place of a name (key, part name, host labels) that held a typed value. +pub const TYPED: &str = "{typed}"; +/// Written in place of a path segment that is an identifier or held a typed value. +pub const ID: &str = "{id}"; +/// Written in place of a name (key, part name, query key) that is not a typed value but does not +/// look like a name either: it is long, or holds characters a name never holds, or is an +/// identifier. Such text is more likely a value, so it is not kept. +pub const OTHER: &str = "{other}"; + +/// The longest key a kept name may have. +const MAX_NAME_CHARS: usize = 64; +/// The longest query piece with no `=` that is kept as a key. +const MAX_BARE_KEY_CHARS: usize = 32; + +/// Whether `s` looks like the name of a field: at most 64 characters, each one of +/// `A-Za-z0-9_.:-[]`, and not an identifier (all digits, a UUID, or 16 or more hex characters). +pub(crate) fn looks_like_a_name(s: &str) -> bool { + s.chars().count() <= MAX_NAME_CHARS + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | ':' | '-' | '[' | ']')) + && !segment_is_id(s) +} + +/// URL schemes that are kept; any other is written `{other}`. +const URL_SCHEMES: [&str; 11] = [ + "http", + "https", + "ws", + "wss", + "data", + "blob", + "about", + "file", + "chrome", + "chrome-extension", + "devtools", +]; + +/// Authorization schemes that are kept (compared lowercased); any other is written `{other}`. +const AUTH_SCHEMES: [&str; 12] = [ + "basic", + "bearer", + "digest", + "negotiate", + "ntlm", + "hoba", + "mutual", + "token", + "apikey", + "oauth", + "hawk", + "aws4-hmac-sha256", +]; + +pub struct Stripped { + pub value: T, + pub query_values_removed: u64, + pub path_segments_replaced: u64, +} + +/// Every hit of every marker in `haystack`, plus the pair, without repeats of (role, encoding). +pub(crate) fn hits_in( + haystack: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> Vec { + let mut out = Vec::new(); + for m in markers { + out.extend(search(haystack, m)); + } + if let Some((u, p)) = pair { + out.extend(search_pair(haystack, u, p)); + } + dedup_hits(out) +} + +/// One hit per (role, encoding), in first-seen order. When two share a key the result is +/// `low_confidence: false` if either is. +pub(crate) fn dedup_hits(hits: Vec) -> Vec { + let mut out: Vec = Vec::with_capacity(hits.len()); + for h in hits { + match out + .iter_mut() + .find(|k| k.role == h.role && k.encoding == h.encoding) + { + Some(known) => known.low_confidence &= h.low_confidence, + None => out.push(h), + } + } + out +} + +/// Whether any marker (or the pair) occurs in `haystack` under any encoding. +pub(crate) fn any_hit( + haystack: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> bool { + !hits_in(haystack, markers, pair).is_empty() +} + +/// `bytes` as text of the SAME byte length: every invalid UTF-8 byte becomes `?`, so byte +/// offsets in the text are offsets in `bytes`. +fn text_same_len(bytes: &[u8]) -> Zeroizing { + let mut out = Zeroizing::new(String::with_capacity(bytes.len())); + for chunk in bytes.utf8_chunks() { + out.push_str(chunk.valid()); + out.extend(std::iter::repeat_n('?', chunk.invalid().len())); + } + out +} + +/// A marker shorter than this many characters matches kept text only as a whole token. +fn is_short(m: &Marker) -> bool { + m.len() < LOW_CONFIDENCE_BELOW +} + +/// Whether `range` in `text` is bounded on each side by the text's edge or by a character that +/// is not ASCII alphanumeric. +fn is_whole_token(text: &str, range: &Range) -> bool { + let not_alnum = |c: char| !c.is_ascii_alphanumeric(); + text[..range.start] + .chars() + .next_back() + .is_none_or(not_alnum) + && text[range.end..].chars().next().is_none_or(not_alnum) +} + +/// Byte ranges of every occurrence (overlapping ones too) of every marker in `text`, ignoring +/// ASCII letter case. A marker shorter than `LOW_CONFIDENCE_BELOW` characters counts only where +/// it is a whole token (see `is_whole_token`): `on` is not found inside `session`. +fn occurrences(text: &str, markers: &[Marker]) -> Vec> { + let mut out = Vec::new(); + for m in markers { + let mut from = 0; + while let Some(r) = text + .get(from..) + .and_then(|rest| m.find_ignore_ascii_case(rest)) + { + let (start, end) = (from + r.start, from + r.end); + if !is_short(m) || is_whole_token(text, &(start..end)) { + out.push(start..end); + } + // Resume one character after the start so overlapping occurrences are found. + from = start + text[start..].chars().next().map_or(1, char::len_utf8); + } + } + out +} + +/// For each span, whether any range overlaps it. +fn touched(ranges: &[Range], spans: &[Range]) -> Vec { + spans + .iter() + .map(|s| ranges.iter().any(|r| r.start < s.end && r.end > s.start)) + .collect() +} + +/// The part of each range inside `start..end`, in coordinates relative to `start`. +fn rebase(ranges: &[Range], start: usize, end: usize) -> Vec> { + ranges + .iter() + .filter_map(|r| { + let (s, e) = (r.start.max(start), r.end.min(end)); + (s < e).then(|| s - start..e - start) + }) + .collect() +} + +/// `search` over kept text. A marker shorter than `LOW_CONFIDENCE_BELOW` characters (and a pair +/// of that combined length) is checked in ALL its forms too, but only as a whole token: the form +/// must be bounded by characters that are not ASCII alphanumeric, or by the text's ends, because +/// its encoded forms would otherwise match inside ordinary words. +fn kept_hit(haystack: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> bool { + markers.iter().any(|m| { + if is_short(m) { + found_as_whole_token(haystack, m) + } else { + !search(haystack, m).is_empty() + } + }) || pair.is_some_and(|(u, p)| { + if u.len().saturating_add(p.len()) >= LOW_CONFIDENCE_BELOW { + !search_pair(haystack, u, p).is_empty() + } else { + pair_found_as_whole_token(haystack, u, p) + } + }) +} + +/// Whether kept text `bytes` holds a typed value: `search` finds any hit (any encoding, the pair +/// too) for a marker of `LOW_CONFIDENCE_BELOW` characters or more, or the value occurs ignoring +/// ASCII case in the text or in its percent-decoded forms (a shorter marker only as a whole token). +pub(crate) fn holds_typed( + bytes: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> bool { + if kept_hit(bytes, markers, pair) || !occurrences(&text_same_len(bytes), markers).is_empty() { + return true; + } + [false, true].into_iter().any(|plus| { + let decoded = percent_decode(bytes, plus); + decoded.as_slice() != bytes && !occurrences(&text_same_len(&decoded), markers).is_empty() + }) +} + +/// Origin, path and query keys of `url`. A path segment becomes "{id}" when it is all ASCII digits, +/// a UUID, 16 or more hex characters, or any match of a marker (see below) touches it. +/// +/// Userinfo and fragment are dropped. A query key or host label holding a typed value is written +/// as "{typed}". Matching is over WHOLE strings: the path, the query and the host each, and also +/// the whole URL as the parser serialises it (`Url::as_str`, with component offsets from +/// `url::Position`) and a percent-decoded copy built piece by piece, so a typed value holding +/// `?`, `#`, `@` or `/` that the parser split into components is still found. All of them ignore +/// ASCII case, and the path also reads `/` as `\` (the parser rewrites it). Every segment, key or +/// label a match touches is replaced. A combined credential (`pair`) found in a segment or key +/// replaces it. A marker shorter than `LOW_CONFIDENCE_BELOW` characters matches only as a whole +/// token (see `occurrences`). +/// +/// A URL that cannot be a base (`data:`, `blob:`, `about:`, `mailto:`) keeps only its scheme. +/// The scheme is kept only when it is in a short known list, else it is `{other}`. +/// Known limit: the port is not tested. +pub fn strip_url( + url: &str, + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> Option> { + let parsed = Url::parse(url).ok()?; + let scheme = if URL_SCHEMES.contains(&parsed.scheme()) { + parsed.scheme() + } else { + OTHER + }; + if parsed.cannot_be_a_base() { + return Some(Stripped { + value: StrippedUrl { + origin: format!("{scheme}:"), + path: String::new(), + query_keys: Vec::new(), + }, + query_values_removed: 0, + path_segments_replaced: 0, + }); + } + + // Component offsets in the serialisation. + let ser = parsed.as_str(); + let at = |p: Position| ser.len() - parsed[p..].len(); + let (host_start, host_end) = (at(Position::BeforeHost), at(Position::AfterHost)); + let (path_start, path_end) = (at(Position::BeforePath), at(Position::AfterPath)); + let query = parsed.query(); + let query_start = path_end + 1; + + let path_parts = PathParts::new(parsed.path()); + let query_parts = QueryParts::new(query.unwrap_or("")); + + // The decoded whole URL: the pieces decoded one by one, in order, with the delimiters between + // them, so offsets of the host, path and query inside it are known. Its capacity covers the + // text, so it is never reallocated (the copy is zeroized on drop). + let host = &ser[host_start..host_end]; + let decode = |s: &str| text_same_len(&percent_decode(s.as_bytes(), false)); + let mut whole = Zeroizing::new(String::with_capacity(ser.len() + 2)); + whole.push_str(&decode(&ser[..host_start])); + let d_host = whole.len()..whole.len() + host.len(); + whole.push_str(host); + whole.push_str(&decode(&ser[host_end..path_start])); + let d_path = whole.len()..whole.len() + path_parts.decoded_path.len(); + whole.push_str(&path_parts.decoded_path); + let mut d_query = whole.len()..whole.len(); + if query.is_some() { + whole.push('?'); + d_query = whole.len()..whole.len() + query_parts.decoded_query.len(); + whole.push_str(&query_parts.decoded_query); + } + if let Some(fragment) = parsed.fragment() { + whole.push('#'); + whole.push_str(&decode(fragment)); + } + + let ranges_in = |text: &str| { + let mut ranges = occurrences(text, markers); + ranges.extend(occurrences( + &Zeroizing::new(text.replace('/', "\\")), + markers, + )); + ranges + }; + let raw_all = ranges_in(ser); + let dec_all = ranges_in(&whole); + + let host_ranges: Vec> = rebase(&raw_all, host_start, host_end) + .into_iter() + .chain(rebase(&dec_all, d_host.start, d_host.end)) + .collect(); + let origin = match parsed.host_str() { + Some(host) => { + let port = parsed.port().map(|p| format!(":{p}")).unwrap_or_default(); + format!( + "{scheme}://{}{port}", + strip_host(host, markers, pair, &host_ranges) + ) + } + None => format!("{scheme}:"), + }; + let (path, path_segments_replaced) = strip_path( + &path_parts, + markers, + pair, + &rebase(&raw_all, path_start, path_end), + &rebase(&dec_all, d_path.start, d_path.end), + ); + let (query_keys, query_values_removed) = strip_query( + &query_parts, + markers, + pair, + &rebase( + &raw_all, + query_start, + query_start + query.map_or(0, str::len), + ), + &rebase(&dec_all, d_query.start, d_query.end), + ); + Some(Stripped { + value: StrippedUrl { + origin, + path, + query_keys, + }, + query_values_removed, + path_segments_replaced, + }) +} + +/// Replaces host labels that hold a typed value. Every occurrence of every marker is located +/// (ASCII case-insensitively: the parser lowercases hosts) and every label an occurrence touches +/// is replaced; `extra` holds matches found over the whole URL, in host coordinates. Adjacent +/// replaced labels become ONE `{typed}` label, so a value that spans labels (`kh.throwaway7`) +/// leaves no part of itself behind. A label that holds a typed value in an encoded form (hex, +/// say; base64 cannot survive the lowercasing), or the combined credential, is replaced too, by +/// `kept_hit`, the same as a path segment. +fn strip_host( + host: &str, + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, + extra: &[Range], +) -> String { + let mut ranges = occurrences(host, markers); + ranges.extend_from_slice(extra); + let mut spans = Vec::new(); + let mut offset = 0; + for label in host.split('.') { + spans.push(offset..offset + label.len()); + offset += label.len() + 1; + } + let mut typed = touched(&ranges, &spans); + for (t, label) in typed.iter_mut().zip(host.split('.')) { + *t |= kept_hit(label.as_bytes(), markers, pair); + } + // An internationalised label arrives as punycode (`xn--jos-dma` for `josé`), which holds + // none of the typed value's bytes, so the host is compared again in its Unicode form, + // lowercased. Known limit: a value typed in a decomposed form (NFD) is not recognised, + // because the host's Unicode form is always composed (NFC). + if host.split('.').any(|label| label.starts_with("xn--")) { + let (unicode, result) = idna::domain_to_unicode(host); + let unicode = Zeroizing::new(unicode); + if result.is_err() || unicode.split('.').count() != spans.len() { + // The Unicode form cannot be lined up with the labels: fail closed. + for (t, label) in typed.iter_mut().zip(host.split('.')) { + *t |= label.starts_with("xn--"); + } + } else { + let lowered_labels: Vec> = unicode + .split('.') + .map(|label| Zeroizing::new(label.to_lowercase())) + .collect(); + let mut lowered = Zeroizing::new(String::with_capacity(unicode.len())); + for (i, label) in lowered_labels.iter().enumerate() { + if i > 0 { + lowered.push('.'); + } + lowered.push_str(label); + } + let lowered_spans = spans_of(lowered_labels.iter().map(|label| label.len())); + let mut lowered_ranges = occurrences(&lowered, markers); + for m in markers { + lowered_ranges.extend(m.find_lowercased_in(&lowered)); + } + let hit = touched(&lowered_ranges, &lowered_spans); + for ((t, hit), label) in typed.iter_mut().zip(hit).zip(&lowered_labels) { + *t |= hit || kept_hit(label.as_bytes(), markers, pair); + } + } + } + let mut labels: Vec<&str> = Vec::new(); + let mut previous_typed = false; + for (label, is_typed) in host.split('.').zip(typed) { + if is_typed { + if !previous_typed { + labels.push(TYPED); + } + } else { + labels.push(label); + } + previous_typed = is_typed; + } + labels.join(".") +} + +/// The pieces of a URL path that matching needs: raw and decoded segments with their spans. +struct PathParts<'a> { + path: &'a str, + segments: Vec<&'a str>, + decoded: Vec>>, + raw_spans: Vec>, + decoded_spans: Vec>, + /// The decoded segments joined with `/`. + decoded_path: Zeroizing, +} + +fn spans_of(lens: impl Iterator) -> Vec> { + let mut offset = 0; + lens.map(|len| { + let span = offset..offset + len; + offset += len + 1; + span + }) + .collect() +} + +impl<'a> PathParts<'a> { + fn new(path: &'a str) -> Self { + let segments: Vec<&str> = path.split('/').collect(); + let decoded: Vec>> = segments + .iter() + .map(|s| percent_decode(s.as_bytes(), false)) + .collect(); + let texts: Vec> = decoded.iter().map(|d| text_same_len(d)).collect(); + let mut decoded_path = Zeroizing::new(String::with_capacity(path.len())); + for (i, t) in texts.iter().enumerate() { + if i > 0 { + decoded_path.push('/'); + } + decoded_path.push_str(t); + } + Self { + path, + raw_spans: spans_of(segments.iter().map(|s| s.len())), + decoded_spans: spans_of(texts.iter().map(|t| t.len())), + segments, + decoded, + decoded_path, + } + } +} + +/// Segments are replaced when they look like an identifier or a match touches them. `raw_extra` +/// and `dec_extra` are matches found over the whole URL, in raw and decoded path coordinates. +fn strip_path( + parts: &PathParts, + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, + raw_extra: &[Range], + dec_extra: &[Range], +) -> (String, u64) { + let segments = &parts.segments; + let mut typed = vec![false; segments.len()]; + let mut mark = |hit: Vec| typed.iter_mut().zip(hit).for_each(|(t, h)| *t |= h); + + mark(touched(raw_extra, &parts.raw_spans)); + mark(touched(dec_extra, &parts.decoded_spans)); + // The raw and decoded whole path, each also with `/` read as `\` (same byte length, so the + // same spans apply). + for (text, spans) in [ + (parts.path, &parts.raw_spans), + (parts.decoded_path.as_str(), &parts.decoded_spans), + ] { + let backslashed = Zeroizing::new(text.replace('/', "\\")); + for variant in [text, backslashed.as_str()] { + mark(touched(&occurrences(variant, markers), spans)); + } + } + for (i, seg) in segments.iter().enumerate() { + typed[i] |= + kept_hit(seg.as_bytes(), markers, pair) || kept_hit(&parts.decoded[i], markers, pair); + } + // A hit over the whole path that no segment explains (an encoded value that spans a `/`): + // every non-empty segment goes. + if !typed.iter().any(|t| *t) + && (kept_hit(parts.path.as_bytes(), markers, pair) + || kept_hit(parts.decoded_path.as_bytes(), markers, pair)) + { + typed = segments.iter().map(|s| !s.is_empty()).collect(); + } + + let mut replaced = 0u64; + let kept: Vec<&str> = segments + .iter() + .zip(typed) + .map(|(seg, is_typed)| { + if is_typed || segment_is_id(seg) { + replaced += 1; + ID + } else { + seg + } + }) + .collect(); + (kept.join("/"), replaced) +} + +fn segment_is_id(seg: &str) -> bool { + !seg.is_empty() + && (seg.bytes().all(|b| b.is_ascii_digit()) + || (seg.len() >= 16 && seg.bytes().all(|b| b.is_ascii_hexdigit())) + || is_uuid(seg)) +} + +fn is_uuid(seg: &str) -> bool { + let b = seg.as_bytes(); + b.len() == 36 + && b.iter().enumerate().all(|(i, c)| match i { + 8 | 13 | 18 | 23 => *c == b'-', + _ => c.is_ascii_hexdigit(), + }) +} + +struct QueryPiece<'a> { + raw_key: &'a str, + /// The piece had an `=`. + has_value: bool, + raw_span: Range, + key: Zeroizing>, + decoded_span: Range, +} + +/// The pieces of a query string that matching needs. +struct QueryParts<'a> { + query: &'a str, + pieces: Vec>, + /// `key=value` pieces decoded (`+` as space, `%XX`) and joined with `&`. + decoded_query: Zeroizing, +} + +impl<'a> QueryParts<'a> { + fn new(query: &'a str) -> Self { + let mut pieces = Vec::new(); + let mut decoded_query = Zeroizing::new(String::with_capacity(query.len())); + let mut offset = 0; + for piece in query.split('&') { + let start = offset; + offset += piece.len() + 1; + if piece.is_empty() { + continue; + } + let (raw_key, value) = match piece.split_once('=') { + Some((k, v)) => (k, Some(v)), + None => (piece, None), + }; + let key = percent_decode(raw_key.as_bytes(), true); + if !decoded_query.is_empty() { + decoded_query.push('&'); + } + let decoded_start = decoded_query.len(); + decoded_query.push_str(&text_same_len(&key)); + let decoded_span = decoded_start..decoded_query.len(); + if let Some(v) = value { + decoded_query.push('='); + decoded_query.push_str(&text_same_len(&percent_decode(v.as_bytes(), true))); + } + pieces.push(QueryPiece { + raw_key, + has_value: value.is_some(), + raw_span: start..start + raw_key.len(), + key, + decoded_span, + }); + } + Self { + query, + pieces, + decoded_query, + } + } +} + +/// Query keys, in order, with a key a match touches written `{typed}` and a key that does not +/// look like a name written `{other}` (a piece with no `=` must also be at most 32 characters). The raw query string and a +/// decoded copy (`+` as space, `%XX`) are each searched whole, as are `raw_extra` and `dec_extra` +/// (matches found over the whole URL, in raw and decoded query coordinates); a match touching a +/// key's bytes (separators included) marks the key. A match inside a value alone costs nothing. +fn strip_query( + parts: &QueryParts, + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, + raw_extra: &[Range], + dec_extra: &[Range], +) -> (Vec, u64) { + let raw_spans: Vec> = parts.pieces.iter().map(|p| p.raw_span.clone()).collect(); + let decoded_spans: Vec> = parts + .pieces + .iter() + .map(|p| p.decoded_span.clone()) + .collect(); + let mut raw_ranges = occurrences(parts.query, markers); + raw_ranges.extend_from_slice(raw_extra); + let mut dec_ranges = occurrences(&parts.decoded_query, markers); + dec_ranges.extend_from_slice(dec_extra); + let by_raw = touched(&raw_ranges, &raw_spans); + let by_decoded = touched(&dec_ranges, &decoded_spans); + let keys = parts + .pieces + .iter() + .enumerate() + .map(|(i, p)| { + if by_raw[i] + || by_decoded[i] + || kept_hit(p.raw_key.as_bytes(), markers, pair) + || kept_hit(&p.key, markers, pair) + { + TYPED.to_owned() + } else { + let key = String::from_utf8_lossy(&p.key); + let bare_too_long = !p.has_value && key.chars().count() > MAX_BARE_KEY_CHARS; + if looks_like_a_name(&key) && !bare_too_long { + key.into_owned() + } else { + OTHER.to_owned() + } + } + }) + .collect(); + (keys, u64::try_from(parts.pieces.len()).unwrap_or(u64::MAX)) +} + +/// scheme = first token, lowercased. For `basic`, the credentials are base64-decoded and the decoded bytes +/// are searched; a marker found there is reported with `Encoding::Base64`. Other schemes are searched as sent. +/// When `pair` is given the value is also searched with `search_pair`. No (role, encoding) is listed twice. +/// +/// The value as sent is always searched too. The scheme is kept only when it is one of a short +/// known list (`AUTH_SCHEMES`); a scheme holding a typed value is `{typed}`; anything else, +/// including a header with no space (a bare credential), is `{other}`. +pub fn reduce_authorization( + header_value: &[u8], + markers: &[Marker], + pair: Option<(&Marker, &Marker)>, +) -> AuthShape { + let (scheme_bytes, credentials) = match header_value.iter().position(|b| *b == b' ') { + Some(i) => (&header_value[..i], &header_value[i + 1..]), + None => (header_value, &header_value[..0]), + }; + let has_credentials = scheme_bytes.len() < header_value.len(); + let mut hits = hits_in(header_value, markers, pair); + + let scheme_lower = scheme_bytes.to_ascii_lowercase(); + if scheme_lower == b"basic" + && let Some(decoded) = decode_basic(credentials) + { + let decoded_hits = hits_in(&decoded, markers, None); + hits.extend(decoded_hits.into_iter().map(|h| MarkerHit { + encoding: Encoding::Base64, + ..h + })); + } + + let known = std::str::from_utf8(&scheme_lower) + .ok() + .filter(|s| has_credentials && AUTH_SCHEMES.contains(s)); + let scheme = if holds_typed(scheme_bytes, markers, pair) { + TYPED.to_owned() + } else { + known.map_or_else(|| OTHER.to_owned(), str::to_owned) + }; + + AuthShape { + scheme, + markers: dedup_hits(hits), + } +} + +/// The decoded `user:password` bytes, or `None` when the credentials are not base64. +fn decode_basic(credentials: &[u8]) -> Option>> { + let text = std::str::from_utf8(credentials).ok()?.trim(); + type Decoder = fn(&str) -> Option>; + let decoders: [Decoder; 4] = [ + |t| Base64::decode_vec(t).ok(), + |t| Base64Unpadded::decode_vec(t).ok(), + |t| Base64Url::decode_vec(t).ok(), + |t| Base64UrlUnpadded::decode_vec(t).ok(), + ]; + decoders + .iter() + .find_map(|decode| decode(text)) + .map(Zeroizing::new) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::{assert_holds_no_typed_value, pw, user}; + use kharon_recording::Role; + + fn hit(role: Role, encoding: Encoding) -> MarkerHit { + MarkerHit { + role, + encoding, + low_confidence: false, + } + } + + #[test] + fn keeps_origin_path_and_query_keys() { + let s = strip_url( + "https://ex.test/login/step?next=%2Fhome&lang=en#frag", + &[], + None, + ) + .unwrap(); + assert_eq!( + s.value, + StrippedUrl { + origin: "https://ex.test".into(), + path: "/login/step".into(), + query_keys: vec!["next".into(), "lang".into()] + } + ); + assert_eq!(s.query_values_removed, 2); + assert_eq!(s.path_segments_replaced, 0); + } + + #[test] + fn origin_has_no_userinfo_keeps_a_port_and_duplicate_keys() { + let s = strip_url("http://bob:s3cret@ex.test:8080/a?k=1&k=2", &[], None).unwrap(); + assert_eq!(s.value.origin, "http://ex.test:8080"); + assert_eq!(s.value.query_keys, ["k", "k"]); + } + + #[test] + fn replaces_id_like_segments() { + for (path, want, n) in [ + ("/users/12345/home", "/users/{id}/home", 1), + ("/s/3f2b8c1e-9a4d-4c1b-8e2f-0a1b2c3d4e5f", "/s/{id}", 1), + ("/s/3F2B8C1E-9A4D-4C1B-8E2F-0A1B2C3D4E5F", "/s/{id}", 1), + ("/t/0123456789abcdef0123", "/t/{id}", 1), + ("/v2/login", "/v2/login", 0), + ("/a/b/", "/a/b/", 0), + ] { + let s = strip_url(&format!("https://ex.test{path}"), &[], None).unwrap(); + assert_eq!(s.value.path, want); + assert_eq!(s.path_segments_replaced, n); + } + } + + #[test] + fn replaces_a_segment_that_contains_a_marker() { + let m = [user("kh.throwaway7")]; + let s = strip_url("https://ex.test/u/kh.throwaway7/dash", &m, None).unwrap(); + assert_eq!(s.value.path, "/u/{id}/dash"); + let s = strip_url("https://ex.test/u/kh%2Ethrow%61way7/dash", &m, None).unwrap(); + assert_eq!(s.value.path, "/u/{id}/dash"); + assert_eq!(s.path_segments_replaced, 1); + } + + #[test] + fn unparseable_url_is_none() { + assert!(strip_url("not a url", &[], None).is_none()); + } + + #[test] + fn a_query_key_that_holds_a_typed_value_is_not_kept() { + let m = [user("kh.throwaway7")]; + let s = strip_url("https://ex.test/x?kh.throwaway7=1&ok=2", &m, None).unwrap(); + assert_eq!(s.value.query_keys, ["{typed}", "ok"]); + let s = strip_url("https://ex.test/x?kh%2Ethrowaway7=1", &m, None).unwrap(); + assert_eq!(s.value.query_keys, ["{typed}"]); + } + + #[test] + fn a_host_label_that_is_an_encoded_typed_value_is_replaced() { + let m = [user("kh.throwaway7")]; + let label = hex::encode("kh.throwaway7"); + let s = strip_url(&format!("https://{label}.ex.test/login"), &m, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + } + + #[test] + fn a_host_label_that_holds_a_typed_value_is_replaced() { + let m = [user("kh.throwaway7")]; + // The value spans two labels: the whole run of labels the match covers is one `{typed}`. + let s = strip_url("https://kh.throwaway7.ex.test/login", &m, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + // Case-insensitive, and a partial label is replaced whole. + let m = [user("Kh.Throwaway7")]; + let s = strip_url("https://x-kh.throwaway7-y.ex.test:81/", &m, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test:81"); + // Every occurrence. + let m = [user("bob12")]; + let s = strip_url("https://bob12.ex.bob12.test/", &m, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.{typed}.test"); + } + + #[test] + fn an_idn_host_label_holding_a_typed_value_is_replaced() { + // The parser writes the host as punycode: `josé` is `xn--jos-dma`. + let m = [user("josé")]; + let s = strip_url("https://xn--jos-dma.ex.test/login", &m, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + // Typed in another case: the host's Unicode form is lowercase. + let m = [user("JOSÉ")]; + let s = strip_url("https://xn--jos-dma.ex.test/login", &m, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + // A value that spans two internationalised labels. + let m = [user("josé.ñu")]; + let s = strip_url("https://josé.ñu.ex.test/login", &m, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + // An internationalised label that holds no typed value is kept as it is. + let s = strip_url("https://xn--bcher-kva.ex.test/login", &m, None).unwrap(); + assert_eq!(s.value.origin, "https://xn--bcher-kva.ex.test"); + } + + #[test] + fn basic_auth_is_reduced_to_scheme_and_hits() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let a = reduce_authorization( + format!("Basic {b64}").as_bytes(), + &[user("kh.throwaway7"), pw("Tr0ub4dor-9x")], + None, + ); + assert_eq!(a.scheme, "basic"); + assert_eq!( + a.markers, + [ + hit(Role::Username, Encoding::Base64), + hit(Role::Password, Encoding::Base64) + ] + ); + } + + #[test] + fn basic_auth_with_the_pair_lists_each_hit_once() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let (u, p) = (user("kh.throwaway7"), pw("Tr0ub4dor-9x")); + let a = reduce_authorization( + format!("Basic {b64}").as_bytes(), + &[user("kh.throwaway7"), pw("Tr0ub4dor-9x")], + Some((&u, &p)), + ); + assert_eq!( + a.markers, + [ + hit(Role::Username, Encoding::Base64), + hit(Role::Password, Encoding::Base64) + ] + ); + } + + #[test] + fn bearer_keeps_no_token() { + let a = reduce_authorization( + b"Bearer eyJhbGciOiJIUzI1NiJ9.abc.def", + &[pw("Tr0ub4dor-9x")], + None, + ); + assert_eq!( + a, + AuthShape { + scheme: "bearer".into(), + markers: vec![] + } + ); + assert!(!serde_json::to_string(&a).unwrap().contains("eyJ")); + } + + #[test] + fn a_typed_password_sent_as_a_bearer_token_is_a_hit() { + let a = reduce_authorization(b"Bearer Tr0ub4dor-9x", &[pw("Tr0ub4dor-9x")], None); + assert_eq!(a.markers, [hit(Role::Password, Encoding::AsTyped)]); + } + + #[test] + fn no_space_is_a_scheme_with_no_credentials() { + let a = reduce_authorization(b"Negotiate", &[], None); + assert_eq!( + a, + AuthShape { + scheme: "{other}".into(), + markers: vec![] + } + ); + // A bare typed value never becomes the scheme. + let a = reduce_authorization(b"Tr0ub4dor-9x", &[pw("Tr0ub4dor-9x")], None); + assert_eq!(a.scheme, "{typed}"); + assert_eq!(a.markers, [hit(Role::Password, Encoding::AsTyped)]); + // Nor does anything that is not a plain token. + let a = reduce_authorization(b"s3cr3t/token+value=", &[], None); + assert_eq!(a.scheme, "{other}"); + } + + #[test] + fn malformed_basic_has_no_markers_but_is_searched_as_sent() { + let a = reduce_authorization(b"Basic !!!not-base64!!!", &[pw("Tr0ub4dor-9x")], None); + assert_eq!( + a, + AuthShape { + scheme: "basic".into(), + markers: vec![] + } + ); + let a = reduce_authorization(b"Basic Tr0ub4dor-9x", &[pw("Tr0ub4dor-9x")], None); + assert_eq!(a.markers, [hit(Role::Password, Encoding::AsTyped)]); + } + + #[test] + fn a_typed_value_holding_a_slash_is_not_kept_in_the_path() { + let m = [pw("Tr0ub4/dor")]; + let s = strip_url("https://ex.test/a/Tr0ub4/dor", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}/{id}"); + assert_eq!(s.path_segments_replaced, 2); + let s = strip_url("https://ex.test/a/Tr0ub4%2Fdor/z", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}/z"); + // The parser rewrites a backslash in the path to a slash. + let m = [pw("Tr0ub4\\dor")]; + let s = strip_url("https://ex.test/a/Tr0ub4\\dor/z", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}/{id}/z"); + } + + #[test] + fn a_site_cased_value_is_not_kept_in_the_path_or_query() { + let m = [user("Bob.Smith@Ex.Test")]; + let s = strip_url( + "https://ex.test/u/bob.smith@ex.test/x?BOB.SMITH%40EX.TEST=1&ok=2", + &m, + None, + ) + .unwrap(); + assert_eq!(s.value.path, "/u/{id}/x"); + assert_eq!(s.value.query_keys, ["{typed}", "ok"]); + } + + #[test] + fn a_separator_inside_a_typed_value_does_not_leave_a_key() { + let m = [pw("pa&ss")]; + let s = strip_url("https://ex.test/x?a=pa&ss=1&b=2", &m, None).unwrap(); + assert_eq!(s.value.query_keys, ["a", "{typed}", "b"]); + let m = [pw("pa=ss")]; + let s = strip_url("https://ex.test/x?pa=ss&b=2", &m, None).unwrap(); + assert_eq!(s.value.query_keys, ["{typed}", "b"]); + // A typed value sitting in a query VALUE does not cost its key. + let m = [pw("Tr0ub4dor-9x")]; + let s = strip_url("https://ex.test/x?p=Tr0ub4dor-9x", &m, None).unwrap(); + assert_eq!(s.value.query_keys, ["p"]); + } + + #[test] + fn the_combined_credential_in_a_path_or_key_is_not_kept() { + let b64 = Base64::encode_string(b"kh.throwaway7:Tr0ub4dor-9x"); + let (u, p) = (user("kh.throwaway7"), pw("Tr0ub4dor-9x")); + let s = strip_url(&format!("https://ex.test/a/{b64}/z"), &[], Some((&u, &p))).unwrap(); + assert!(s.path_segments_replaced >= 1); + assert_holds_no_typed_value(&s.value.path, &["kh.throwaway7:Tr0ub4dor-9x"]); + let url = format!("https://ex.test/a?{}=1", b64.trim_end_matches('=')); + let s = strip_url(&url, &[], Some((&u, &p))).unwrap(); + assert_eq!(s.value.query_keys, ["{typed}"]); + } + + #[test] + fn a_url_that_cannot_be_a_base_keeps_only_its_scheme() { + for (url, origin) in [ + ("data:text/plain,Tr0ub4dor-9x", "data:"), + ("blob:https://ex.test/3f2b", "blob:"), + ("about:blank", "about:"), + ("mailto:a@b.test?subject=x", "{other}:"), + ] { + let s = strip_url(url, &[], None).unwrap(); + assert_eq!( + s.value, + StrippedUrl { + origin: origin.into(), + path: String::new(), + query_keys: vec![] + } + ); + } + } + + #[test] + fn an_unknown_url_scheme_is_other() { + let s = strip_url("kh.throwaway7://ex.test/x", &[user("kh.throwaway7")], None).unwrap(); + assert_eq!(s.value.origin, "{other}://ex.test"); + let s = strip_url("chrome-extension://abc/x", &[], None).unwrap(); + assert_eq!(s.value.origin, "chrome-extension://abc"); + } + + #[test] + fn only_a_known_authorization_scheme_is_kept() { + for (header, want) in [ + (&b"Digest username=x"[..], "digest"), + (b"AWS4-HMAC-SHA256 Credential=x", "aws4-hmac-sha256"), + (b"Negotiate abc", "negotiate"), + (b"s3cr3t-session-token-0123456789 x", "{other}"), + (b"s3cr3t-session-token-0123456789", "{other}"), + (b"Tr0ub4dor-9x", "{typed}"), + (b"Tr0ub4dor-9x rest", "{typed}"), + ] { + let a = reduce_authorization(header, &[pw("Tr0ub4dor-9x")], None); + assert_eq!(a.scheme, want); + } + } + + #[test] + fn a_confident_duplicate_wins_over_a_low_confidence_one() { + let h = |low| MarkerHit { + role: Role::Password, + encoding: Encoding::AsTyped, + low_confidence: low, + }; + assert_eq!(dedup_hits(vec![h(true), h(false)]), [h(false)]); + assert_eq!(dedup_hits(vec![h(false), h(true)]), [h(false)]); + } + + #[test] + fn a_value_holding_a_url_delimiter_is_not_split_across_components() { + // `?`: the value is cut into a path segment and a query key. + let m = [user("ab?cd")]; + let s = strip_url("https://ex.test/a/ab?cd=1", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}"); + assert_eq!(s.value.query_keys, ["{typed}"]); + // `#`: the tail is a fragment, which is dropped; the head must not stay. + let m = [user("ab#cd")]; + let s = strip_url("https://ex.test/a/ab#cd", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}"); + // `@`: userinfo and host. + let m = [user("ab@cd")]; + let s = strip_url("https://ab@cd.ex.test/x", &m, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + let s = strip_url("https://ex.test/x/ab@cd", &m, None).unwrap(); + assert_eq!(s.value.path, "/x/{id}"); + let m = [pw("Tr0ub4dor?9x")]; + let s = strip_url("https://ex.test/login/Tr0ub4dor?9x", &m, None).unwrap(); + assert_eq!(s.value.path, "/login/{id}"); + assert_eq!(s.value.query_keys, ["{typed}"]); + } + + #[test] + fn a_match_across_a_delimiter_is_found_in_the_decoded_url_too() { + let m = [user("ab?cd")]; + // Fully encoded: one segment. + let s = strip_url("https://ex.test/a/ab%3Fcd?x=1", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}"); + assert_eq!(s.value.query_keys, ["x"]); + // Only the key is encoded: the decoded URL holds `ab?cd` across path and query. + let s = strip_url("https://ex.test/a/ab?c%64=1", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}"); + assert_eq!(s.value.query_keys, ["{typed}"]); + let m = [user("ab#cd")]; + let s = strip_url("https://ex.test/a/ab#%63d", &m, None).unwrap(); + assert_eq!(s.value.path, "/a/{id}"); + } + + #[test] + fn a_short_marker_matches_kept_text_only_as_a_whole_token() { + let on = [user("on")]; + let s = strip_url("https://ex.test/login/session?json=1", &on, None).unwrap(); + assert_eq!(s.value.path, "/login/session"); + assert_eq!(s.value.query_keys, ["json"]); + assert!(!holds_typed(b"application/json", &on, None)); + let s = strip_url("https://ex.test/u/on/x?on=1", &on, None).unwrap(); + assert_eq!(s.value.path, "/u/{id}/x"); + assert_eq!(s.value.query_keys, ["{typed}"]); + let one = [user("1")]; + let s = strip_url("https://ex.test/v1/login", &one, None).unwrap(); + assert_eq!(s.value.path, "/v1/login"); + let s = strip_url("https://ex.test/item/1", &one, None).unwrap(); + assert_eq!(s.value.path, "/item/{id}"); + let bob = [user("bob")]; + for (url, path, keys) in [ + ("https://ex.test/u/bob/x", "/u/{id}/x", vec![]), + ("https://ex.test/user-bob", "/{id}", vec![]), + ("https://ex.test/bob@ex.test", "/{id}", vec![]), + ("https://ex.test/x?bob=1", "/x", vec!["{typed}"]), + ("https://ex.test/bobby/x?bobby=1", "/bobby/x", vec!["bobby"]), + ] { + let s = strip_url(url, &bob, None).unwrap(); + assert_eq!(s.value.path, path, "{url}"); + assert_eq!(s.value.query_keys, keys, "{url}"); + } + // Six characters or more still match as a substring. + let s = strip_url("https://ex.test/x/zzabcdefzz", &[user("abcdef")], None).unwrap(); + assert_eq!(s.value.path, "/x/{id}"); + // Hosts follow the same rule. + let s = strip_url("https://person.ex.test/", &on, None).unwrap(); + assert_eq!(s.value.origin, "https://person.ex.test"); + let s = strip_url("https://on.ex.test/", &on, None).unwrap(); + assert_eq!(s.value.origin, "https://{typed}.ex.test"); + } + + #[test] + fn the_base64_of_a_short_password_is_found_as_a_whole_path_segment_only() { + // base64 of "abcde". + let m = [pw("abcde")]; + for seg in ["YWJjZGU=", "YWJjZGU"] { + let s = strip_url(&format!("https://ex.test/x/{seg}/y"), &m, None).unwrap(); + assert_eq!(s.value.path, "/x/{id}/y", "{seg}"); + } + // The same letters inside a longer word are not a match. + let s = strip_url("https://ex.test/x/helloYWJjZGUworld/y", &m, None).unwrap(); + assert_eq!(s.value.path, "/x/helloYWJjZGUworld/y"); + // Nor in a query key. + let s = strip_url("https://ex.test/x?YWJjZGU=1&aYWJjZGUa=2", &m, None).unwrap(); + assert_eq!(s.value.query_keys, ["{typed}", "aYWJjZGUa"]); + } + + #[test] + fn a_name_is_kept_only_when_it_looks_like_one() { + for ok in [ + "user", + "user_name", + "a.b:c-d", + "items[0]", + "x", + "", + &"g".repeat(64), + ] { + assert!(looks_like_a_name(ok), "{ok:?}"); + } + for bad in [ + "12345", + "3f2b8c1e-9a4d-4c1b-8e2f-0a1b2c3d4e5f", + "0123456789abcdef", + "has space", + "a@b.c", + "a=b", + "naïve", + &"g".repeat(65), + ] { + assert!(!looks_like_a_name(bad), "{bad:?}"); + } + } + + #[test] + fn a_query_key_that_is_not_a_name_is_other() { + let token = + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.abcdefghijklmnopqrstuvwxyz0123456789"; + let s = strip_url( + &format!("https://ex.test/x?{token}&ok=1&a%20b=2&{token}=3"), + &[], + None, + ) + .unwrap(); + assert_eq!(s.value.query_keys, ["{other}", "ok", "{other}", "{other}"]); + // A bare piece of 33 characters is other; of 32 it is a key. + let s = strip_url(&format!("https://ex.test/x?{}", "k".repeat(33)), &[], None).unwrap(); + assert_eq!(s.value.query_keys, ["{other}"]); + let s = strip_url(&format!("https://ex.test/x?{}", "k".repeat(32)), &[], None).unwrap(); + assert_eq!(s.value.query_keys, ["k".repeat(32)]); + // With a value, 33 is still a name (up to 64). + let s = strip_url( + &format!("https://ex.test/x?{}=1", "k".repeat(33)), + &[], + None, + ) + .unwrap(); + assert_eq!(s.value.query_keys, ["k".repeat(33)]); + let s = strip_url("https://ex.test/x?12345=1", &[], None).unwrap(); + assert_eq!(s.value.query_keys, ["{other}"]); + } +} diff --git a/crates/kharon-teach/src/testing.rs b/crates/kharon-teach/src/testing.rs new file mode 100644 index 0000000..33e9396 --- /dev/null +++ b/crates/kharon-teach/src/testing.rs @@ -0,0 +1,104 @@ +//! Support for this crate's tests. It takes only test constants, never a live typed value. +//! +//! It is public (and hidden from the docs) so unit tests and the integration tests under +//! `tests/` share one definition of "the encodings a site might apply to a value". + +use base64ct::{Base64, Base64Unpadded, Base64Url, Base64UrlUnpadded, Encoding as B64}; +use kharon_recording::Role; +use md5::Md5; +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode, percent_encode}; +use sha1::Sha1; +use sha2::{Digest, Sha256, Sha512}; +use zeroize::Zeroizing; + +use crate::marker::Marker; + +/// A username marker holding `value`. +pub fn user(value: &str) -> Marker { + Marker::new(Zeroizing::new(value.to_owned()), Role::Username) +} + +/// A password marker holding `value`. +pub fn pw(value: &str) -> Marker { + Marker::new(Zeroizing::new(value.to_owned()), Role::Password) +} + +const UNRESERVED: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~'); + +/// Label and bytes of `value` as typed, ASCII-lowercased, ASCII-uppercased, percent-encoded +/// (RFC 3986 strict, upper and lower hex), form-encoded (space as `+`), JSON-escaped, base64 and +/// base64url (padded and not), hex (lower and upper), and the hex of MD5, SHA-1, SHA-256 and +/// SHA-512. A form identical to an earlier one is skipped. +pub fn needles_for(value: &str) -> Vec<(&'static str, Vec)> { + let bytes = value.as_bytes(); + let strict: String = percent_encode(bytes, UNRESERVED).to_string(); + let lower_hex = { + let mut out = String::new(); + for chunk in percent_encode(bytes, UNRESERVED) { + if chunk.starts_with('%') { + out.push_str(&chunk.to_ascii_lowercase()); + } else { + out.push_str(chunk); + } + } + out + }; + let json = serde_json::to_string(value).unwrap_or_default(); + let json = json + .strip_prefix('"') + .and_then(|j| j.strip_suffix('"')) + .unwrap_or_default() + .to_owned(); + let hex_of = |b: &[u8]| hex::encode(b); + let candidates: Vec<(&'static str, Vec)> = vec![ + ("as typed", bytes.to_vec()), + ("lowercased", value.to_ascii_lowercase().into_bytes()), + ("uppercased", value.to_ascii_uppercase().into_bytes()), + ("percent upper", strict.clone().into_bytes()), + ("percent lower", lower_hex.into_bytes()), + ("form encoded", strict.replace("%20", "+").into_bytes()), + ("json escaped", json.into_bytes()), + ("base64", Base64::encode_string(bytes).into_bytes()), + ( + "base64 unpadded", + Base64Unpadded::encode_string(bytes).into_bytes(), + ), + ("base64url", Base64Url::encode_string(bytes).into_bytes()), + ( + "base64url unpadded", + Base64UrlUnpadded::encode_string(bytes).into_bytes(), + ), + ("hex lower", hex_of(bytes).into_bytes()), + ("hex upper", hex_of(bytes).to_uppercase().into_bytes()), + ("md5 hex", hex_of(&Md5::digest(bytes)).into_bytes()), + ("sha1 hex", hex_of(&Sha1::digest(bytes)).into_bytes()), + ("sha256 hex", hex_of(&Sha256::digest(bytes)).into_bytes()), + ("sha512 hex", hex_of(&Sha512::digest(bytes)).into_bytes()), + ]; + let mut out: Vec<(&'static str, Vec)> = Vec::new(); + for (label, form) in candidates { + if !form.is_empty() && !out.iter().any(|(_, f)| *f == form) { + out.push((label, form)); + } + } + out +} + +/// Panics when any needle of any of `values` occurs in `text` or in its percent-decoded form. +/// The message names the value's position and the encoding, never `text` or a value. +pub fn assert_holds_no_typed_value(text: &str, values: &[&str]) { + let decoded: Vec = percent_decode(text.as_bytes()).collect(); + for (i, value) in values.iter().enumerate() { + for (label, needle) in needles_for(value) { + let found = |h: &[u8]| h.windows(needle.len()).any(|w| w == needle.as_slice()); + assert!( + !found(text.as_bytes()) && !found(&decoded), + "typed value #{i} found in the output, as: {label}" + ); + } + } +} diff --git a/crates/kharon-teach/tests/req_tch_001_own_browser.rs b/crates/kharon-teach/tests/req_tch_001_own_browser.rs new file mode 100644 index 0000000..27d5097 --- /dev/null +++ b/crates/kharon-teach/tests/req_tch_001_own_browser.rs @@ -0,0 +1,120 @@ +//! SPEC req_tch_001: the recorder launches its own browser on its own +//! throwaway profile and the DevTools port is loopback only. + +mod support; + +use kharon_teach::browser::{Browser, Launch, Profile}; +use std::path::Path; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +/// Local addresses (hex ip) of every listening row for `port` in a +/// `/proc/net/tcp*` file. +fn listeners(file: &str, port: u16) -> Vec { + let text = std::fs::read_to_string(file).unwrap_or_default(); + let want = format!(":{port:04X}"); + text.lines() + .skip(1) + .filter_map(|l| { + let cols: Vec<&str> = l.split_whitespace().collect(); + let local = *cols.get(1)?; + let state = *cols.get(3)?; + (state == "0A" && local.ends_with(&want)) + .then(|| local[..local.len() - want.len()].to_owned()) + }) + .collect() +} + +#[tokio::test] +async fn launches_on_its_own_profile_and_binds_loopback_only() { + let Some(bin) = support::chromium_or_skip("launches_on_its_own_profile") else { + return; + }; + let profile = Profile::create().unwrap(); + let profile_path = profile.path().to_owned(); + let (browser, dt) = Browser::launch(Launch { + browser: &bin, + profile: &profile, + first_url: "about:blank", + headless: true, + }) + .await + .unwrap(); + + let cmdline = std::fs::read(format!("/proc/{}/cmdline", browser.pid())).unwrap(); + let cmdline = String::from_utf8_lossy(&cmdline).replace('\0', " "); + assert!( + cmdline.contains(&format!("--user-data-dir={}", profile_path.display())), + "{cmdline}" + ); + + let mut s = tokio::net::TcpStream::connect(("127.0.0.1", dt.port)) + .await + .unwrap(); + s.write_all(b"GET /json/version HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + // Chromium keeps the connection open, so read until the document is complete. + let mut body = String::new(); + let mut buf = [0u8; 4096]; + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while !body.trim_end().ends_with('}') { + let n = s.read(&mut buf).await.unwrap(); + assert!(n > 0, "closed early: {body}"); + body.push_str(&String::from_utf8_lossy(&buf[..n])); + } + }) + .await + .expect("no /json/version answer"); + assert!(body.starts_with("HTTP/1.1 200"), "{body}"); + assert!(body.contains("webSocketDebuggerUrl"), "{body}"); + + let v4 = listeners("/proc/net/tcp", dt.port); + let v6 = listeners("/proc/net/tcp6", dt.port); + assert!(!v4.is_empty() || !v6.is_empty(), "no listener found"); + assert!(v4.iter().all(|a| a == "0100007F"), "{v4:?}"); + assert!( + v6.iter() + .all(|a| a == "00000000000000000000000001000000" + || a == "0000000000000000FFFF00000100007F"), + "{v6:?}" + ); + + browser.close().await.unwrap(); + profile.remove().unwrap(); + assert!(!Path::new(&profile_path).exists()); +} + +#[test] +fn the_library_has_no_way_to_attach_elsewhere() { + let mut launch_src = None; + for (f, shipped) in support::shipped_sources() { + let shipped = shipped.as_str(); + for line in shipped.lines() { + for lit in ["ws://", "http://"] { + assert!( + !line.contains(lit) || line.contains("127.0.0.1"), + "{}: a {lit} literal that is not the loopback CDP URL: {line}", + f.display() + ); + } + } + if let Some(at) = shipped.find("pub struct Launch<") { + let rest = &shipped[at..]; + let open = rest.find('{').unwrap(); + let close = rest.find('}').unwrap(); + launch_src = Some(rest[open + 1..close].to_owned()); + } + } + let body = launch_src.expect("Launch struct not found"); + let mut names: Vec = body + .split(',') + .map(str::trim) + .filter(|f| !f.is_empty()) + .map(|f| { + let f = f.strip_prefix("pub ").unwrap_or(f); + f.split(':').next().unwrap().trim().to_owned() + }) + .collect(); + names.sort(); + assert_eq!(names, ["browser", "first_url", "headless", "profile"]); +} diff --git a/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs b/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs new file mode 100644 index 0000000..7c81519 --- /dev/null +++ b/crates/kharon-teach/tests/req_tch_002_no_typed_value.rs @@ -0,0 +1,190 @@ +//! SPEC req_tch_002: nothing a person typed reaches the recording. + +mod support; + +use kharon_teach::session::{End, Outcome}; +use kharon_teach::testing::assert_holds_no_typed_value; +use support::{Login, PASS, Run, USER, record_login}; + +#[tokio::test] +async fn no_typed_value_in_the_recording_for_any_fixture() { + for (name, which) in [ + ("form", Login::Form), + ("script", Login::Script), + ("two_step", Login::TwoStep), + ("hashed", Login::Hashed), + ] { + let Some(r) = record_login(&format!("no_typed_value_{name}"), which).await else { + return; + }; + let json = r.to_json(); + // The file is a real one: the typed fields are in it as lengths, not as text. + assert!(json.contains("\"value_len\""), "{name}: no input steps"); + assert_holds_no_typed_value(&json, &[USER, PASS]); + } +} + +/// The typed values are in text the recording KEEPS: the next page's path and query key, and the +/// key of the JSON body. They must be stripped there, not merely absent from the values. This +/// fails if `strip_url` or the name check returned its input unchanged. +#[tokio::test] +async fn typed_values_in_a_url_and_a_json_key_are_stripped_from_kept_text() { + use kharon_recording::{BodyShape, JsonShape, Role, Step}; + let Some(run) = Run::start("typed_in_text", "/typed").await else { + return; + }; + run.begin("/typed").await; + run.person.type_into("input[name=user]", USER).await; + run.person.type_into("input[name=pass]", PASS).await; + run.person.click("#go").await; + run.person.wait_for_path(&format!("/u/{USER}")).await; + let Outcome::Recorded(r) = run.finish(End::Confirmed).await else { + panic!("expected a recording"); + }; + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); + + // The navigation to /u/?=1 is in the file, with both typed places replaced. + let moved = |url: &kharon_recording::StrippedUrl| { + url.path == "/u/{id}" && url.query_keys == ["{typed}"] + }; + assert!( + r.steps + .iter() + .any(|s| matches!(s, Step::Navigation { url, .. } if moved(url))), + "no navigation step with the typed path replaced" + ); + assert!(r.requests.iter().any(|q| moved(&q.url))); + assert!(r.final_url.as_ref().is_some_and(moved)); + + // The login posts JSON keyed by the user name: the key is {typed} and its hit is not lost. + let login = r + .requests + .iter() + .find(|q| q.url.path == "/api/typed") + .expect("the login request is stored"); + let BodyShape::Json { + root: JsonShape::Object { fields }, + } = &login.body + else { + panic!("the body is not a JSON object"); + }; + assert_eq!(fields.len(), 1); + assert_eq!(fields[0].key, "{typed}"); + let hits = login.hits(); + assert!(hits.iter().any(|h| h.role == Role::Password)); + assert!(hits.iter().any(|h| h.role == Role::Username)); +} + +#[tokio::test] +async fn nothing_typed_reaches_the_file_or_the_output() { + let Some(p) = support::run_program("whole_program_no_typed_value", "/form").await else { + return; + }; + assert_eq!(p.exit, kharon_teach::Exit::Done, "{}", p.stderr); + assert_eq!(p.file_mode, 0o600); + assert!(p.file_text.contains("\"value_len\""), "the file is empty"); + assert!(p.stdout.contains(&p.path.display().to_string())); + for text in [&p.file_text, &p.stdout, &p.stderr] { + assert_holds_no_typed_value(text, &[USER, PASS]); + } +} + +/// A page that copies what is typed into a field into the field's own `id` must not get it into +/// the recording through the descriptor or the selectors, not even on the Enter-key report and +/// not for a value of two characters (which no search for typed values can safely look for). +#[tokio::test] +async fn a_page_that_mirrors_the_value_into_an_attribute_leaks_nothing() { + use kharon_recording::Step; + let Some(run) = Run::start("mirror", "/mirror").await else { + return; + }; + run.begin("/mirror").await; + run.person.type_into("input[name=user]", "q7").await; + run.person.type_into("input[name=pass]", PASS).await; + run.person.press_enter().await; + run.person.wait_for_path("/home").await; + let Outcome::Recorded(r) = run.finish(End::Confirmed).await else { + panic!("expected a recording"); + }; + assert_holds_no_typed_value(&r.to_json(), &[PASS]); + let mut seen = 0; + for step in &r.steps { + let element = match step { + Step::Input { element, .. } | Step::EnterKey { element, .. } => element, + _ => continue, + }; + seen += 1; + let texts = element + .selectors + .iter() + .chain(element.id.iter()) + .chain(element.name.iter()) + .chain(element.autocomplete.iter()); + for t in texts { + assert!(!t.contains("q7"), "a selector or attribute holds the value"); + } + assert!(!element.selectors.is_empty(), "the field keeps a selector"); + } + assert!(seen >= 3, "two inputs and the Enter key were recorded"); +} + +/// The listener itself, run in the page's main world with a stand-in binding, so the raw reports +/// can be read. Whatever the recorder later strips, the script must not send the typed text in +/// a descriptor or a selector. +#[tokio::test] +async fn the_listener_never_describes_an_element_by_what_was_typed_into_it() { + use kharon_teach::events::ListenerReport; + let Some(run) = Run::start("listener_raw", "/mirror").await else { + return; + }; + run.begin("/mirror").await; + let source = kharon_teach::session::LISTENER_JS.replace("__NONCE__", "raw-test"); + let install = format!( + "window.__reports=[];window.__kharonTeach=(p)=>{{window.__reports.push(p)}};\ + document.addEventListener('submit',(e)=>e.preventDefault(),true);{source}\n;'ok'" + ); + assert_eq!(run.person.eval(&install).await.as_deref(), Some("ok")); + run.person.type_into("input[name=user]", "q7").await; + run.person.press_enter().await; + let raw = run + .person + .eval("JSON.stringify(window.__reports)") + .await + .expect("the reports"); + let reports: Vec = serde_json::from_str(&raw).unwrap(); + let mut seen = (0, 0); + for r in &reports { + let Ok(rep) = serde_json::from_str::(r) else { + continue; // the hello + }; + match rep.kind { + kharon_teach::events::ReportKind::Input => seen.0 += 1, + kharon_teach::events::ReportKind::EnterKey => seen.1 += 1, + _ => {} + } + let e = &rep.element; + let texts = e + .selectors + .iter() + .chain(e.id.iter()) + .chain(e.name.iter()) + .chain(e.autocomplete.iter()) + .chain(e.placeholder.iter()) + .chain(e.aria_label.iter()); + for t in texts { + assert!( + !t.contains("q7"), + "a descriptor or selector holds typed text" + ); + } + assert!( + !e.selectors.is_empty(), + "the field keeps a selector that does not hold it" + ); + } + assert!( + seen.0 >= 1 && seen.1 >= 1, + "an input and an Enter key were reported" + ); + let _ = run.finish(End::Discard).await; +} diff --git a/crates/kharon-teach/tests/req_tch_003_file_contents.rs b/crates/kharon-teach/tests/req_tch_003_file_contents.rs new file mode 100644 index 0000000..ca61704 --- /dev/null +++ b/crates/kharon-teach/tests/req_tch_003_file_contents.rs @@ -0,0 +1,63 @@ +//! SPEC req_tch_003: the file holds only what the spec keeps. + +mod support; + +use kharon_recording::Recording; +use support::{COOKIE_VALUE, run_program}; + +const NEXT_VALUE: &str = "secret-next-value"; + +#[tokio::test] +async fn the_file_is_0600_and_holds_only_what_the_spec_keeps() { + let Some(p) = run_program("file_contents", &format!("/form?next=/{NEXT_VALUE}")).await else { + return; + }; + assert_eq!(p.exit, kharon_teach::Exit::Done, "{}", p.stderr); + assert_eq!(p.file_mode, 0o600); + let text = &p.file_text; + let rec = Recording::from_json(text).expect("the file reads back"); + // Query keys are kept, their values are not. + let keys_kept = std::iter::once(&rec.start_url) + .chain(rec.requests.iter().map(|r| &r.url)) + .any(|u| u.query_keys.iter().any(|k| k == "next")); + assert!(keys_kept, "the query key `next` was not kept"); + assert!(!text.contains(NEXT_VALUE), "a query value is in the file"); + // No cookie, no response body, no page title. + assert!( + !text.contains(COOKIE_VALUE), + "a cookie value is in the file" + ); + assert!(!text.contains("sid="), "a cookie is in the file"); + assert!(!text.contains("signed in"), "response text is in the file"); + for title in ["", ">form<", "\"home\"", "step one"] { + assert!( + !text.contains(title), + "a page title is in the file: {title}" + ); + } + // Headers are reduced to the content type: no other header the fixture and the browser + // exchanged is in the file, by name or by value. + let lower = text.to_ascii_lowercase(); + for header in [ + "set-cookie", + "user-agent", + "referer", + "accept-language", + "accept-encoding", + "content-length", + "location", + "\"cookie\"", + ] { + assert!(!lower.contains(header), "header {header} is in the file"); + } + // The cookie was set by the response, and the file says only that it was. + let login = rec + .requests + .iter() + .find(|r| r.url.path == "/form-login") + .expect("the sign-in request was stored"); + assert!( + login.response.as_ref().is_some_and(|r| r.set_cookie), + "the response's Set-Cookie was not noted as a flag" + ); +} diff --git a/crates/kharon-teach/tests/req_tch_004_no_network.rs b/crates/kharon-teach/tests/req_tch_004_no_network.rs new file mode 100644 index 0000000..7e20fae --- /dev/null +++ b/crates/kharon-teach/tests/req_tch_004_no_network.rs @@ -0,0 +1,79 @@ +//! SPEC req_tch_004: the recorder opens no socket but loopback CDP, and links no TLS or HTTP client. + +mod support; + +use support::shipped_sources; + +#[test] +fn the_crate_listens_on_nothing_and_has_no_tls() { + for (path, text) in shipped_sources() { + for banned in ["TcpListener", "UdpSocket", "bind(", "accept("] { + assert!(!text.contains(banned), "{} holds {banned}", path.display()); + } + } + let out = std::process::Command::new(env!("CARGO")) + .args([ + "tree", + "-p", + "kharon-teach", + "-e", + "normal", + "--prefix", + "none", + ]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .output() + .expect("cargo tree runs"); + assert!( + out.status.success(), + "cargo tree failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + let tree = String::from_utf8_lossy(&out.stdout); + assert!(tree.lines().any(|l| l.starts_with("kharon-teach "))); + for banned in [ + "rustls", + "native-tls", + "openssl", + "openssl-sys", + "reqwest", + "hyper", + "kharon-connector", + "kharond", + ] { + let hit = tree + .lines() + .any(|l| l.split_whitespace().next() == Some(banned)); + assert!(!hit, "the dependency tree holds {banned}"); + } +} + +#[test] +fn the_only_dial_is_loopback() { + let sources = shipped_sources(); + let cdp = sources + .iter() + .find(|(p, _)| p.ends_with("cdp.rs")) + .expect("cdp.rs exists"); + assert!( + cdp.1.contains("ws://127.0.0.1:"), + "cdp.rs does not build its URL from the loopback literal" + ); + let dials: Vec<_> = sources + .iter() + .flat_map(|(p, t)| { + ["connect_async", "TcpStream::connect", "UnixStream::connect"] + .into_iter() + .filter(|d| t.contains(d)) + .map(move |d| format!("{}: {d}", p.display())) + }) + .collect(); + assert!( + !dials.is_empty(), + "cdp.rs no longer dials with connect_async" + ); + assert!( + dials.iter().all(|d| d.contains("cdp.rs")), + "a dial outside cdp.rs: {dials:?}" + ); +} diff --git a/crates/kharon-teach/tests/req_tch_005_shapes.rs b/crates/kharon-teach/tests/req_tch_005_shapes.rs new file mode 100644 index 0000000..bb6d310 --- /dev/null +++ b/crates/kharon-teach/tests/req_tch_005_shapes.rs @@ -0,0 +1,321 @@ +//! SPEC req_tch_005: the recorder tells the shapes of login apart in a real browser. + +mod support; + +use kharon_recording::{ + EndedBy, ListenerWorld, PasswordLeg, Recording, ResourceKind, RungCeiling, Shape, +}; +use kharon_teach::session::POST_DATA_FETCHES; +use kharon_teach::session::{End, Outcome}; +use kharon_teach::testing::assert_holds_no_typed_value; +use support::{COOKIE_VALUE, Login, PASS, Run, USER, record_login}; + +#[tokio::test] +async fn form_page_classifies_as_form_post() { + let Some(r) = record_login("form_page", Login::Form).await else { + return; + }; + let c = &r.classification; + assert_eq!(c.shape, Shape::FormPost, "{:?}", c.notes); + assert_eq!(c.rung_ceiling, RungCeiling::Milliseconds); + assert!(c.network_swap_candidate); + assert_eq!(r.ended_by, EndedBy::Confirmed); + assert!(r.success_confirmed); + assert_eq!(r.listener_world, ListenerWorld::Isolated); + assert!(!r.browser.product.is_empty()); + assert!(r.browser.viewport.width > 0 && r.browser.viewport.height > 0); + + // The 302 that sets a cookie is stored with its response, and the cookie's value is not. + let post = r + .requests + .iter() + .find(|q| q.method == "POST" && q.url.path == "/form-login") + .expect("the login POST is stored"); + assert_eq!(post.resource, ResourceKind::Document); + let resp = post.response.as_ref().expect("the 302 is stored with it"); + assert_eq!(resp.status, 302); + assert!(resp.set_cookie); + assert!(!r.to_json().contains(COOKIE_VALUE)); +} + +#[tokio::test] +async fn script_page_classifies_as_script_post() { + let Some(r) = record_login("script_page", Login::Script).await else { + return; + }; + assert_eq!( + r.classification.shape, + Shape::ScriptPost, + "{:?}", + r.classification.notes + ); + assert_eq!( + r.classification.rung_ceiling, + RungCeiling::Milliseconds, + "{:?}", + r.classification.notes + ); +} + +#[tokio::test] +async fn two_step_page_classifies_as_two_step() { + let Some(r) = record_login("two_step_page", Login::TwoStep).await else { + return; + }; + let c = &r.classification; + assert_eq!( + c.shape, + Shape::TwoStep { + password_leg: PasswordLeg::FormPost + }, + "{:?}", + c.notes + ); + assert_eq!(c.rung_ceiling, RungCeiling::AgentContinues); +} + +#[tokio::test] +async fn hashed_page_classifies_as_transformed() { + let Some(r) = record_login("hashed_page", Login::Hashed).await else { + return; + }; + let c = &r.classification; + assert_eq!( + c.shape, + Shape::Transformed { + digest: Some("sha256".to_owned()) + }, + "{:?}", + c.notes + ); + assert_eq!(c.rung_ceiling, RungCeiling::Milliseconds, "{:?}", c.notes); + assert!(!c.network_swap_candidate); +} + +/// The first selector of an element is one a later visit can use to find the same field. +#[tokio::test] +async fn the_first_selector_of_each_typed_element_matches() { + let Some(r) = record_login("first_selector", Login::Form).await else { + return; + }; + let typed = typed_elements(&r); + assert_eq!(typed.len(), 2, "two typed fields"); + for sel in &typed { + assert!(!sel.is_empty()); + } + // Re-open the fixture and check each selector finds exactly the field it describes. + let Some(run) = Run::start("first_selector_reopen", "/form").await else { + return; + }; + run.begin("/form").await; + for (sel, want_name) in typed.iter().zip(["user", "pass"]) { + let found = run.person.attr_of(sel, "name").await; + assert_eq!(found.as_deref(), Some(want_name), "selector {sel}"); + } + assert!(matches!(run.finish(End::Discard).await, Outcome::Discarded)); +} + +fn typed_elements(r: &Recording) -> Vec<String> { + use kharon_recording::Step; + r.steps + .iter() + .filter_map(|s| match s { + Step::Input { element, .. } => element.selectors.first().cloned(), + _ => None, + }) + .collect() +} + +fn request_to<'a>(r: &'a Recording, path: &str) -> &'a kharon_recording::StoredRequest { + r.requests + .iter() + .find(|q| q.method == "POST" && q.url.path == path) + .unwrap_or_else(|| panic!("no stored POST to {path}")) +} + +fn has_password_hit(parts: &[kharon_recording::FieldShape], key: &str) -> bool { + parts.iter().any(|p| { + p.key == key + && p.value + .markers + .iter() + .any(|m| m.role == kharon_recording::Role::Password) + }) +} + +#[tokio::test] +async fn a_form_data_post_is_stored_as_multipart_with_the_password_found() { + let Some(r) = record_login("script_formdata", Login::ScriptFormData).await else { + return; + }; + assert_eq!( + r.classification.shape, + Shape::ScriptPost, + "{:?}", + r.classification.notes + ); + match &request_to(&r, "/api/login-form").body { + kharon_recording::BodyShape::Multipart { parts } => { + assert!( + has_password_hit(parts, "pass"), + "no password hit on the pass part" + ) + } + _ => panic!("the body is not multipart"), + } + assert!(!r.flags.body_unavailable); + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); +} + +#[tokio::test] +async fn a_blob_body_is_fetched_with_get_request_post_data() { + use std::sync::atomic::Ordering; + let before = POST_DATA_FETCHES.load(Ordering::Relaxed); + let Some(r) = record_login("script_blob", Login::ScriptBlob).await else { + return; + }; + // Whether Chromium sends a Blob body inline or not, the stored shape must carry the password. + assert_eq!( + r.classification.shape, + Shape::ScriptPost, + "{:?}", + r.classification.notes + ); + match &request_to(&r, "/api/login-blob").body { + kharon_recording::BodyShape::Json { .. } | kharon_recording::BodyShape::Opaque { .. } => {} + other => panic!("unexpected body shape kind {}", shape_name(other)), + } + assert!(!r.flags.body_unavailable); + assert!( + POST_DATA_FETCHES.load(Ordering::Relaxed) > before, + "the body was inline: getRequestPostData never ran" + ); + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); +} + +fn shape_name(b: &kharon_recording::BodyShape) -> &'static str { + use kharon_recording::BodyShape::*; + match b { + None => "none", + Unavailable => "unavailable", + Urlencoded { .. } => "urlencoded", + Json { .. } => "json", + Multipart { .. } => "multipart", + Opaque { .. } => "opaque", + } +} + +#[tokio::test] +async fn the_main_world_fallback_still_records_a_login() { + use kharon_teach::session::WorldProbe; + let Some(run) = Run::start_with("main_world", "/form", WorldProbe::SkipIsolated).await else { + return; + }; + run.begin("/form").await; + run.person.type_into("input[name=user]", USER).await; + run.person.type_into("input[name=pass]", PASS).await; + run.person.click("#go").await; + run.person.wait_for_path("/home").await; + let Outcome::Recorded(r) = run.finish(End::Confirmed).await else { + panic!("expected a recording"); + }; + assert_eq!(r.listener_world, ListenerWorld::Main); + assert_eq!( + r.classification.shape, + Shape::FormPost, + "{:?}", + r.classification.notes + ); + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); +} + +#[tokio::test] +async fn children_named_like_form_properties_do_not_hide_the_form() { + let Some(r) = record_login("clobber", Login::Clobber).await else { + return; + }; + assert_eq!( + r.classification.shape, + Shape::FormPost, + "{:?}", + r.classification.notes + ); + let form = r + .steps + .iter() + .find_map(|s| match s { + kharon_recording::Step::Submit { form, .. } => Some(form.clone()), + _ => None, + }) + .expect("a submit step"); + assert_eq!(form.method, "post"); + assert_eq!(form.action.map(|a| a.path).as_deref(), Some("/form-login")); + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); +} + +#[tokio::test] +async fn a_popup_sets_the_flag_and_is_not_a_login() { + let Some(run) = Run::start("popup", "/popup").await else { + return; + }; + run.begin("/popup").await; + run.person.click("#go").await; + run.person.wait_for_pages(2).await; + let Outcome::Recorded(r) = run.finish(End::Confirmed).await else { + panic!("expected a recording"); + }; + assert!(r.flags.popup); + assert_eq!( + r.classification.shape, + Shape::Unrecognized, + "{:?}", + r.classification.notes + ); +} + +#[tokio::test] +async fn a_login_in_a_cross_origin_frame_sets_framed_login() { + let Some(run) = Run::start("framed", "/frame-outer").await else { + return; + }; + run.begin("/frame-outer").await; + let frame = run.person.attach_frame("localhost").await; + run.person + .type_into_frame(&frame, "input[name=user]", USER) + .await; + run.person + .type_into_frame(&frame, "input[name=pass]", PASS) + .await; + run.person.press_enter().await; + run.person.wait_for_frame_path(&frame, "/home").await; + let Outcome::Recorded(r) = run.finish(End::Confirmed).await else { + panic!("expected a recording"); + }; + assert!(r.flags.framed_login, "{:?}", r.classification.notes); + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); +} + +#[tokio::test] +async fn a_checkbox_is_not_a_typed_value() { + let Some(run) = Run::start("checkbox", "/form").await else { + return; + }; + run.begin("/form").await; + run.person.click("#rem").await; + run.person.type_into("input[name=user]", USER).await; + run.person.type_into("input[name=pass]", PASS).await; + run.person.click("#go").await; + run.person.wait_for_path("/home").await; + let Outcome::Recorded(r) = run.finish(End::Confirmed).await else { + panic!("expected a recording"); + }; + assert_eq!(r.stripped.typed_values_removed, 2); + assert_eq!( + r.classification.shape, + Shape::FormPost, + "{:?}", + r.classification.notes + ); + assert!(r.requests.iter().any(|q| q.url.path == "/form-login")); + assert_holds_no_typed_value(&r.to_json(), &[USER, PASS]); +} diff --git a/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs new file mode 100644 index 0000000..1c1fa20 --- /dev/null +++ b/crates/kharon-teach/tests/req_tch_006_profile_and_visible.rs @@ -0,0 +1,751 @@ +//! SPEC req_tch_006: the browser keeps nothing from a login, and the profile is always removed. + +mod support; + +use kharon_recording::EndedBy; +use kharon_teach::session::{End, Outcome}; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use support::{PASS, Run, USER}; + +/// Row counts of `tables` in a copy of the SQLite file at `db`; a missing table counts as 0. +/// `None` when `python3` is absent. +fn row_counts(db: &Path, tables: &[&str]) -> Option<Vec<u64>> { + static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + let copy = std::env::temp_dir().join(format!( + "kharon-teach-test-copy-{}-{}", + std::process::id(), + N.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + )); + let _ = std::fs::remove_file(©); + std::fs::copy(db, ©).unwrap(); + // A write-ahead log or journal beside the database holds rows the main file does not yet. + let siblings: Vec<PathBuf> = ["-wal", "-journal"] + .iter() + .map(|suffix| { + let mut from = db.as_os_str().to_owned(); + from.push(suffix); + let mut to = copy.as_os_str().to_owned(); + to.push(suffix); + let (from, to) = (PathBuf::from(from), PathBuf::from(to)); + if from.exists() { + let _ = std::fs::copy(&from, &to); + } + to + }) + .collect(); + let script = [ + "import sqlite3,sys", + "c=sqlite3.connect(sys.argv[1])", + "for t in sys.argv[2:]:", + " try:", + " print(c.execute('select count(*) from ' + t).fetchone()[0])", + " except sqlite3.Error:", + " print(0)", + ] + .join("\n"); + let out = std::process::Command::new("python3") + .args(["-I", "-c", &script]) + .arg(©) + .args(tables) + .output(); + let _ = std::fs::remove_file(©); + for sib in &siblings { + let _ = std::fs::remove_file(sib); + } + let out = match out { + Ok(o) if o.status.success() => o, + Ok(o) => panic!( + "python3 failed reading {}: {} {}", + db.display(), + o.status, + String::from_utf8_lossy(&o.stderr) + ), + Err(_) => { + eprintln!("SKIP: python3 is not installed"); + return None; + } + }; + Some( + String::from_utf8_lossy(&out.stdout) + .lines() + .map(|l| l.trim().parse().unwrap()) + .collect(), + ) +} + +/// True when `path` goes away and stays away: a second look a second later, because a helper +/// process of a killed browser can recreate files in a directory that was just removed. +async fn gone(path: &Path) -> bool { + for _ in 0..100 { + if !path.exists() { + tokio::time::sleep(Duration::from_secs(1)).await; + return !path.exists(); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + false +} + +async fn type_a_login(run: &Run) { + run.begin("/form").await; + run.person.type_into("input[name=user]", USER).await; + run.person.type_into("input[name=pass]", PASS).await; +} + +#[tokio::test] +async fn nothing_is_saved_in_the_profile_during_a_login() { + let Some(run) = Run::start("nothing_saved", "/form").await else { + return; + }; + type_a_login(&run).await; + run.person.click("#go").await; + run.person.wait_for_path("/home").await; + let profile = run.ready.profile_dir.clone(); + assert!(profile.is_dir(), "the profile directory exists"); + // At least one of the two databases is there to be looked at (Chromium makes them lazily). + let dbs = ["Login Data", "Web Data"].map(|f| profile.join("Default").join(f)); + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while !dbs.iter().any(|d| d.exists()) { + assert!( + tokio::time::Instant::now() < deadline, + "neither Login Data nor Web Data appeared in the profile" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + for (db, tables) in dbs.iter().zip([ + &["logins"][..], + &["autofill", "autofill_profiles", "credit_cards"][..], + ]) { + if !db.exists() { + continue; + } + if let Some(counts) = row_counts(db, tables) { + assert!( + counts.iter().all(|n| *n == 0), + "{}: {counts:?}", + db.display() + ); + } + } + // The preferences the recorder wrote are still in force after the login. + #[derive(serde::Deserialize)] + struct Prefs { + credentials_enable_service: bool, + } + let prefs: Prefs = serde_json::from_str( + &std::fs::read_to_string(profile.join("Default").join("Preferences")).unwrap(), + ) + .unwrap(); + assert!(!prefs.credentials_enable_service); + let _ = run.finish(End::Discard).await; +} + +#[tokio::test] +async fn profile_is_gone_after_confirm() { + let Some(run) = Run::start("gone_after_confirm", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + type_a_login(&run).await; + run.person.click("#go").await; + run.person.wait_for_path("/home").await; + assert!(matches!( + run.finish(End::Confirmed).await, + Outcome::Recorded(_) + )); + assert!(gone(&profile).await, "the profile is still there"); +} + +#[tokio::test] +async fn profile_is_gone_after_discard() { + let Some(run) = Run::start("gone_after_discard", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + type_a_login(&run).await; + assert!(matches!(run.finish(End::Discard).await, Outcome::Discarded)); + assert!(gone(&profile).await, "the profile is still there"); +} + +#[tokio::test] +async fn closing_the_window_ends_unconfirmed() { + let Some(mut run) = Run::start("close_mid_login", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + type_a_login(&run).await; + let begun = std::time::Instant::now(); + run.person.close_all_pages().await; + match run.wait().await { + Outcome::Recorded(r) => { + assert_eq!(r.ended_by, EndedBy::WindowClosed); + assert!(!r.success_confirmed); + } + _ => panic!("expected a recording"), + } + assert!(begun.elapsed() < Duration::from_secs(10)); + assert!(gone(&profile).await, "the profile is still there"); +} + +#[tokio::test] +async fn closing_before_start_is_never_started() { + let Some(mut run) = Run::start("close_before_start", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + run.person.close_all_pages().await; + assert!(matches!(run.wait().await, Outcome::NeverStarted)); + assert!(gone(&profile).await, "the profile is still there"); +} + +#[tokio::test] +async fn dropping_the_future_removes_the_profile() { + let Some(run) = Run::start("drop_future", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + let port = run.ready.devtools_port; + type_a_login(&run).await; + // The Ctrl-C path: the task is dropped mid-recording. + run.abort(); + assert!(gone(&profile).await, "the profile is still there"); + // The browser went with it: its DevTools port stops answering. + let mut closed = false; + for _ in 0..100 { + if tokio::net::TcpStream::connect(("127.0.0.1", port)) + .await + .is_err() + { + closed = true; + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!(closed, "the browser is still listening"); +} + +#[tokio::test] +async fn a_stale_profile_is_swept_at_the_next_start() { + // What an earlier, killed run leaves: the name this program makes, and an owner file naming + // a pid that is gone. + let stale: PathBuf = + std::env::temp_dir().join(format!("kharon-teach-{:020}", std::process::id())); + std::fs::create_dir_all(stale.join("Default")).unwrap(); + std::fs::write(stale.join("Default").join("Preferences"), "{}").unwrap(); + std::fs::write(stale.join("kharon-teach.owner"), "4294967295").unwrap(); + let Some(run) = Run::start("stale_swept", "/form").await else { + let _ = std::fs::remove_dir_all(&stale); + return; + }; + // Tests in this file start at the same time, and any of them may have done the sweep. + assert!(!stale.exists(), "the stale profile was not swept"); + let _ = run.finish(End::Discard).await; +} + +#[tokio::test] +async fn a_directory_the_program_did_not_make_survives_the_sweep() { + // Same prefix, but no owner file, a junk owner file, or a name of another shape. + let id = std::process::id(); + let base = std::env::temp_dir(); + let ownerless = base.join(format!("kharon-teach-foo-{id}")); + let shaped = base.join(format!( + "kharon-teach-{:020}", + u64::from(id) + 1_000_000_000_000 + )); + let junk = base.join(format!( + "kharon-teach-{:020}", + u64::from(id) + 2_000_000_000_000 + )); + for d in [&ownerless, &shaped, &junk] { + std::fs::create_dir_all(d).unwrap(); + std::fs::write(d.join("keep"), "x").unwrap(); + } + std::fs::write(junk.join("kharon-teach.owner"), "not a pid").unwrap(); + let kept = |d: &Path| d.join("keep").exists(); + let ran = match Run::start("sweep_keeps_foreign", "/form").await { + Some(run) => { + let _ = run.finish(End::Discard).await; + true + } + None => false, + }; + let all_kept = [&ownerless, &shaped, &junk].iter().all(|d| kept(d)); + for d in [&ownerless, &shaped, &junk] { + let _ = std::fs::remove_dir_all(d); + } + if ran { + assert!(all_kept, "a directory the program did not make was swept"); + } +} + +#[test] +fn the_source_scan_covers_all_shipped_code() { + // A test-only item above the test module would hide shipped code from every scan. + assert!(support::split_shipped("fn a() {}\n#[cfg(test)]\nmod tests {}\n").is_ok()); + assert!(support::split_shipped("fn a() {}\n#[cfg(test)]\npub(crate) mod tests {}\n").is_ok()); + assert!( + support::split_shipped( + "#[cfg(test)]\nfn helper() {}\nfn b() {}\n#[cfg(test)]\nmod tests {}\n" + ) + .is_err() + ); + assert!(support::split_shipped("fn a() {}\n#[cfg(test)]\nfn helper() {}\n").is_err()); + assert_eq!( + support::split_shipped("fn a() {}\n#[cfg(test)]\nmod tests {}\n").unwrap(), + "fn a() {}\n" + ); + // The real sources: capture's `finish` and `request` are scanned, and no file has a + // test-only item outside its test module (`shipped_sources` panics when one does). + let sources = support::shipped_sources(); + let capture = sources + .iter() + .find(|(p, _)| p.ends_with("capture/mod.rs")) + .expect("capture/mod.rs"); + assert!(capture.1.contains("pub fn finish(")); + assert!(capture.1.contains("fn request(")); + assert!(!capture.1.contains("with_body_budget")); +} + +#[test] +fn the_shipped_main_never_asks_for_headless() { + let sources = support::shipped_sources(); + let mut env_reads = Vec::new(); + for (p, shipped) in &sources { + let name = p.file_name().unwrap().to_string_lossy().into_owned(); + assert!( + !shipped.contains("headless: true"), + "{} asks for headless", + p.display() + ); + // Every `headless:` initializer outside the session and the browser launcher is the + // literal `false` or a pass-through of the hook inside `run_with`. + if name != "session.rs" && name != "browser.rs" { + for (n, line) in shipped.lines().enumerate() { + let t = line.trim(); + if t.starts_with("//") || !t.contains("headless:") { + continue; + } + let ok = t == "headless: false," || t == "headless: hooks.headless,"; + let decl = t.starts_with("pub headless: bool"); + assert!(ok || decl, "{}:{}: {t}", p.display(), n + 1); + } + } + for pat in [ + "env::var(", + "env::var_os(", + "env::vars(", + "env::vars_os(", + "use std::env::{", + "use std::env::var", + "use std::env;", + ] { + for (n, line) in shipped.lines().enumerate() { + if !line.trim_start().starts_with("//") && line.contains(pat) { + env_reads.push(format!("{name}:{}: {}", n + 1, line.trim())); + } + } + } + } + // Browser discovery reads PATH, and that is the only read of the environment. + assert_eq!(env_reads.len(), 1, "{env_reads:?}"); + assert!( + env_reads[0].starts_with("browser.rs:") && env_reads[0].contains("var_os(\"PATH\")"), + "{env_reads:?}" + ); +} + +/// A `kharon-teach-*` profile directory in the temp directory whose owner file holds `pid`. +fn profile_of(pid: u32) -> Option<PathBuf> { + std::fs::read_dir(std::env::temp_dir()) + .ok()? + .flatten() + .filter(|e| e.file_name().to_string_lossy().starts_with("kharon-teach-")) + .map(|e| e.path()) + .find(|p| { + std::fs::read_to_string(p.join("kharon-teach.owner")) + .is_ok_and(|t| t.trim() == pid.to_string()) + }) +} + +/// Runs the built program against a browser that never opens a DevTools port, sends it `signal`, +/// and checks: exit code 1, the profile removed, no file written, all within 10 s. +fn interrupted_by(signal: &str) { + use std::os::unix::fs::PermissionsExt; + let dir = std::env::temp_dir().join(format!( + "kharon-sig-{}-{:08x}", + std::process::id(), + rand::random::<u32>() + )); + std::fs::create_dir(&dir).unwrap(); + let fake = dir.join("fake-browser"); + // The shell stays, with the switches on its command line, as a real browser's would be. + std::fs::write( + &fake, + "#!/bin/sh\nexec sh -c 'sleep 30; true' fake-browser \"$@\"\n", + ) + .unwrap(); + std::fs::set_permissions(&fake, std::fs::Permissions::from_mode(0o755)).unwrap(); + let out = dir.join("login.kharon-recording.json"); + let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_kharon-teach")) + .args(["--yes", "https://ex.test/login", "--browser"]) + .arg(&fake) + .arg("--out") + .arg(&out) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let pid = child.id(); + let limit = std::time::Instant::now() + Duration::from_secs(10); + let profile = loop { + if let Some(p) = profile_of(pid) { + break p; + } + assert!( + std::time::Instant::now() < limit, + "the program made no profile in 10 s" + ); + std::thread::sleep(Duration::from_millis(20)); + }; + let sent = std::process::Command::new("kill") + .args([&format!("-{signal}"), &pid.to_string()]) + .status() + .unwrap(); + assert!(sent.success()); + let status = loop { + if let Some(s) = child.try_wait().unwrap() { + break s; + } + if std::time::Instant::now() >= limit + Duration::from_secs(10) { + let _ = child.kill(); + panic!("the program did not stop after {signal}"); + } + std::thread::sleep(Duration::from_millis(20)); + }; + assert_eq!(status.code(), Some(1), "exit status after {signal}"); + let mut err = String::new(); + std::io::Read::read_to_string(child.stderr.as_mut().unwrap(), &mut err).unwrap(); + assert!(err.contains("Interrupted. Nothing was saved."), "{err}"); + assert!(!profile.exists(), "the profile is still there"); + assert!(!out.exists(), "a file was written"); + assert!( + no_process_on(&profile), + "a process on the profile is still running after {signal}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +/// True when no process has `--user-data-dir=<profile>` on its command line. Polls up to 5 s. +fn no_process_on(profile: &Path) -> bool { + let needle = format!("--user-data-dir={}", profile.display()); + let limit = std::time::Instant::now() + Duration::from_secs(5); + loop { + let any = std::fs::read_dir("/proc").unwrap().flatten().any(|e| { + std::fs::read(e.path().join("cmdline")) + .is_ok_and(|c| c.split(|b| *b == 0).any(|arg| arg == needle.as_bytes())) + }); + if !any { + return true; + } + if std::time::Instant::now() >= limit { + return false; + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +#[test] +fn sigint_exits_1_and_removes_the_profile() { + interrupted_by("INT"); +} + +#[test] +fn sigterm_exits_1_and_removes_the_profile() { + interrupted_by("TERM"); +} + +#[test] +fn sighup_exits_1_and_removes_the_profile() { + interrupted_by("HUP"); +} + +#[test] +fn sigquit_exits_1_and_removes_the_profile() { + interrupted_by("QUIT"); +} + +async fn outcome_is_unconfirmed_or_never_started(outcome: Outcome) { + match outcome { + Outcome::Recorded(r) => { + assert_eq!(r.ended_by, EndedBy::WindowClosed); + assert!(!r.success_confirmed); + } + Outcome::NeverStarted => {} + Outcome::Discarded => panic!("a dead browser is not a discard"), + } +} + +#[tokio::test] +async fn a_browser_killed_from_outside_ends_the_session_and_the_profile_stays_gone() { + let Some(mut run) = Run::start("killed_outside", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + type_a_login(&run).await; + let pid = support::Person::browser_pid(&profile).expect("the browser process"); + let status = std::process::Command::new("kill") + .args(["-KILL", &pid.to_string()]) + .status() + .unwrap(); + assert!(status.success()); + outcome_is_unconfirmed_or_never_started(run.wait().await).await; + assert!(gone(&profile).await, "the profile is still there"); +} + +#[tokio::test] +async fn a_browser_that_quits_by_itself_ends_the_session_and_the_profile_stays_gone() { + let Some(mut run) = Run::start("quits_itself", "/form").await else { + return; + }; + let profile = run.ready.profile_dir.clone(); + type_a_login(&run).await; + run.person.close_browser().await; + outcome_is_unconfirmed_or_never_started(run.wait().await).await; + assert!(gone(&profile).await, "the profile is still there"); +} + +/// The shipped binary, past `Ready`, in a browser without a display. +struct Live { + child: std::process::Child, + /// Held open and, unless a test writes an answer, never written to: end of input must not + /// end the recording. + stdin: std::process::ChildStdin, + stdout: std::sync::Arc<std::sync::Mutex<String>>, + dir: PathBuf, + out: PathBuf, + profile: PathBuf, + _fixture: support::Fixture, + _permit: tokio::sync::SemaphorePermit<'static>, +} + +impl Live { + /// `None` when there is no browser and none is required. + async fn start(test: &str) -> Option<Live> { + use std::io::Read; + use std::os::unix::fs::PermissionsExt; + let chromium = support::chromium_or_skip(test)?; + let permit = support::browser_permit().await; + let fixture = support::Fixture::start().await; + let dir = std::env::temp_dir().join(format!( + "kharon-live-{}-{:08x}", + std::process::id(), + rand::random::<u32>() + )); + std::fs::create_dir(&dir).unwrap(); + // Only this test wrapper adds the switch; the program believes it launched a visible browser. + let wrapper = dir.join("browser"); + std::fs::write( + &wrapper, + format!( + "#!/bin/sh\nexec {} --headless=new \"$@\"\n", + chromium.display() + ), + ) + .unwrap(); + std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o755)).unwrap(); + let out = dir.join("login.kharon-recording.json"); + let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_kharon-teach")) + .args(["--yes", fixture.url("/form").as_str(), "--browser"]) + .arg(&wrapper) + .arg("--out") + .arg(&out) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap(); + let stdin = child.stdin.take().unwrap(); + let stdout = std::sync::Arc::new(std::sync::Mutex::new(String::new())); + let mut pipe = child.stdout.take().unwrap(); + let sink = stdout.clone(); + std::thread::spawn(move || { + let mut buf = [0u8; 1024]; + while let Ok(n) = pipe.read(&mut buf) { + if n == 0 { + break; + } + sink.lock() + .unwrap() + .push_str(&String::from_utf8_lossy(&buf[..n])); + } + }); + let live = Live { + profile: PathBuf::new(), + child, + stdin, + stdout, + dir, + out, + _fixture: fixture, + _permit: permit, + }; + let mut live = live; + let limit = std::time::Instant::now() + Duration::from_secs(40); + while !live + .printed() + .contains("Press Enter once you are logged in") + { + assert!( + std::time::Instant::now() < limit, + "the end prompt never showed" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + live.profile = profile_of(live.child.id()).expect("the profile exists"); + Some(live) + } + + fn printed(&self) -> String { + self.stdout.lock().unwrap().clone() + } + + /// The exit code, or `None` when the program is still running after `secs`. + async fn exit_within(&mut self, secs: u64) -> Option<Option<i32>> { + let limit = std::time::Instant::now() + Duration::from_secs(secs); + loop { + if let Some(s) = self.child.try_wait().unwrap() { + return Some(s.code()); + } + if std::time::Instant::now() >= limit { + return None; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } +} + +impl Drop for Live { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +/// After the prompt: send `signal`, expect exit 1 without Enter, no file, no profile and no +/// browser process left. +async fn signal_after_the_prompt(signal: &str, test: &str) { + let Some(mut live) = Live::start(test).await else { + return; + }; + let pid = live.child.id().to_string(); + assert!( + std::process::Command::new("kill") + .args([&format!("-{signal}"), &pid]) + .status() + .unwrap() + .success() + ); + assert_eq!( + live.exit_within(10).await, + Some(Some(1)), + "the program did not exit on SIG{signal} without Enter" + ); + assert!(!live.out.exists(), "a file was written"); + assert!(gone(&live.profile).await, "the profile is still there"); + assert!( + no_process_on(&live.profile), + "a browser process is still running after SIG{signal}" + ); +} + +#[tokio::test] +async fn sigint_after_the_prompt_exits_1_without_enter() { + signal_after_the_prompt("INT", "sigint_after_prompt").await; +} + +#[tokio::test] +async fn sighup_after_the_prompt_exits_1_and_leaves_no_browser() { + signal_after_the_prompt("HUP", "sighup_after_prompt").await; +} + +#[tokio::test] +async fn sigquit_after_the_prompt_exits_1_and_leaves_no_browser() { + signal_after_the_prompt("QUIT", "sigquit_after_prompt").await; +} + +#[tokio::test] +async fn a_line_that_is_not_an_answer_asks_again_and_quit_discards() { + use std::io::Write; + let Some(mut live) = Live::start("end_prompt_retry").await else { + return; + }; + for not_an_answer in ["no\n", "discard\n"] { + let before = live + .printed() + .matches("Press Enter to save, or type q to discard.") + .count(); + live.stdin.write_all(not_an_answer.as_bytes()).unwrap(); + live.stdin.flush().unwrap(); + let limit = std::time::Instant::now() + Duration::from_secs(10); + while live + .printed() + .matches("Press Enter to save, or type q to discard.") + .count() + == before + { + assert!( + std::time::Instant::now() < limit, + "no new prompt after {not_an_answer:?}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + // Still recording: nothing was confirmed or discarded. + assert_eq!( + live.exit_within(1).await, + None, + "{not_an_answer:?} ended it" + ); + } + live.stdin.write_all(b"QUIT\n").unwrap(); + live.stdin.flush().unwrap(); + assert_eq!(live.exit_within(10).await, Some(Some(0))); + assert!(live.printed().contains("Discarded. Nothing was saved.")); + assert!(!live.out.exists(), "a file was written"); + assert!(gone(&live.profile).await, "the profile is still there"); +} + +#[tokio::test] +async fn closing_the_window_after_the_prompt_saves_and_exits_without_enter() { + use std::os::unix::fs::PermissionsExt; + let Some(mut live) = Live::start("close_after_prompt").await else { + return; + }; + let port: u16 = std::fs::read_to_string(live.profile.join("DevToolsActivePort")) + .unwrap() + .lines() + .next() + .unwrap() + .parse() + .unwrap(); + let person = support::Person::connect(port).await; + person.click("a").await; + person.wait_for_path("/form").await; + person.type_into("input[name=user]", USER).await; + person.type_into("input[name=pass]", PASS).await; + person.close_all_pages().await; + assert_eq!( + live.exit_within(10).await, + Some(Some(0)), + "the program did not exit after the window closed" + ); + assert!(live.printed().contains("Saved:"), "{}", live.printed()); + let mode = std::fs::metadata(&live.out).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + let rec = kharon_recording::Recording::from_json(&std::fs::read_to_string(&live.out).unwrap()) + .unwrap(); + assert_eq!(rec.ended_by, EndedBy::WindowClosed); + assert!(!rec.success_confirmed); +} diff --git a/crates/kharon-teach/tests/support/mod.rs b/crates/kharon-teach/tests/support/mod.rs new file mode 100644 index 0000000..7ecf853 --- /dev/null +++ b/crates/kharon-teach/tests/support/mod.rs @@ -0,0 +1,1022 @@ +// Shared by several test files; each uses only some of these items. +#![allow(dead_code)] + +use std::path::PathBuf; + +pub fn chromium_bin() -> Option<PathBuf> { + if let Ok(p) = std::env::var("KHARON_CHROMIUM") { + return Some(PathBuf::from(p)); + } + [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/local/bin/google-chrome", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + ] + .into_iter() + .map(PathBuf::from) + .find(|p| p.exists()) +} + +/// The Chromium binary, or `None` to skip (unless a run requires it). +/// Same search as `chromium_bin` in `crates/kharond/tests/req_arch_003_org_chromium.rs`. +pub fn chromium_or_skip(test: &str) -> Option<PathBuf> { + let bin = chromium_bin(); + if bin.is_none() { + if std::env::var_os("KHARON_REQUIRE_CHROMIUM").is_some() { + panic!("KHARON_REQUIRE_CHROMIUM is set but no Chromium binary was found"); + } + eprintln!("SKIP {test}: no Chromium binary"); + } + bin +} + +// ---- the fixture login server, the person at the keyboard, and one recording run ---- + +use kharon_teach::browser::DevTools; +use kharon_teach::cdp::Cdp; +use kharon_teach::session::{ + End, Outcome, Ready, RecordOptions, SessionError, WorldProbe, record_with, +}; +use serde::{Deserialize, Serialize}; +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::{Semaphore, oneshot}; +use tokio::task::JoinHandle; + +pub const USER: &str = "kh.throwaway7"; +pub const PASS: &str = "Tr0ub4dor-9x"; +/// Set by the fixture on a successful login; must never reach a recording. +pub const COOKIE_VALUE: &str = "COOKIEVALUE123"; + +/// Browsers running at once. Parallel Chromium instances on a small machine make timing flaky. +static GATE: Semaphore = Semaphore::const_new(2); + +const FORM: &str = "<!doctype html><title>form
\ +\ +
"; + +const SCRIPT: &str = "script\ +"; + +const HASHED: &str = "hashed\ +"; + +const TWO_STEP: &str = "step one
\ +
"; + +const TWO_STEP_PASSWORD: &str = "step two
\ +
"; + +const SCRIPT_FORMDATA: &str = "formdata\ +"; + +const SCRIPT_BLOB: &str = "blob\ +"; + +/// A form that copies what is typed into the user field into the field's own `id`. +const MIRROR: &str = "mirror
\ +
\ +"; + +/// A form whose children replace `action`, `method`, `id` and `tagName` on the form element. +const CLOBBER: &str = "clobber
\ +\ +
"; + +const POPUP: &str = "popup\ +"; + +/// `{HOST}` is the same server under another host name, so it is another origin. +const FRAME_OUTER: &str = "outer\ +"; + +/// Puts the typed values into text the recording keeps: a JSON key, and the next URL's path and +/// query key. +const TYPED_IN_TEXT: &str = "typed\ +"; + +const HOME: &str = "home

signed in

"; + +fn response(status: &str, extra: &str, body: &str, ctype: &str) -> String { + format!( + "HTTP/1.1 {status}\r\n{extra}Content-Type: {ctype}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) +} + +fn route(method: &str, path: &str, host: &str) -> String { + let html = "text/html; charset=utf-8"; + match (method, path) { + ("GET", "/form") => response("200 OK", "", FORM, html), + ("POST", "/form-login") => response( + "302 Found", + &format!("Location: /home\r\nSet-Cookie: sid={COOKIE_VALUE}; Path=/\r\n"), + "", + html, + ), + ("GET", "/script") => response("200 OK", "", SCRIPT, html), + ("POST", "/api/login") => response("200 OK", "", "{\"ok\":true}", "application/json"), + ("GET", "/two-step") => response("200 OK", "", TWO_STEP, html), + ("POST", "/two-step/password") => response("200 OK", "", TWO_STEP_PASSWORD, html), + ("GET", "/script-formdata") => response("200 OK", "", SCRIPT_FORMDATA, html), + ("POST", "/api/login-form") => response("200 OK", "", "{\"ok\":true}", "application/json"), + ("GET", "/script-blob") => response("200 OK", "", SCRIPT_BLOB, html), + ("POST", "/api/login-blob") => response("200 OK", "", "{\"ok\":true}", "application/json"), + ("GET", "/mirror") => response("200 OK", "", MIRROR, html), + ("GET", "/clobber") => response("200 OK", "", CLOBBER, html), + ("GET", "/popup") => response("200 OK", "", POPUP, html), + ("GET", "/frame-outer") => { + // Same server, another host name: a different origin. + let other = host.replace("127.0.0.1", "localhost"); + response("200 OK", "", &FRAME_OUTER.replace("{HOST}", &other), html) + } + ("GET", "/hashed") => response("200 OK", "", HASHED, html), + ("GET", "/typed") => response("200 OK", "", TYPED_IN_TEXT, html), + ("POST", "/api/typed") => response("200 OK", "", "{\"ok\":true}", "application/json"), + ("GET", p) if p.starts_with("/u/") => response("200 OK", "", HOME, html), + ("GET", "/home") => response("200 OK", "", HOME, html), + _ => response("404 Not Found", "", "not found", "text/plain"), + } +} + +async fn serve_one(mut sock: tokio::net::TcpStream) { + let mut buf = Vec::new(); + let mut chunk = [0u8; 4096]; + let head_end = loop { + if let Some(i) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + break i + 4; + } + match sock.read(&mut chunk).await { + Ok(0) | Err(_) => return, + Ok(n) => buf.extend_from_slice(&chunk[..n]), + } + }; + let head = String::from_utf8_lossy(&buf[..head_end]).into_owned(); + let mut first = head.lines().next().unwrap_or("").split_whitespace(); + let (method, path) = (first.next().unwrap_or(""), first.next().unwrap_or("")); + let want: usize = head + .lines() + .find_map(|l| { + l.to_ascii_lowercase() + .strip_prefix("content-length:") + .and_then(|v| v.trim().parse().ok()) + }) + .unwrap_or(0); + while buf.len() < head_end + want { + match sock.read(&mut chunk).await { + Ok(0) | Err(_) => break, + Ok(n) => buf.extend_from_slice(&chunk[..n]), + } + } + let path = path.split('?').next().unwrap_or(""); + let host = head + .lines() + .find_map(|l| { + l.to_ascii_lowercase() + .starts_with("host:") + .then(|| l[5..].trim().to_owned()) + }) + .unwrap_or_default(); + let _ = sock.write_all(route(method, path, &host).as_bytes()).await; + let _ = sock.shutdown().await; +} + +pub struct Fixture { + pub port: u16, + task: JoinHandle<()>, +} + +impl Fixture { + pub async fn start() -> Fixture { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let task = tokio::spawn(async move { + while let Ok((sock, _)) = listener.accept().await { + tokio::spawn(serve_one(sock)); + } + }); + Fixture { port, task } + } + + pub fn url(&self, path: &str) -> url::Url { + url::Url::parse(&format!("http://127.0.0.1:{}{path}", self.port)).unwrap() + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + self.task.abort(); + } +} + +// ---- the person ---- + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct VersionDoc { + web_socket_debugger_url: String, +} + +#[derive(Deserialize)] +struct Targets { + #[serde(rename = "targetInfos")] + infos: Vec, +} + +#[derive(Deserialize)] +struct TargetRow { + #[serde(rename = "targetId")] + id: String, + #[serde(rename = "type")] + kind: String, + #[serde(default)] + url: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct TargetIdParam<'a> { + target_id: &'a str, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct AttachParam<'a> { + target_id: &'a str, + flatten: bool, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Attached { + session_id: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct EvalParam<'a> { + expression: &'a str, + return_by_value: bool, +} + +#[derive(Deserialize)] +struct EvalResult { + result: EvalValue, +} + +#[derive(Deserialize)] +struct EvalValue { + #[serde(default)] + value: Option, +} + +#[derive(Serialize)] +struct TextParam<'a> { + text: &'a str, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct MouseParam { + #[serde(rename = "type")] + kind: &'static str, + x: f64, + y: f64, + button: &'static str, + click_count: u32, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct KeyParam { + #[serde(rename = "type")] + kind: &'static str, + key: &'static str, + code: &'static str, + windows_virtual_key_code: u32, + #[serde(skip_serializing_if = "Option::is_none")] + text: Option<&'static str>, +} + +#[derive(Serialize)] +struct UrlParam<'a> { + url: &'a str, +} + +#[derive(Deserialize)] +struct Ignored {} + +async fn get_version_doc(port: u16) -> VersionDoc { + let mut s = tokio::net::TcpStream::connect(("127.0.0.1", port)) + .await + .unwrap(); + s.write_all(b"GET /json/version HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + let mut text = String::new(); + let mut buf = [0u8; 4096]; + tokio::time::timeout(Duration::from_secs(10), async { + while !text.trim_end().ends_with('}') { + let n = s.read(&mut buf).await.unwrap(); + assert!(n > 0, "closed early"); + text.push_str(&String::from_utf8_lossy(&buf[..n])); + } + }) + .await + .expect("no /json/version answer"); + let body = &text[text.find("\r\n\r\n").unwrap() + 4..]; + serde_json::from_str(body).unwrap() +} + +/// A second DevTools connection that does what a person does in the window. +pub struct Person { + cdp: Cdp, + session: String, +} + +impl Person { + pub async fn connect(port: u16) -> Person { + let doc = get_version_doc(port).await; + let at = doc.web_socket_debugger_url.find("/devtools/").unwrap(); + let dev = DevTools { + port, + ws_path: doc.web_socket_debugger_url[at..].to_owned(), + }; + let (cdp, rx) = Cdp::connect(&dev).await.unwrap(); + // Nobody reads this connection's events; keep the channel open. + std::mem::forget(rx); + let page = Self::first_page(&cdp).await.expect("no page target"); + let session: Attached = cdp + .call( + "Target.attachToTarget", + &AttachParam { + target_id: &page, + flatten: true, + }, + None, + ) + .await + .unwrap(); + Person { + cdp, + session: session.session_id, + } + } + + async fn first_page(cdp: &Cdp) -> Option { + let t: Targets = cdp + .call("Target.getTargets", &Ignored2 {}, None) + .await + .ok()?; + t.infos.into_iter().find(|t| t.kind == "page").map(|t| t.id) + } + + pub async fn eval(&self, expression: &str) -> Option { + let r: EvalResult = self + .cdp + .call( + "Runtime.evaluate", + &EvalParam { + expression, + return_by_value: true, + }, + Some(&self.session), + ) + .await + .ok()?; + r.result.value + } + + pub async fn goto(&self, url: &str) { + let _: Ignored = self + .cdp + .call("Page.navigate", &UrlParam { url }, Some(&self.session)) + .await + .unwrap(); + } + + pub async fn type_into(&self, css: &str, text: &str) { + let sel = serde_json::to_string(css).unwrap(); + let focus = format!("(()=>{{document.querySelector({sel}).focus();return 'ok'}})()"); + assert_eq!( + self.eval(&focus).await.as_deref(), + Some("ok"), + "focus {css}" + ); + let _: Ignored = self + .cdp + .call("Input.insertText", &TextParam { text }, Some(&self.session)) + .await + .unwrap(); + } + + pub async fn press_enter(&self) { + for (kind, text) in [("keyDown", Some("\r")), ("keyUp", None)] { + let _: Ignored = self + .cdp + .call( + "Input.dispatchKeyEvent", + &KeyParam { + kind, + key: "Enter", + code: "Enter", + windows_virtual_key_code: 13, + text, + }, + Some(&self.session), + ) + .await + .unwrap(); + } + } + + pub async fn click(&self, css: &str) { + let sel = serde_json::to_string(css).unwrap(); + let centre = format!( + "(()=>{{const r=document.querySelector({sel}).getBoundingClientRect();\ + return (r.x+r.width/2)+','+(r.y+r.height/2)}})()" + ); + let at = self.eval(¢re).await.expect("element centre"); + let (x, y) = at.split_once(',').unwrap(); + let (x, y): (f64, f64) = (x.parse().unwrap(), y.parse().unwrap()); + for (kind, count) in [("mousePressed", 1), ("mouseReleased", 1)] { + let _: Ignored = self + .cdp + .call( + "Input.dispatchMouseEvent", + &MouseParam { + kind, + x, + y, + button: "left", + click_count: count, + }, + Some(&self.session), + ) + .await + .unwrap(); + } + } + + pub async fn wait_for_path(&self, path: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + if self.eval("location.pathname").await.as_deref() == Some(path) { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "the page never reached {path}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + pub async fn close_all_pages(&self) { + let t: Targets = self + .cdp + .call("Target.getTargets", &Ignored2 {}, None) + .await + .unwrap(); + for p in t.infos.into_iter().filter(|t| t.kind == "page") { + let _: Result = self + .cdp + .call( + "Target.closeTarget", + &TargetIdParam { target_id: &p.id }, + None, + ) + .await; + } + } +} + +#[derive(Serialize)] +struct Ignored2 {} + +// ---- one recording run ---- + +pub struct Run { + pub fixture: Fixture, + pub person: Person, + pub ready: Ready, + end: Option>, + task: JoinHandle>, + _permit: tokio::sync::SemaphorePermit<'static>, +} + +impl Run { + /// Starts `record` headless on the fixture's `login_path` and waits until the notice page is + /// showing. `None` when there is no browser and the run does not require one. + pub async fn start(test: &str, login_path: &str) -> Option { + Self::start_with(test, login_path, WorldProbe::Normal).await + } + + /// [`Run::start`] with a way to make the isolated world fail. + pub async fn start_with(test: &str, login_path: &str, probe: WorldProbe) -> Option { + let bin = chromium_or_skip(test)?; + let permit = GATE.acquire().await.unwrap(); + let fixture = Fixture::start().await; + let login = fixture.url(login_path); + let (ready_tx, ready_rx) = oneshot::channel(); + let (end_tx, end_rx) = oneshot::channel(); + let task = tokio::spawn(async move { + record_with( + RecordOptions { + login_url: &login, + browser: &bin, + headless: true, + tool_version: "test", + }, + ready_tx, + end_rx, + probe, + ) + .await + }); + let ready = match tokio::time::timeout(Duration::from_secs(40), ready_rx) + .await + .expect("the session was not ready in 40 s") + { + Ok(r) => r, + Err(_) => { + let ended = task.await.expect("the session task panicked"); + match ended { + Err(e) => panic!("the session failed before it was ready: {e:?}"), + Ok(Outcome::NeverStarted) => panic!("the window closed before it was ready"), + Ok(_) => panic!("the session ended before it was ready"), + } + } + }; + let person = Person::connect(ready.devtools_port).await; + Some(Run { + fixture, + person, + ready, + end: Some(end_tx), + task, + _permit: permit, + }) + } + + /// What a person does: click the notice page's link, then the login page is open. + pub async fn begin(&self, login_path: &str) { + self.person.click("a").await; + self.person.wait_for_path(login_path).await; + } + + pub async fn finish(mut self, how: End) -> Outcome { + if let Some(tx) = self.end.take() { + let _ = tx.send(how); + } + self.wait().await + } + + /// The outcome of a session that ends by itself (the window closing). + pub async fn wait(&mut self) -> Outcome { + tokio::time::timeout(Duration::from_secs(10), &mut self.task) + .await + .expect("the session did not end in 10 s") + .expect("the session task panicked") + .expect("the session failed") + } + + pub fn abort(&self) { + self.task.abort(); + } +} + +/// The four fixture logins. +#[derive(Clone, Copy)] +pub enum Login { + Form, + Script, + TwoStep, + Hashed, + ScriptFormData, + ScriptBlob, + Mirror, + Clobber, +} + +impl Login { + pub fn path(self) -> &'static str { + match self { + Login::Form => "/form", + Login::Script => "/script", + Login::TwoStep => "/two-step", + Login::Hashed => "/hashed", + Login::ScriptFormData => "/script-formdata", + Login::ScriptBlob => "/script-blob", + Login::Mirror => "/mirror", + Login::Clobber => "/clobber", + } + } +} + +/// Starts a run, does what a person does on `which`, confirms, and hands back the recording. +/// `None` when the run is skipped for want of a browser. +pub async fn record_login(test: &str, which: Login) -> Option { + let run = Run::start(test, which.path()).await?; + run.begin(which.path()).await; + let p = &run.person; + match which { + Login::Mirror => { + p.type_into("input[name=user]", USER).await; + p.type_into("input[name=pass]", PASS).await; + p.press_enter().await; + } + Login::Form + | Login::Script + | Login::Hashed + | Login::ScriptFormData + | Login::ScriptBlob + | Login::Clobber => { + p.type_into("input[name=user]", USER).await; + p.type_into("input[name=pass]", PASS).await; + p.click("#go").await; + } + Login::TwoStep => { + p.type_into("input[name=user]", USER).await; + p.click("#go").await; + p.wait_for_path("/two-step/password").await; + p.type_into("input[name=pass]", PASS).await; + p.click("#go").await; + } + } + p.wait_for_path("/home").await; + match run.finish(End::Confirmed).await { + Outcome::Recorded(r) => Some(*r), + _ => panic!("expected a recording"), + } +} + +impl Person { + /// The `attr` attribute of the element `css` matches, or `None` when it matches nothing. + pub async fn attr_of(&self, css: &str, attr: &str) -> Option { + let sel = serde_json::to_string(css).unwrap(); + let attr = serde_json::to_string(attr).unwrap(); + self.eval(&format!( + "(()=>{{const e=document.querySelector({sel});return e?e.getAttribute({attr}):null}})()" + )) + .await + } +} + +// ---- the whole program, driven through `run_with` ---- + +/// A cloneable in-memory writer; the program gets one clone and the test reads the other. +#[derive(Clone, Default)] +pub struct Shared(std::sync::Arc>>); + +impl Shared { + pub fn text(&self) -> String { + String::from_utf8_lossy(&self.0.lock().unwrap()).into_owned() + } +} + +impl std::io::Write for Shared { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// What a whole run of the program left behind. +pub struct Program { + pub exit: kharon_teach::Exit, + pub path: PathBuf, + pub file_text: String, + pub file_mode: u32, + pub stdout: String, + pub stderr: String, +} + +/// Runs the program headless against the fixture's `/form` login (the path may carry a query), +/// as a person would: says `yes`, opens the login page from the notice, types the throwaway +/// values, signs in, presses Enter. `None` when there is no browser and none is required. +pub async fn run_program(test: &str, form_path_and_query: &str) -> Option { + use clap::Parser; + use std::os::unix::fs::PermissionsExt; + let bin = chromium_or_skip(test)?; + let _permit = GATE.acquire().await.unwrap(); + let fixture = Fixture::start().await; + let login = fixture.url(form_path_and_query); + let dir = std::env::temp_dir().join(format!( + "kharon-prog-{}-{:08x}", + std::process::id(), + rand::random::() + )); + std::fs::create_dir(&dir).unwrap(); + let out = dir.join("login.kharon-recording.json"); + let cli = kharon_teach::cli::Cli::try_parse_from([ + "kharon-teach", + login.as_str(), + "--browser", + bin.to_str().unwrap(), + "--out", + out.to_str().unwrap(), + ]) + .unwrap(); + let (stdout, stderr) = (Shared::default(), Shared::default()); + let (ready_tx, ready_rx) = oneshot::channel(); + let (mut keys, stdin) = tokio::io::duplex(1024); + let task = tokio::spawn({ + let (stdout, stderr) = (stdout.clone(), stderr.clone()); + async move { + kharon_teach::run_with( + cli, + tokio::io::BufReader::new(stdin), + stdout, + stderr, + kharon_teach::Hooks { + headless: true, + ready: Some(ready_tx), + }, + ) + .await + } + }); + keys.write_all(b"yes\n").await.unwrap(); + let ready = tokio::time::timeout(Duration::from_secs(40), ready_rx) + .await + .expect("the program was not ready in 40 s") + .expect("the program ended before it was ready"); + let person = Person::connect(ready.devtools_port).await; + person.click("a").await; + person.wait_for_path(path_of(form_path_and_query)).await; + person.type_into("input[name=user]", USER).await; + person.type_into("input[name=pass]", PASS).await; + person.click("#go").await; + person.wait_for_path("/home").await; + keys.write_all(b"\n").await.unwrap(); + let exit = tokio::time::timeout(Duration::from_secs(20), task) + .await + .expect("the program did not end in 20 s") + .expect("the program panicked"); + let file_text = std::fs::read_to_string(&out).unwrap_or_default(); + let file_mode = std::fs::metadata(&out) + .map(|m| m.permissions().mode() & 0o777) + .unwrap_or(0); + let _ = std::fs::remove_dir_all(&dir); + Some(Program { + exit, + path: out, + file_text, + file_mode, + stdout: stdout.text(), + stderr: stderr.text(), + }) +} + +fn path_of(path_and_query: &str) -> &str { + path_and_query.split('?').next().unwrap_or("") +} + +#[derive(Serialize)] +struct FlattenAttach<'a> { + #[serde(rename = "targetId")] + target_id: &'a str, + flatten: bool, +} + +impl Person { + /// How many page targets the browser has. + pub async fn page_count(&self) -> usize { + let t: Targets = self + .cdp + .call("Target.getTargets", &Ignored2 {}, None) + .await + .unwrap(); + t.infos.iter().filter(|t| t.kind == "page").count() + } + + pub async fn wait_for_pages(&self, n: usize) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while self.page_count().await < n { + assert!(tokio::time::Instant::now() < deadline, "no page number {n}"); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + /// The browser closes itself, as when someone quits it from its menu. + pub async fn close_browser(&self) { + let _: Result = self.cdp.call("Browser.close", &Ignored2 {}, None).await; + } + + /// A session on the page's embedded frame whose URL is on `host`, waiting for it to exist. + /// The frame is in another process, so the page's own session cannot reach into it. + pub async fn attach_frame(&self, host: &str) -> String { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + let t: Targets = self + .cdp + .call("Target.getTargets", &Ignored2 {}, None) + .await + .unwrap(); + if let Some(f) = t + .infos + .iter() + .find(|t| t.kind == "iframe" && t.url.contains(host)) + { + let a: Attached = self + .cdp + .call( + "Target.attachToTarget", + &FlattenAttach { + target_id: &f.id, + flatten: true, + }, + None, + ) + .await + .unwrap(); + return a.session_id; + } + assert!( + tokio::time::Instant::now() < deadline, + "no frame target for {host}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + async fn eval_in(&self, session: &str, expression: &str) -> Option { + let r: EvalResult = self + .cdp + .call( + "Runtime.evaluate", + &EvalParam { + expression, + return_by_value: true, + }, + Some(session), + ) + .await + .ok()?; + r.result.value + } + + /// Focus `css` inside the frame, then type into it. + pub async fn type_into_frame(&self, frame: &str, css: &str, text: &str) { + let sel = serde_json::to_string(css).unwrap(); + let focus = format!( + "(()=>{{const e=document.querySelector({sel});if(!e)return 'no';e.focus();return 'ok'}})()" + ); + // The frame's document may not be there yet when its target first appears. + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while self.eval_in(frame, &focus).await.as_deref() != Some("ok") { + assert!( + tokio::time::Instant::now() < deadline, + "the field {css} never appeared in the frame" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + let _: Ignored = self + .cdp + .call("Input.insertText", &TextParam { text }, Some(frame)) + .await + .unwrap(); + } + + pub async fn wait_for_frame_path(&self, frame: &str, path: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + if self.eval_in(frame, "location.pathname").await.as_deref() == Some(path) { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "the frame never reached {path}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + /// The pid of the main browser process of the profile at `profile`, found through `/proc`. + pub fn browser_pid(profile: &std::path::Path) -> Option { + let want = format!("--user-data-dir={}", profile.display()); + for e in std::fs::read_dir("/proc").ok()?.flatten() { + let Ok(pid) = e.file_name().to_string_lossy().parse::() else { + continue; + }; + let Ok(cmd) = std::fs::read(e.path().join("cmdline")) else { + continue; + }; + // Chrome rewrites its own argv into one space-separated string; no path here has a space. + let args: Vec<&[u8]> = cmd.split(|b| *b == 0 || *b == b' ').collect(); + if args.contains(&want.as_bytes()) && !args.iter().any(|a| a.starts_with(b"--type=")) { + return Some(pid); + } + } + None + } +} + +// ---- shared by tests that drive or scan the shipped program ---- + +/// Waits for one of the browser slots (see `GATE`); hold the permit while a browser runs. +pub async fn browser_permit() -> tokio::sync::SemaphorePermit<'static> { + GATE.acquire().await.unwrap() +} + +/// Every `.rs` file under `dir`. +pub fn rust_files(dir: &std::path::Path) -> Vec { + let mut out = Vec::new(); + for e in std::fs::read_dir(dir).unwrap().flatten() { + let p = e.path(); + if p.is_dir() { + out.extend(rust_files(&p)); + } else if p.extension().is_some_and(|x| x == "rs") { + out.push(p); + } + } + out +} + +/// The shipped part of a source file: the text before its test MODULE (`#[cfg(test)]` followed +/// by `mod tests`, with any visibility). A `#[cfg(test)]` item anywhere before it would hide +/// shipped code from every scan, so that is an error. +pub fn split_shipped(text: &str) -> Result<&str, String> { + let mut from = 0; + while let Some(at) = text[from..].find("#[cfg(test)]") { + let at = from + at; + let after = text[at + "#[cfg(test)]".len()..].trim_start(); + let is_module = ["mod tests", "pub mod tests", "pub(crate) mod tests"] + .iter() + .any(|m| after.starts_with(m)); + if is_module { + return if text[..at].contains("#[cfg(test)]") { + Err("a #[cfg(test)] item sits above the test module".to_owned()) + } else { + Ok(&text[..at]) + }; + } + from = at + 1; + } + // No test module: the whole file is shipped, so it must have no test-only item either. + if text.contains("#[cfg(test)]") { + Err("a #[cfg(test)] item with no test module after it".to_owned()) + } else { + Ok(text) + } +} + +/// Shipped code only: for each file under `crates/kharon-teach/src`, the text before its test +/// module (see [`split_shipped`]). Panics when a `#[cfg(test)]` item sits above that module. +pub fn shipped_sources() -> Vec<(PathBuf, String)> { + let files = rust_files(&std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src")); + assert!(!files.is_empty()); + files + .into_iter() + .map(|p| { + let text = std::fs::read_to_string(&p).unwrap(); + let shipped = match split_shipped(&text) { + Ok(s) => s.to_owned(), + Err(why) => panic!("{}: {why}", p.display()), + }; + (p, shipped) + }) + .collect() +} diff --git a/crates/kharond/src/cdp/page_js.rs b/crates/kharond/src/cdp/page_js.rs index 6e644cb..4d881a0 100644 --- a/crates/kharond/src/cdp/page_js.rs +++ b/crates/kharond/src/cdp/page_js.rs @@ -121,6 +121,7 @@ function fillInPage(specs, fields, expectedTopOrigin, submitRequested, budgetMs) function resolveField(kind, selector) { if (selector) return document.querySelector(selector); if (kind === "password") return document.querySelector('input[type="password"]'); + // Keep in step with is_otp_descriptor in crates/kharon-recording/src/classify.rs. if (kind === "totp") { return document.querySelector( 'input[autocomplete*="one-time"], input[name*="otp" i], input[type="tel"]' @@ -234,6 +235,7 @@ function clearInPage(fields) { function resolveField(kind, selector) { if (selector) return document.querySelector(selector); if (kind === "password") return document.querySelector('input[type="password"]'); + // Keep in step with is_otp_descriptor in crates/kharon-recording/src/classify.rs. if (kind === "totp") { return document.querySelector( 'input[autocomplete*="one-time"], input[name*="otp" i], input[type="tel"]' diff --git a/docs/superpowers/plans/2026-11-18-361-teach-recorder.md b/docs/superpowers/plans/2026-11-18-361-teach-recorder.md index 13759d5..3ab1dcb 100644 --- a/docs/superpowers/plans/2026-11-18-361-teach-recorder.md +++ b/docs/superpowers/plans/2026-11-18-361-teach-recorder.md @@ -12,7 +12,7 @@ ## Global Constraints -- Work in the worktree `~/projects/kharon-361-teach`, on a branch named `duet/361-teach-recorder` cut from `duet/361-teach-recorder-design` (the branch that holds the spec and this plan, PR #362). The implementation PR is opened with that branch as its base; when #362 merges, its base moves to `master` and master is merged in. No rebase, no force-push. Set `CARGO_TARGET_DIR=$PWD/target` in every cargo command; the machine's default target dir is shared between checkouts. +- Work in the worktree `~/projects/kharon-361-teach`, on branch `duet/361-teach-recorder`, cut from `master` at `947a457` (the merge of #362, which holds the spec and this plan). No rebase, no force-push. Set `CARGO_TARGET_DIR=$PWD/target` in every cargo command; the machine's default target dir is shared between checkouts. - Every commit message ends with these two lines and no other `Co-Authored-By` line: ``` Claude-Session: https://claude.ai/code/session_017zqkEAhTEMgVMZCGQw2mT8 @@ -27,6 +27,8 @@ - The binary listens on no socket and opens no connection except loopback CDP to the browser it launched. - Real-browser tests follow the existing convention: find the binary with the same path list as `chromium_bin` in `crates/kharond/tests/req_arch_003_org_chromium.rs`, honor `KHARON_CHROMIUM`, skip when none is found unless `KHARON_REQUIRE_CHROMIUM` is set, in which case panic. - Test files that prove a SPEC row are named for it: `req_tch_00N_.rs`. +- A source scan in a test covers shipped code only: for each file under `crates/kharon-teach/src`, the text before its first `#[cfg(test)]`. Unit-test modules sit at the bottom of their file, so a test server or a test URL there does not trip a scan. +- `cargo-deny` is not installed on the dev machine. Where a task says to run `cargo deny check`, CI runs it on the PR. - Linux only is built and tested. Platform-specific code sits behind `cfg`. ## Review Focus @@ -136,11 +138,13 @@ pub enum Step { EnterKey { t_ms: u64, frame: FrameRef, element: ElementDescriptor }, } pub enum EndedBy { Confirmed, WindowClosed } +pub enum ListenerWorld { Isolated, Main } pub struct Viewport { pub width: u32, pub height: u32 } pub struct BrowserInfo { pub product: String, pub user_agent: String, pub viewport: Viewport } pub struct StrippedCounts { pub requests_seen: u64, pub requests_stored: u64, pub request_bodies_reduced: u64, pub query_values_removed: u64, pub path_segments_replaced: u64, pub typed_values_removed: u64 } pub struct Recording { pub format_version: u32, pub tool_version: String, pub browser: BrowserInfo, + pub listener_world: ListenerWorld, pub started_at: String, pub ended_at: String, pub ended_by: EndedBy, pub success_confirmed: bool, pub start_url: StrippedUrl, pub final_url: Option, pub steps: Vec, pub requests: Vec, pub flags: Flags, pub classification: Classification, @@ -216,7 +220,7 @@ fn hits_walks_url_authorization_and_nested_json() { pub struct Flags { pub captcha: bool, pub otp_step: bool, pub cross_site: bool, pub popup: bool, pub framed_login: bool, pub shadow_dom: bool, pub multiple_passwords: bool, pub body_unavailable: bool } #[serde(rename_all = "snake_case")] pub enum PasswordLeg { FormPost, ScriptPost, Transformed } -#[serde(tag = "shape", rename_all = "snake_case")] +#[serde(tag = "kind", rename_all = "snake_case")] // on the wire: "shape": {"kind": "two_step", "password_leg": "form_post"} pub enum Shape { FormPost, ScriptPost, TwoStep { password_leg: PasswordLeg }, Transformed { digest: Option }, Unrecognized } #[serde(rename_all = "snake_case")] pub enum RungCeiling { Milliseconds, AgentContinues, NeedsPerson } @@ -288,6 +292,9 @@ impl Marker { } /// Every way `marker` occurs in `haystack`. Digests are searched only for Role::Password. pub fn search(haystack: &[u8], marker: &Marker) -> Vec; +/// The base64 and base64url (padded or not) of `username:password`. A match returns two hits, +/// one per role, both `Encoding::Base64` (or `Base64Url`). +pub fn search_pair(haystack: &[u8], username: &Marker, password: &Marker) -> Vec; pub const LOW_CONFIDENCE_BELOW: u32 = 6; ``` @@ -315,6 +322,10 @@ fn encodings(h: &[u8], m: &Marker) -> Vec { search(h, m).into_iter().m #[test] fn non_ascii_percent_encoded() { /* "pässwörd✓12" as UTF-8 percent-encoded -> [Percent] */ } #[test] fn non_ascii_json_escaped_both_forms() { /* raw UTF-8 inside quotes -> [AsTyped]; ä-style escapes -> [JsonEscaped] */ } #[test] fn two_hundred_characters() { /* a 200-char value is found as typed */ } +#[test] fn pair_is_found_where_neither_value_alone_is() { + // haystack: {"auth":""}; search() finds nothing for either marker; + // search_pair() returns [Username Base64, Password Base64] +} ``` Also add this doc test on `Marker`, which is the compile-time guard: @@ -347,19 +358,23 @@ Also add this doc test on `Marker`, which is the compile-time guard: // strip.rs pub struct Stripped { pub value: T, pub query_values_removed: u64, pub path_segments_replaced: u64 } /// Origin, path and query keys of `url`. A path segment becomes "{id}" when it is all ASCII digits, -/// a UUID, 16 or more hex characters, or contains any marker (as typed or percent-encoded). -pub fn strip_url(url: &str, markers: &[Marker]) -> Option>; // None when `url` does not parse +/// a UUID, 16 or more hex characters, or is touched by a marker match anywhere in the path (any encoding `search` +/// knows, any letter case, with or without `pair`). A query key or host label a marker touches is written `{typed}`. +/// A URL that cannot be a base (`data:`, `blob:`, `about:`) keeps its scheme only. +pub fn strip_url(url: &str, markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> Option>; // None when `url` does not parse /// scheme = first token, lowercased. For `basic`, the credentials are base64-decoded and the decoded bytes /// are searched; a marker found there is reported with `Encoding::Base64`. Other schemes are searched as sent. -pub fn reduce_authorization(header_value: &[u8], markers: &[Marker]) -> AuthShape; +/// When `pair` is given the value is also searched with `search_pair`. No (role, encoding) is listed twice. +pub fn reduce_authorization(header_value: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> AuthShape; // body.rs pub const MAX_BODY_BYTES: usize = 1 << 20; /// `content_type` picks the reader: application/x-www-form-urlencoded, application/json (and any +json), /// multipart/form-data; anything else, or a body that fails to parse as its declared type, is Opaque. -pub fn shape_of_body(content_type: Option<&str>, body: &[u8], markers: &[Marker]) -> BodyShape; +/// `pair` is (username, password) once both are known; see below. +pub fn shape_of_body(content_type: Option<&str>, body: &[u8], markers: &[Marker], pair: Option<(&Marker, &Marker)>) -> BodyShape; ``` -A multipart part with a `filename` is `ValueKind::File` with its length and no marker search. Every other value is searched with every marker. `shape_of_body` never returns `None` or `Unavailable`; the caller (Task 5) chooses those. +A multipart part with a `filename` is `ValueKind::File` with its length and no marker search. Every other value is searched with every marker and, when `pair` is given, with `search_pair`, so a combined credential is recorded on the value that holds it (an `Opaque` body holds it at whole-body level). A place never lists the same role and encoding twice. `shape_of_body` never returns `None` or `Unavailable`; the caller (Task 5) chooses those. - [ ] **Step 1: Write the failing tests** @@ -388,11 +403,13 @@ A multipart part with a `filename` is `ValueKind::File` with its length and no m #[test] fn multipart_text_and_file() { /* a text part "password" with a hit, a file part kind File with no hits */ } #[test] fn unknown_type_is_opaque_with_whole_body_hit() { /* content type application/octet-stream holding the password bytes -> Opaque { len, markers: [AsTyped] } */ } #[test] fn bad_json_falls_back_to_opaque() {} +#[test] fn a_combined_credential_lands_on_the_value_that_holds_it() { /* {"auth":"","n":1} with pair -> the auth value has [Username Base64, Password Base64]; without pair -> no hits */ } +#[test] fn basic_auth_with_the_pair_lists_each_hit_once() { /* reduce_authorization on a Basic header with pair given -> still exactly [Username Base64, Password Base64] */ } #[test] fn serialized_shapes_never_contain_a_marker() { /* for each case above: serde_json::to_string(&shape) contains neither "Tr0ub4dor-9x" nor "kh.throwaway7" */ } ``` -- [ ] **Step 2: Run** `cargo test -p kharon-teach strip body`. Expected: FAIL. -- [ ] **Step 3: Implement.** Add `url = "2"`, `serde_json`. The JSON body is the one place a site payload is read as `serde_json::Value`; say so in a comment. Urlencoded keys and JSON keys are kept as they are; values never are. +- [ ] **Step 2: Run** `cargo test -p kharon-teach -- strip body` (cargo takes one filter before `--`; several go after it). Expected: FAIL. +- [ ] **Step 3: Implement.** Add `url = "2"`, `serde_json`. The JSON body is the one place a site payload is read as `serde_json::Value`; say so in a comment. Urlencoded keys, JSON keys and multipart part names are kept unless a typed value is found in one, in which case it is written `{typed}`; values never are. An `Authorization` scheme is kept only when it is a known scheme name. `kharon_teach::testing` (doc-hidden) holds the needle list and the "holds no typed value" assertion that this task's tests, Task 5's and Task 8's all use. - [ ] **Step 4: Run** tests, clippy, fmt. Expected: pass. - [ ] **Step 5: Commit** `feat(teach): strip URLs and reduce request bodies to shapes (#361)` @@ -423,7 +440,7 @@ pub struct RawFrame { pub main: bool, pub url: String } pub enum PageEventKind { Input, Change, Click, Submit, EnterKey } pub enum RawBody { None, Unavailable, Bytes(zeroize::Zeroizing>) } pub enum CaptureEvent { - Started { browser: BrowserInfo }, // the main frame left the notice page + Started { browser: BrowserInfo, listener_world: ListenerWorld }, // the main frame left the notice page Navigation { t_ms: u64, frame: RawFrame }, // main and child frames alike Page { t_ms: u64, kind: PageEventKind, frame: RawFrame, element: RawElement, value: Option> }, // value only for Input and Change @@ -446,6 +463,7 @@ impl Capture { Decisions `finish` implements, beyond the spec's text: +- Username candidates are tried against the raw buffered bytes first. Once the username is resolved, every buffered request is reduced with the pair: `search_pair` over the URL bytes goes into `url_markers`, and `reduce_authorization` and `shape_of_body` are called with `pair = Some((username, password))`, so a combined credential is recorded where it sits. - Elements are keyed by `(document_id, element_id)`. One `Marker` per element holds its latest value. An element is a password element if any of its reports had `is_password_now`. - Password markers: the latest values of password elements, with equal values merged. More than one distinct value sets `multiple_passwords`. - Username: among non-password elements with a non-empty value, prefer one in the same form as a password element; else one whose value is found in a request that also carries the password; else one whose value is found in any request; else the last one typed in before the first password input. Other typed values are dropped. @@ -471,18 +489,19 @@ Decisions `finish` implements, beyond the spec's text: #[test] fn captcha_frame_sets_the_flag() #[test] fn otp_input_after_the_password_sets_the_flag() #[test] fn oversized_body_is_unavailable_and_flagged() // a MAX_BODY_BYTES + 1 body on a POST to the login site +#[test] fn a_combined_base64_credential_in_a_json_body_is_a_plain_hit() // classifies as ScriptPost, not Transformed #[test] fn only_the_three_kinds_of_request_are_stored() // a GET image and a third-party POST are counted, not stored #[test] fn events_before_started_are_ignored_for_steps() #[test] fn the_serialized_recording_holds_no_typed_value() { // for every scenario above: recording.to_json() contains neither marker in any encoding - // (as typed, percent, base64, hex, sha256 hex). Share the needle list with Task 8's test via a pub(crate) helper. + // (as typed, percent, base64, hex, sha256 hex): `kharon_teach::testing::assert_holds_no_typed_value`, from Task 4. } // site.rs #[test] fn registrable_domains() { /* login.ex.co.uk -> ex.co.uk; 127.0.0.1 and localhost unchanged */ } ``` -- [ ] **Step 2: Run** `cargo test -p kharon-teach capture site`. Expected: FAIL. -- [ ] **Step 3: Implement.** Load the public suffix list the way `crates/kharond/src/origin.rs` does. Requests are buffered with their bodies in `Zeroizing` buffers and analyzed in `finish`, after the markers are known. +- [ ] **Step 2: Run** `cargo test -p kharon-teach -- capture site`. Expected: FAIL. +- [ ] **Step 3: Implement.** Load the public suffix list the way `crates/kharond/src/origin.rs` does: `include_bytes!` of the same vendored snapshot, `crates/kharond/data/public_suffix_list.dat`, by relative path. That is one data file shared, not a crate dependency; do not copy it. Requests are buffered with their bodies in `Zeroizing` buffers and analyzed in `finish`, after the markers are known. - [ ] **Step 4: Run** tests, clippy, fmt. Expected: pass. - [ ] **Step 5: Commit** `feat(teach): capture state machine from events to a stripped recording (#361)` @@ -600,7 +619,7 @@ impl Cdp { #[test] fn headers_are_read_case_insensitively() ``` -- [ ] **Step 2: Run** `cargo test -p kharon-teach cdp events`. Expected: FAIL. +- [ ] **Step 2: Run** `cargo test -p kharon-teach -- cdp events`. Expected: FAIL. - [ ] **Step 3: Implement.** Add `tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "handshake"] }`, `futures-util`. One reader task routes replies by `id` to `oneshot` senders and everything else to the event channel. Outgoing frames are typed structs, never `json!`. - [ ] **Step 4: Run** tests, clippy, fmt. Expected: pass. - [ ] **Step 5: Commit** `feat(teach): a CDP client that subscribes to events (#361)` @@ -635,13 +654,15 @@ pub struct Ready { pub devtools_port: u16, pub profile_dir: std::path::PathBuf, pub enum SessionError { #[error(transparent)] Browser(#[from] BrowserError), #[error(transparent)] Cdp(#[from] CdpError), #[error(transparent)] Io(#[from] std::io::Error) } pub enum Outcome { Recorded(Box), Discarded, NeverStarted } +/// RFC 3339 UTC, with the `time` crate as `crates/kharond/src/audit/log.rs` does. Fills `EndInfo`; Task 9 reuses it. +pub fn utc_rfc3339(t: std::time::SystemTime) -> String; /// Sweeps stale profiles, launches, records until `end` fires or the window closes, cleans up. /// `ready` fires once the browser is up. Dropping the future (Ctrl-C) still removes the profile. pub async fn record(opts: RecordOptions<'_>, ready: tokio::sync::oneshot::Sender, end: tokio::sync::oneshot::Receiver) -> Result; ``` -Wiring, in order: `Profile::sweep_stale`; `Profile::create`; `Browser::launch` with `first_url = notice_page_url(..)`; `Cdp::connect`; `Target.setAutoAttach { autoAttach: true, waitForDebuggerOnStart: true, flatten: true }` on the browser session and again on each attached page session; for each attached target of type `page` or `iframe`: `Page.enable`, `Network.enable`, `Runtime.addBinding { name: "__kharonTeach", executionContextName: "kharon-teach" }`, `Page.addScriptToEvaluateOnNewDocument { source: LISTENER_JS, worldName: "kharon-teach" }`, then `Runtime.runIfWaitingForDebugger`. If the binding is not visible in the isolated world on the CI browser, install script and binding in the main world instead and record that in the commit message. Other target types are released without attaching streams. A second `page` target after the first sends `CaptureEvent::Popup`. `CaptureEvent::Started` is sent when the main frame first navigates to a URL that is not the notice page; its `BrowserInfo` comes from `Browser.getVersion` (product, user agent) and `Page.getLayoutMetrics` (the CSS viewport). A request with `hasPostData` and no inline body is completed with `Network.getRequestPostData`; an error there is `RawBody::Unavailable`. The window counts as closed when the last `page` target is destroyed or the browser process exits; that ends as `EndedBy::WindowClosed` with `success_confirmed: false`, or `Outcome::NeverStarted` if `Started` never fired. +Wiring, in order: `Profile::sweep_stale`; `Profile::create`; `Browser::launch` with `first_url = notice_page_url(..)`; `Cdp::connect`; `Target.setAutoAttach { autoAttach: true, waitForDebuggerOnStart: true, flatten: true }` on the browser session and again on each attached page session; for each attached target of type `page` or `iframe`: `Page.enable`, `Network.enable`, `Runtime.addBinding { name: "__kharonTeach", executionContextName: "kharon-teach" }`, `Page.addScriptToEvaluateOnNewDocument { source: LISTENER_JS, worldName: "kharon-teach" }`, then `Runtime.runIfWaitingForDebugger`. If the binding is not visible in the isolated world, install script and binding in the main world instead. Which world was used goes into `CaptureEvent::Started` as `listener_world`, and so into the file; say in the commit message which one the CI browser needed. Other target types are released without attaching streams. A second `page` target after the first sends `CaptureEvent::Popup`. `CaptureEvent::Started` is sent when the main frame first navigates to a URL that is not the notice page; its `BrowserInfo` comes from `Browser.getVersion` (product, user agent) and `Page.getLayoutMetrics` (the CSS viewport). A request with `hasPostData` and no inline body is completed with `Network.getRequestPostData`; an error there is `RawBody::Unavailable`. The window counts as closed when the last `page` target is destroyed or the browser process exits; that ends as `EndedBy::WindowClosed` with `success_confirmed: false`, or `Outcome::NeverStarted` if `Started` never fired. `listener.js` reports, in the capture phase on `document`: `input` and `change` on `input`, `textarea`, `select`; `click` on `button`, `a`, `input[type=submit]`, `input[type=button]`, `[role=button]`; `submit` on forms; `keydown` with `key === "Enter"` on an `input`. It numbers elements with a `WeakMap` counter, gives each document a random `document_id`, uses `event.composedPath()[0]` to see into open shadow roots, and builds `selectors` (how is the builder's call; the first entry must match the element with `document.querySelector` at the time of the event when the element is not in a shadow root). It sends `value` only for `input` and `change`. It keeps no state other than the counter. @@ -667,7 +688,7 @@ Wiring, in order: `Profile::sweep_stale`; `Profile::create`; `Browser::launch` w // req_tch_002_no_typed_value.rs #[tokio::test] async fn no_typed_value_in_the_recording_for_any_fixture() { // username "kh.throwaway7", password "Tr0ub4dor-9x": recording.to_json() contains neither, in none of the - // encodings marker::search knows (reuse the needle helper from Task 5). + // encodings marker::search knows (`kharon_teach::testing::assert_holds_no_typed_value`). } // req_tch_006_profile_and_visible.rs @@ -767,8 +788,8 @@ Exit codes: `0` a file was written, or the recording was discarded; `2` a refusa ``` - [ ] **Step 2: Run** `cargo test -p kharon-teach`. Expected: the new tests FAIL. -- [ ] **Step 3: Implement.** Add `clap` (derive) and, Linux only, `rustix` with the `process` feature. For the UTC timestamp use whatever `crates/kharond/src/audit/log.rs` uses for its `ts` field; do not add a second date crate. -- [ ] **Step 4: Run** `KHARON_REQUIRE_CHROMIUM=1 cargo test -p kharon-teach`, clippy, fmt, `cargo deny check`. Then one manual run on this machine: `cargo run -p kharon-teach -- https://login.page.twkr.io/login.html`, log in with made-up values, press Enter, and read the file. Paste the printed summary into the commit body. +- [ ] **Step 3: Implement.** Add `clap` (derive) and, Linux only, `rustix` with the `process` feature. For the file name's UTC timestamp use the `time` crate, already a dependency since Task 8's `utc_rfc3339`; do not add a second date crate. +- [ ] **Step 4: Run** `KHARON_REQUIRE_CHROMIUM=1 cargo test -p kharon-teach`, clippy, fmt, `cargo deny check`. The manual run (`cargo run -p kharon-teach -- https://login.page.twkr.io/login.html`, log in with made-up values, press Enter, read the file) needs a person at a display. The implementer does not do it; the controller arranges it with Lothrop after this task and puts the printed summary in the PR body. - [ ] **Step 5: Commit** `feat(teach): kharon-teach command line, consent and the recording file (#361)` --- diff --git a/docs/superpowers/specs/2026-11-18-teach-recorder-design.md b/docs/superpowers/specs/2026-11-18-teach-recorder-design.md index 15b8ec4..5df9ba1 100644 --- a/docs/superpowers/specs/2026-11-18-teach-recorder-design.md +++ b/docs/superpowers/specs/2026-11-18-teach-recorder-design.md @@ -41,7 +41,8 @@ The work is three pieces, in this order (Duet's call, accepted): that names the login's shape. 2. The login-method format, a player that replays a method with a real secret at rung 2, importing a recording into a broker, and the - per-site registry. + per-site registry. Rung 2 includes the post-submit scrub, so the + player needs #355 to land first. 3. The never-in-the-page rung. The recorder goes first because pieces 2 and 3 need real recordings to @@ -288,6 +289,12 @@ password it is also searched for the hex and base64 of its MD5, SHA-1, SHA-256 and SHA-512. A marker shorter than six characters is searched but its matches are marked low-confidence. +Some logins encode the two values together, as HTTP Basic does with +`username:password`. Neither value's own base64 appears in that. So +once the username is known, each request is also searched for the +base64 and base64url of `username:password`, and a match counts as a +hit for both (Muse's review of #362). + What is recorded is the **role** and the place: "the `password` key of this urlencoded body is the password, as typed". The value is not. @@ -323,6 +330,10 @@ directory unless `--out` says otherwise. It holds: - the tool version, the browser's product string and user agent, the viewport size, start and end times, how it ended, and `success_confirmed` +- `listener_world`: `isolated` or `main`, the script world the page + listener ran in. In the main world a page's own scripts can see and + fake the listener's reports, so a reader should trust such a + recording less - the start URL and the last main-frame URL, stripped as above - the ordered steps: navigations and the person's events - the stored requests, as shapes with roles @@ -457,3 +468,7 @@ For the implementation PR, wording to be reviewed there: recorder still writes it nowhere, but the program read it and the browser held it until the profile was deleted. - Passkey logins are not detected in this piece. +- `transformed` with no named digest will mostly be sites that encrypt + the password in the page with a key the site supplies, which banks + often do. It is the expected common case of `needs_person`, not an + oddity, and piece 2 should be designed with that in mind. diff --git a/docs/teach-mode.md b/docs/teach-mode.md new file mode 100644 index 0000000..9debebf --- /dev/null +++ b/docs/teach-mode.md @@ -0,0 +1,178 @@ +# Teach mode: recording a login + +## What this is for + +Some sites log you in with one form. Others ask for the user name on one page and the password on the next, or run a script that changes the password before it sends it. kharon needs to know which kind a site is before it can handle that login safely. Teach mode is how it learns. You log in by hand once, with a throwaway account, and a small program called `kharon-teach` writes down how the login worked. + +This is the first piece of teach mode. It records. kharon cannot yet turn a recording into a login method or replay one; that is later work (#361). For now the recording is a file that a person or a model can read. + +## Before you start + +Use a throwaway account. Never use a real one. The program reads everything you type in the window it opens, because that is how it learns which field is which. It does not save what you type, but it does read it, and the browser keeps your session until the window closes. If you type a real password by mistake, change that password. + +Linux only for now. A Chromium-family browser must be installed: Chromium, Google Chrome, Microsoft Edge or Brave. The program looks for them on your `PATH` in that order, or you can name one with `--browser`. + +A browser installed as a snap package cannot use the folder the program makes for its profile, because a snap gets its own private temporary folder. On Ubuntu the default Chromium is a snap. The program skips a snap browser and tries the next one. If a snap is the only browser it finds, it stops and prints "the only browser found is a snap package, which cannot use the recorder's profile folder; install Google Chrome or pass --browser ". A browser you name with `--browser` is used as you gave it. + +The program always opens a visible window. There is no way to turn that off. + +## Recording a login + +Give the program the address of the login page: + +```text +kharon-teach https://login.example.com/ +``` + +The address must start with `https://`. Plain `http://` is accepted only for `localhost` and other addresses on your own computer. An address that holds a user name or password is refused. + +The terminal prints this notice and asks you to agree: + +```text +kharon-teach records how a login works so kharon can learn it. +Use a THROWAWAY account. Everything you type in the window that +opens is read by this program. Nothing you type is saved, and +nothing is sent anywhere. The result is one file on this computer. +Type yes to open the browser window and start: +``` + +Type `yes` and press Enter. Anything else ends the program with "Nothing was recorded." and opens nothing. + +A browser window opens on a new, empty profile. Its first page repeats the notice and has one link, "Start recording and open" followed by the site's name. Under the link it says "Open the link in this tab." Recording starts when you click it in that tab. Nothing before that click is recorded. Opening the link in a new tab, or typing an address in another tab, does not start the recording. + +The terminal then prints: + +```text +Press Enter once you are logged in. Type q to discard. +``` + +Log in by hand in the window. When you are signed in, go back to the terminal. A recording can end in three ways: + +- Press Enter. The recording is saved and marked as a login that worked. +- Type `q` or `quit` (in any letter case) and press Enter. The recording is thrown away and nothing is saved. +- Close the browser window. The recording is saved and marked as a login that was not confirmed. If you close the window before you click the link, nothing is saved. + +Any other line you type is not an answer. The program prints "Press Enter to save, or type q to discard." and waits for another line. If the terminal's input ends without a line (it is closed or piped from a file that has run out), the program treats that as neither answer and keeps recording until the window closes. + +If the session ends before the recording started, because you pressed Enter before you clicked the link, closed the window first, or opened the link in another tab, nothing is saved. The program prints "Recording never started: the login page was not opened in the first tab. Nothing was saved." + +Ctrl-C, a stop request (SIGTERM), closing the terminal window (SIGHUP) or Ctrl-\ (SIGQUIT) ends the program with "Interrupted. Nothing was saved." and a failure exit code. This line and every refusal go to the error output, each starting with "kharon-teach: ". + +On every one of these, the program closes the browser and deletes the profile it made. The browser runs in its own process group, so that closing it also closes its helper processes. + +If the program itself is killed without a chance to react (SIGKILL, a crash, a power cut), it cannot clean up. Then the browser window stays open, still holding the throwaway session, until you close it. The profile folder is deleted by a later run, and only after that window is closed. A later run deletes a folder only if the program made it: its name is `kharon-teach-` followed by exactly 20 letters and digits, it holds the owner file the program writes with its own process number, that process is no longer running, and no browser is running on it. Any other folder, even one named `kharon-teach-something`, is left alone. When it deletes one, it says "Removed 1 leftover browser profile(s) from an earlier run." before the Enter prompt. + +When a recording is saved, the program prints a summary of it and then "Saved:" and the path. The file is named after the site and the time, for example `login.example.com-20261118T061500Z.kharon-recording.json`, and is written in the current directory. Only you can read it (mode `0600`). If a file of that name already exists, the program adds `.1`, then `.2`, up to `.9`, and does not overwrite anything. For a generated name the number goes before `.kharon-recording.json`. For a name you gave with `--out` it goes at the end. + +## Options + +```text +kharon-teach [--out ] [--browser ] [--yes] +kharon-teach show +``` + +| Option | What it does | +|---|---| +| `--out ` | Where to write the recording. The file must not exist yet, and its folder must exist and be one you can write in. Without it, the file goes in the current directory under the generated name. | +| `--browser ` | The browser program to use, instead of searching `PATH`. The program expects a Chromium-family browser and does not check. | +| `--yes` | Skips the question "Type yes to open the browser window and start". The notice is still printed. | +| `show ` | Prints what a recording file says, in plain words. It opens no browser and changes nothing. A file larger than 16 MiB, or one that is not a recording this version can read, is refused. | + +There is no option to use a browser that is already running, or to pick the profile folder. The program starts its own browser every time. + +## What is in the file and what is not + +The file describes the structure of the login. It is one JSON document (a plain-text format that programs can read), and it holds nothing you typed. Before the program writes it, everything you typed is reduced to a role (user name or password) and a length. What is kept and what is dropped: + +| What the browser showed | What the file keeps | +|---|---| +| What you typed | The role (user name or password) and the length. Every field you typed in counts, not only those two. | +| Request bodies | The shape: the names of the fields, and for each value its kind, its length and, where a typed value was found, its role and how it was encoded. Never the value. A body the program cannot read as the kind its content type says, and a body that would make a shape of more than 2,000 values, is kept as a length only, with the role and encoding of any typed value found in it. A body over 1 MiB is not read at all and is marked `unavailable`: the file says only that its body could not be read. | +| Query strings (`?a=b`) | The names only. | +| Names that do not look like names | A field, part or query name is kept only when it looks like a name: at most 64 characters, all of them letters, digits or one of `_ . : - [ ]`, and not all digits, not a UUID and not 16 or more hex characters. A query piece with no `=` must also be at most 32 characters. Anything else is written `{other}`, because it is more likely a value than a name. A urlencoded body with a piece that has no `=`, or in which most names do not look like names, is not treated as fields at all; it is kept as a length only. | +| Paths (`/a/b`) | The path, with a part replaced by `{id}` when it is all digits, looks like a UUID, is 16 or more hex characters, or contains something you typed. | +| Names that contained something you typed | Replaced by `{typed}`. This covers the names of fields and parts, the labels of a host name and the content type. A typed value found in a name is still recorded as a hit on that field, so the login is read correctly. | +| Web addresses | The scheme is kept only if it is a common one (http, https, ws, wss, data, blob, about, file, chrome, chrome-extension, devtools). Any other is written `{other}`. The content of a `data:` address is never kept. | +| Headers | The content type only. For `Authorization`, only the scheme (for example Basic or Bearer), and only from a known list; any other scheme is written `{other}`. Whether a typed value was in it is recorded. | +| Cookies | Whether a response set one. Nothing else. | +| Responses | For each stored request, the status number (for example 200) and the content type of the reply. | +| Response bodies | Never read. | +| Page text, titles and HTML | Never read. Only the attributes of the fields you used, and of the buttons and links you clicked, are kept: tag, type, `id`, `name`, `autocomplete`, placeholder, aria-label, the form it is in, and a list of ways to find it again. An attribute or way of finding an element that held something you typed is left out. | + +Only three kinds of request are stored: pages the main window moved to, requests in which a typed value was found, and any request other than GET to the login site after your first keystroke. All other requests are counted and dropped. A typed value of fewer than six characters can match by chance, so such a match decides nothing when a longer, surer match of the same kind (user name or password) exists in the recording. + +The program holds at most 5,000 requests and keeps at most 10,000 steps. Requests past the first 5,000 are counted and not looked at. Steps past 10,000 are not written, but they are counted, and what you typed in them is still found and removed. + +The file also records the program version, the browser's name and user agent (the line a browser sends to say what it is), the window size, the start and end times, how the recording ended, and the steps you took: pages visited, fields typed in (by length), clicks, form submissions and Enter presses. + +At the end of the file, a `stripped` block counts what was removed: how many requests the browser made and how many were kept, how many typed values, query values and path parts were dropped, how many bodies were reduced to a shape, and how many steps were dropped for being past the limit. The summary prints the same counts, and a line "Not recorded: N step(s) past the recorder's limit." when any steps were dropped. + +## Reading the result + +The program ends with a summary, and `show` prints the same one later. Each line starts with a label. + +Shape of the login ("How the login works"): + +| Shape | Meaning | +|---|---| +| `form_post` | One page. The form sends the password to the site as typed or encoded. Encoded means changed back and forth without losing it, for example percent-encoded or base64. | +| `script_post` | One page. The page's script sends the password to the site as typed or encoded. | +| `two_step` | The user name goes first, then the password on a later step. The summary says how the password was sent: by a form, by a script, or changed by the page first. | +| `transformed` | The page changes the password before sending it, so no request holds it as typed. If the change was a known digest, the summary names it. A digest is a fixed-length scramble of the password that cannot be turned back (md5, sha1, sha256 or sha512). | +| `unrecognized` | None of the above. This also covers a recording in which no password field was used, one in which the password was sent in a way the program does not know, one in which request bodies were unavailable and the password was found nowhere, and one with a flag below that needs a person to look. | + +Strongest protection ("Strongest protection"), the best that kharon could do for this login: + +| Value | Meaning | +|---|---| +| `milliseconds` | kharon could fill and submit in one step, so the password is in the page for milliseconds. | +| `agent_continues` | The agent has to continue after the fill. This is the case for two-step logins, logins with a captcha or a one-time-code step, and one-page logins where the password field was not in a form and no Enter key and no click on a button followed it. | +| `needs_person` | A person has to look at the recording. This includes a two-step login whose password step is changed by the page, unless the change is a digest the program can name. With a named digest, the result is `agent_continues` and a note names the digest. | + +"Network swap" says whether the password was found, as typed or encoded but not hashed, in exactly one stored request. Such a login might let kharon put the password in as the request leaves the browser, so the page never holds it. kharon cannot do this yet. The line records which logins might allow it. + +Flags, each printed as "Flag:" with a sentence: + +| Flag | Meaning | +|---|---| +| `captcha` | A captcha was on the page. The program knows five captcha hosts, plus Google's. | +| `otp_step` | There was a field for a one-time code after the password. | +| `cross_site` | The browser moved to another web site before the password was last sent, as single sign-on does. Only a move to another web site (an `http` or `https` page of another registrable domain) counts; a browser error page does not. A move after the last request that carried the password is not a flag. A login that lands on another site afterwards is still read normally, and the summary adds the note "After the login the browser went on to another site." | +| `popup` | The login opened another window. | +| `framed_login` | A login field was inside a frame from another web address. | +| `shadow_dom` | A login field was inside a shadow root, a part of the page that hides its fields from ordinary lookups. | +| `multiple_passwords` | The page had two password fields with different values, as a sign-up or change-password form does. | +| `body_unavailable` | The browser would not hand over the body of some request, so the shape may be incomplete. | + +`cross_site`, `popup`, `framed_login` and `multiple_passwords` make the shape `unrecognized`. The summary may also print "Note:" lines that say why in plain words. + +A "Caution" line appears when the page's own scripts could see and fake what the program reports. Trust such a recording less. + +## Sending a recording to someone + +A recording is an ordinary file. The program never sends it anywhere. If you want someone to read it, you copy it to them yourself. + +Before you do, run `kharon-teach show ` and read what it says. The `stripped` block at the end of the file lets the reader check what was removed: the number of typed values, query values and path parts dropped, and how many requests were kept out of how many seen. The file still names the site you recorded and the structure of its login, so send it only to someone who may know which site that is. + +## What it cannot do + +- It records in a visible desktop browser. The agent's browser is usually headless, and a site may lay itself out differently there. The file records the window size and the user agent so a mismatch can be seen later. +- A site with bot detection may treat a fresh profile with a debugging port differently from your everyday browser. +- If you type a real password despite the notice, the program still writes it nowhere, but it read it, and the browser held it until the profile was deleted. +- If the program is killed without a chance to react (SIGKILL, a crash, a power cut), the browser window stays open with the throwaway session in it. Its profile is deleted by a later run, only after that window is closed. +- While a recording runs, the browser's debugging port listens on 127.0.0.1. Any program on the computer, including another user's program on a shared machine, can connect to it and read what is typed. Record only on a computer you do not share. +- A browser installed as a snap package is skipped, because it cannot use the profile folder (see "Before you start"). This has not been tried on an Ubuntu machine yet. If the program fails to start the browser there, pass `--browser` with Google Chrome or a Chromium that is not a snap. +- It does not detect passkey logins. +- It does not follow web workers or service workers. +- It finds a typed value in a request only in forms it knows: as typed, percent-encoded, escaped for JSON, base64, base64url and hex, and for the password the md5, sha1, sha256 and sha512 digests in hex or base64. A site that lowercases or re-encodes a typed value before sending it may not be recognised as the user name. +- A typed value that a site sends as a JSON number is looked for as its text. If the site writes the number differently from how it was typed, it is not found. +- A typed value shorter than six characters, such as a PIN, is searched for, but a match is marked low confidence, because it may be chance. In text the program keeps (paths, query names, hosts) it is removed only where it stands as a whole word, so that it does not damage ordinary words. In requests its matches are weak hints, and they can point at the wrong request when the same few characters appear by chance in another one. +- If you mistype the password and the page reloads before you get it right, the recording reads as a sign-up form (two password values) and needs a person. Start the recording again instead. +- Logins the page's script cannot see are not recognised: the browser's own Basic or Digest login dialog, passkeys, and fields drawn on a canvas or typed into a `contenteditable` element. +- A captcha script that only loads on the page (reCAPTCHA v3, Cloudflare Turnstile) sets the `captcha` flag and lowers the result to `agent_continues`, even when no challenge was shown. +- A password sent over a WebSocket, through a service worker or from another tab is not found, and neither is a change outside the encodings and digests listed above. Such a login comes out as `transformed` or `unrecognized`. +- Opening the link on the notice page in a new tab, or typing another address, means the recording never starts. The page says to open the link in the same tab. +- Once 32 MiB of request bodies are held, later bodies are recorded as unavailable. +- A `transformed` login with no named digest is mostly a site that encrypts the password in the page with a key the site supplies, as banks often do. It is an expected result and is marked `needs_person`. +- A recording made with the listener in the page's own script world, not the isolated one, is marked as such and is less trustworthy. +- Turning a recording into a login method is not built yet.