diff --git a/CHANGELOG.md b/CHANGELOG.md index 56303d44..cbcc2af3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ All notable changes to RustChan will be documented in this file. +## RustChan 1.7.0 — 2026-10-08 + +### Upgrade notes + +- Back up the complete data directory before upgrading. Native Linux updates retain the signed-package verification, pre-upgrade snapshots and full data-and-binary rollback introduced in 1.6.6; the minimum updater remains 1.6.6. Containers, macOS, Windows and custom installations continue to use their deployment tools. +- Rust 1.99 remains the minimum supported compiler. + +### Improved + +- Improved thumbnail delivery and caching while preserving board permissions, private-media access checks and existing preview formats. +- Refined thread controls, post actions and archive navigation, including consistent handling of locked, archived, sticky and pruned threads. +- Refreshed Frutiger Aero and refined the other themes, with focused readability improvements for Forest and Terminal. +- Updated Rust dependencies within the existing dependency set. + +### Fixed + +- Release settings locks explicitly when configuration operations finish, preventing duplicate process descriptors from delaying subsequent saves or restarts. +- Hardened posting and thread lifecycle transactions, retry receipts, reply counters, retention and recoverable media publication/deletion. Posting checks current thread state and board policy before committing. +- Hardened hostile input handling across uploads, paths, redirects, administrator authentication/settings, backups and restores. +- Bounded expensive search, CAPTCHA and request work, strengthened rate limiting and connection handling, and expanded security and lifecycle regression coverage. + ## RustChan 1.6.6 — 2026-10-03 - Fixed migration of supported earlier package baselines, including 1.6.0 databases missing the two newer post-query indexes. Unknown schema drift remains rejected; upgrades preserve existing content and retain full data-and-binary rollback. diff --git a/Cargo.lock b/Cargo.lock index 3927f890..5d3c6172 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1192,9 +1192,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" dependencies = [ "cmov", ] @@ -1662,9 +1662,9 @@ dependencies = [ [[package]] name = "either" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" [[package]] name = "elliptic-curve" @@ -1846,6 +1846,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" +[[package]] +name = "fearless_simd" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3296eb97d869113eaf148a6bdf01884e90b3828ad7ae2340b0f93362fc158611" + [[package]] name = "ff" version = "0.13.1" @@ -1952,9 +1958,9 @@ dependencies = [ [[package]] name = "fs-err" -version = "3.3.1" +version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" +checksum = "b5c95b673b8f6f7235229ae11c5642d81b04c2e64c1e2fb417bc0cf73ca45f29" dependencies = [ "autocfg", "tokio", @@ -2236,9 +2242,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" dependencies = [ "atomic-waker", "bytes", @@ -2336,6 +2342,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f4d0e94ddd48749f06bbe4e5389fb9799a0c45bcaf00495042076ef05e3241a" +[[package]] +name = "hayro-ccitt" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2866f0d976e76497480b853d5e258dab6e4b89ecfe991124e08f15dfa8fd03ad" + [[package]] name = "hayro-cmap" version = "0.1.0" @@ -2366,12 +2378,12 @@ dependencies = [ [[package]] name = "hayro-jbig2" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69374b3668dd45aeb3d3145cda68f2c7b4f223aaa2511e67d076f1c7d741388d" +checksum = "000e025850f4592ddcaee85640db70704151f2bafd43b6f431aa5b6a2c6a35a9" dependencies = [ - "fearless_simd", - "hayro-ccitt", + "fearless_simd 1.1.0", + "hayro-ccitt 0.4.0", ] [[package]] @@ -2380,7 +2392,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c75ab947623ef4ccaa7acf0579edf7cbb5a73838e3839a7be73335e522f433a1" dependencies = [ - "fearless_simd", + "fearless_simd 0.4.1", ] [[package]] @@ -2396,7 +2408,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0edeafd70aa2db743de8ede8637d07ec87db05efe69cde371d03f1b185fcef27" dependencies = [ "flate2", - "hayro-ccitt", + "hayro-ccitt 0.3.0", "hayro-jbig2", "hayro-jpeg2000", "memchr", @@ -2532,9 +2544,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", @@ -2862,9 +2874,9 @@ dependencies = [ [[package]] name = "inventory" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" dependencies = [ "rustversion", ] @@ -2916,9 +2928,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.37" +version = "0.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" dependencies = [ "defmt", "jiff-core", @@ -2943,9 +2955,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.37" +version = "0.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" dependencies = [ "jiff-core", "proc-macro2", @@ -2955,9 +2967,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" [[package]] name = "jiff-tzdb-platform" @@ -4054,9 +4066,9 @@ dependencies = [ [[package]] name = "powerfmt" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" [[package]] name = "ppv-lite86" @@ -4093,7 +4105,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.15+spec-1.1.0", + "toml_edit 0.25.16+spec-1.1.0", ] [[package]] @@ -4619,7 +4631,7 @@ dependencies = [ [[package]] name = "rustchan" -version = "1.6.6" +version = "1.7.0" dependencies = [ "anyhow", "argon2", @@ -4676,7 +4688,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-util", - "toml 1.1.6+spec-1.1.0", + "toml 1.1.7+spec-1.1.0", "tor-cell", "tor-hsservice", "tower", @@ -5052,9 +5064,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c" dependencies = [ "serde_core", ] @@ -5325,9 +5337,9 @@ dependencies = [ [[package]] name = "spki" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9" dependencies = [ "base64ct", "der 0.8.2", @@ -5941,14 +5953,14 @@ dependencies = [ [[package]] name = "toml" -version = "1.1.6+spec-1.1.0" +version = "1.1.7+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "920602543f0911ab71da12c50d59701da54c196d1a2bf5cb4b75667f137a406a" +checksum = "7874cd34dd99040fc24e30923aa64a60a81ff3ddfe2b8129ccc84ba24949deed" dependencies = [ "indexmap 2.14.2", "serde_core", - "serde_spanned 1.1.1", - "toml_datetime 1.1.1+spec-1.1.0", + "serde_spanned 1.1.2", + "toml_datetime 1.1.2+spec-1.1.0", "toml_parser", "toml_writer", "winnow 1.0.4", @@ -5965,9 +5977,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.1+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" dependencies = [ "serde_core", ] @@ -5988,21 +6000,21 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.15+spec-1.1.0" +version = "0.25.16+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +checksum = "88f38f3af8bf71841c67a990f4e2b52143b2d0a1a994c72513ff4c14977fc722" dependencies = [ "indexmap 2.14.2", - "toml_datetime 1.1.1+spec-1.1.0", + "toml_datetime 1.1.2+spec-1.1.0", "toml_parser", "winnow 1.0.4", ] [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.1.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "bed810ee007d8e7f9cf20b4774e7198142fa27186e0f805871da7f3c39eaf333" dependencies = [ "winnow 1.0.4", ] @@ -6015,9 +6027,9 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] name = "toml_writer" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770" [[package]] name = "tor-async-utils" @@ -6272,7 +6284,7 @@ dependencies = [ "serde_ignored", "strum", "thiserror 2.0.21", - "toml 1.1.6+spec-1.1.0", + "toml 1.1.7+spec-1.1.0", "tor-basic-utils", "tor-error", "tor-rtcompat", @@ -7352,9 +7364,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" [[package]] name = "unicode-ident" @@ -7469,7 +7481,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3361bff7f7d82c0c496b92048db83846691f0e844cc28dee92b1c824291b55ee" dependencies = [ "bytemuck", - "fearless_simd", + "fearless_simd 0.4.1", "guillotiere", "hashbrown 0.17.1", "log", @@ -7486,7 +7498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d672facaa2d697285a786cd9d44d614cd2ce54cdc022504bf339f8fff3b750" dependencies = [ "bytemuck", - "fearless_simd", + "fearless_simd 0.4.1", "guillotiere", "hashbrown 0.17.1", "log", @@ -7544,9 +7556,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -8068,7 +8080,7 @@ checksum = "105ef4642d9cb137ef83d623d0e4bf08b8adf69e9918ca904a174adb6d3d038b" dependencies = [ "const-oid 0.10.2", "der 0.8.2", - "spki 0.8.0", + "spki 0.8.1", "tls_codec", ] @@ -8140,18 +8152,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.59" +version = "0.8.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +checksum = "879fb705ce98c32e41ebdb970fbe1204f8492423b314c6ab0354c3e7b5542866" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.59" +version = "0.8.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +checksum = "708882a28301d604fa039cc7727607a98d04c4b86dc76ec9cc683f805d709759" dependencies = [ "proc-macro2", "quote", @@ -8181,9 +8193,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" dependencies = [ "zeroize_derive", ] diff --git a/Cargo.toml b/Cargo.toml index 23fb1421..454388a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustchan" -version = "1.6.6" +version = "1.7.0" edition = "2021" # Rust 1.99 is the supported minimum for the package and build tooling. rust-version.workspace = true @@ -53,7 +53,7 @@ captcha = { version = "1.0.0", default-features = false } # Arti 0.47 permits rusqlite >=0.36,<0.41. Keep the latest application and # Tor directory-cache SQLite dependencies on the same bundled library. -rusqlite = { version = "0.40.2", features = ["bundled", "backup"] } +rusqlite = { version = "0.40.2", features = ["bundled", "backup", "hooks", "blob"] } r2d2 = "0.8.10" r2d2_sqlite = "0.35.0" time = "0.3.55" diff --git a/README.md b/README.md index 72a42112..52ac3548 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Conversations are organized into **boards** (topics such as photography or techn RustChan runs as a single app, with a built-in database. You do not need to set up a separate database server. -**Source version:** v1.6.6. **Downloads:** [latest published release](https://github.com/csd113/RustChan/releases/latest). **Docker image:** `ghcr.io/csd113/rustchan:latest`. +**Source version:** v1.7.0. **Downloads:** [latest published release](https://github.com/csd113/RustChan/releases/latest). **Docker image:** `ghcr.io/csd113/rustchan:latest`. ## Quick start @@ -70,14 +70,14 @@ For **Docker Compose, updates, backups, or a public website**, follow the [conta Open the [latest release](https://github.com/csd113/RustChan/releases/latest), download the ZIP for your operating system, and extract it into a folder where the app can save files. -ZIP names include the release version; the v1.6.6 names are shown below. Choose the matching files from the published release you download. +ZIP names include the release version; the v1.7.0 names are shown below. Choose the matching files from the published release you download. | Your computer | Download | |---|---| -| Linux, Intel or AMD 64-bit | `rustchan-cli-v1.6.6-linux-x86_64.zip` | -| Linux, ARM64 | `rustchan-cli-v1.6.6-linux-arm64.zip` | -| macOS, Apple silicon | `rustchan-cli-v1.6.6-macos-apple-silicon.zip` | -| Windows, Intel or AMD 64-bit | `rustchan-cli-v1.6.6-windows-x86_64.zip` | +| Linux, Intel or AMD 64-bit | `rustchan-cli-v1.7.0-linux-x86_64.zip` | +| Linux, ARM64 | `rustchan-cli-v1.7.0-linux-arm64.zip` | +| macOS, Apple silicon | `rustchan-cli-v1.7.0-macos-apple-silicon.zip` | +| Windows, Intel or AMD 64-bit | `rustchan-cli-v1.7.0-windows-x86_64.zip` | From a terminal in the extracted folder, run: diff --git a/SETUP.md b/SETUP.md index 45015054..06baf820 100644 --- a/SETUP.md +++ b/SETUP.md @@ -2,7 +2,7 @@ Current setup and deployment guide for Linux, macOS, and Windows. -Current version: `1.6.6`. +Current version: `1.7.0`. This guide reflects the current RustChan architecture: @@ -723,3 +723,42 @@ resource limits and verification. ## Applying administrator configuration Live controls keep applying immediately. Other controls show their active, saved and next-start values. Saving never restarts RustChan; managed Linux and opted-in supervised containers expose **Restart RustChan** for pending settings. See [settings restarts](docs/settings-restarts.md) for the complete classification, supervisor setup, graceful shutdown and failure recovery. + +## Thread archives and retention + +Threads leave the active index when background maintenance finds more than the +board's **Max threads** non-sticky active threads. It keeps the newest last-bump +and thread-ID pairs; locked threads count, sticky active threads do not. New +thread creation durably schedules maintenance, so overflow can be visible +briefly. Startup and periodic reconciliation recover outstanding work. + +Replies bump until the board's bump limit is reached. Sage replies count toward +that limit without bumping. The bump limit stops bumps; it does not immediately +archive the thread. There is no time-based thread expiry. + +**Archive overflow threads** controls automatic archival, with the global +`archive_before_prune` safety net (default `true`) also enabling it. Hard deletion +of active overflow occurs only when both are off. Existing archives and their +navigation remain available even when automatic archival is disabled. + +**Max archived threads** is a count cap (1–10,000), not permanent retention. +Maintenance keeps the most recently bumped archived threads, breaking ties by +ascending thread ID. Archived sticky threads are also subject to the cap; +sticky is not an archive preservation setting. Lowering the cap permanently +removes overflow threads. Invalid retention values are rejected rather than +clamped; omitted retention fields preserve the saved limits. + +Archival keeps the same `/{board}/thread/{id}` URL and media references. Archived +threads are read-only, including polls; users can browse their results, quoting, +spoilers and attachments. Board search includes archived posts. Board index, +catalog and unread history show active threads. Administrators can manually +archive or delete a thread using the existing authenticated, CSRF-protected +controls. There is no public unarchive/preservation control. + +The archive and prune transitions use one SQLite write transaction. Pruned +media is removed only when no other database references remain, using durable, +restart-replayable file cleanup. Failed filesystem cleanup stays queued and +logged. Archive policy is read when maintenance executes, not from stale job +payloads. Changing the global TOML/environment safety net requires restart; +board settings are live. Monitor background-job failures if maintenance is +unable to acquire the database lock or complete cleanup. diff --git a/docs/post-actions-audit.md b/docs/post-actions-audit.md new file mode 100644 index 00000000..50848615 --- /dev/null +++ b/docs/post-actions-audit.md @@ -0,0 +1,141 @@ +# Post and thread actions audit + +## Pipeline mapped before implementation + +| Entry point | Request and authorization | State and rendered outcome | Inverse/failure | +| --- | --- | --- | --- | +| Catalog three-dot menu | Delegated `main.js` disclosure; native POST forms; no-JS `
` fallback | Server supplies report OP ID, thread ID, board, CSRF, personal pin/hide labels | Outside click/Escape close; vertical positioning only; no keyboard arrow navigation | +| Report thread / post | `POST /report`; CSRF, board-view password, ban check, post-to-board and post-to-thread checks | `db::file_report` derives target metadata from posts; unique open report per post/reporter; moderator reports table escapes text; redirect to target thread | Resolution is admin-only; target deletion cascades reports; duplicate inserts succeed without distinction in confirmation | +| Personal Pin / Unpin | `POST /{board}/thread-preference`; CSRF, board-view access, matching active thread | `user_thread_preferences.pinned`; catalog sorts personal pins, then sticky, then bump | Explicit setters; zero/default rows removed; no effect on global sticky flag or other visitors | +| Personal Hide / Unhide | Same preference route and access checks | `user_thread_preferences.hidden`; active catalog split; `/{board}/hidden` restores threads | Reload persists through server DB; only catalog consumes preferences; no reply-hide feature | +| Global Sticky / Unsticky | Native `POST /admin/thread/action`; session-scoped CSRF plus trusted origin; admin session checked inside immediate transaction | `threads.sticky`; board/catalog active listing orders sticky, bump, ID; state and mod log commit together | Explicit idempotent setters; unsticky schedules retention; archive ordering deliberately ignores sticky | +| Lock / Unlock / Archive | Same admin route and transaction | Canonical thread flags; archive also locks; prune intent persisted | No moderator board scopes exist: admin accounts have instance-wide moderation rights | +| Edit / Delete / Ban+delete | Existing post controls, separate native POST paths with own ownership/admin checks and confirmation | Not currently inside catalog overflow; existing routes and hooks must remain | Existing self-action expiry and moderation tests cover them; no new destructive menu actions needed | + +Personal preference identity reuses the existing IP/Tor identity hash. Loopback/unknown peers use the visitor cookie. It is independent of administrator account identity and deliberately remains so. Cookie clearing on public-IP identities does not erase database preferences. Foreign keys cascade preferences when their target thread is deleted; valid preferences need no time-based eviction. Boolean domain triggers reject malformed stored flags. + +## Verified problems and implementation scope + +- Catalog and hidden management fetch only the first 200 active threads before filtering/sorting, making older hidden/pinned preferences inaccessible. +- Board index ignores personal preferences, including its pagination and counts. Its summary fragment also leaves the thread container open, allowing later summaries/action wrappers to become nested; the missing closing tag is now fixed. +- Preference access/target checks and writes are separate autocommit operations, allowing archive/delete races. +- Form boards are lossily sanitized rather than rejected; actions use loose normalized strings. +- Popup positioning has no horizontal clamp, remains inside clipping ancestors, and does layout work on every scroll event. Closing scans every menu. +- A popup has no arrow/Home/End navigation and may remain open after focus moves away or history restores it. +- Report uses lossy board sanitation and silently treats duplicate submissions as newly filed. Report POSTs have no dedicated abuse limit when browsing-only limiting is enabled. +- Catalog ETags omit lock/media/reply metadata and hidden-management responses lack explicit personalized cache headers. +- Admin thread moderation ignores submitted board for target validation, logs forged board text for most actions, and performs a second redirect lookup after committing. + +The user confirmed preserving personal Pin/Unpin and auditing global Sticky separately. Hide remains personal listing visibility; explicit thread/search/archive navigation remains available. No individual reply hiding, account preferences, thread move operation, SPA, third-party popup package, or schema redesign is introduced. + + + +## 1–2. Menu findings and polish + +The original menu could clip inside a card or transformed ancestor, overflow a narrow viewport horizontally, detach after layout changes, and retain stale open state through focus changes or history. It also scanned every menu when closing or repositioning. The controller now tracks one open menu, temporarily moves it to the document body, measures its width in viewport space, clamps both axes to the visual viewport, flips above when appropriate, and limits scrolling only when space requires it. Scroll/resize work is coalesced through one animation frame; one shared resize observer runs only while a menu is open to follow header, text-size, media, and page-layout changes. Closing restores its original DOM position and clears transient state. + +The trigger remains a click/tap disclosure. Minimum popup width remains 150px, ordinary rows retain 6px/8px padding, and touch rows retain 38px minimum height with 9px/10px padding. Border, background, radius, hover, active, focus, and open states use existing theme tokens. The trigger stays visible when keyboard focus moves into the portaled menu. A scoped text rule prevents generic theme button styling from dimming menu labels; translucent panel tokens are painted over an opaque theme base so underlying card text/media cannot show through. Index and thread pages now reuse the catalog's actions and authoritative labels. Existing per-post report/edit/delete controls and native fallback forms remain. + +## 3–4. Hide architecture and fixes + +Hide is a personal **thread listing preference**, stored in SQLite, not moderation and not localStorage. It hides the OP and its reply previews together from active board index and catalog. Direct thread links, explicit search, archives, feeds, and read-only API navigation remain available. No per-reply hiding or thread moving subsystem exists. + +The previous 200-thread cutoff and filtering after fetching caused inaccessible older preferences and inconsistent counts. Both active listings now filter hidden threads in SQL before pagination and aggregate only selected threads. The index count uses the same renderable-OP filter. Catalog and Hidden management read all applicable active threads. Hiding several threads, new replies, pin/sticky/lock changes, reloads, and multiple tabs remain consistent. A direct hidden thread explains the listing preference and offers Unhide. Archived threads retain valid preferences and permit clearing them from their direct page; new Pin/Hide actions are disabled and rejected. Deleting a target cascades preferences. Clearing both flags removes the default row. Existing Boolean-domain triggers remain the malformed-state boundary. + +## 5–6. Personal pin and moderator Sticky + +The user explicitly confirmed preserving personal Pin/Unpin. These actions update `user_thread_preferences.pinned` for the viewer; they never change `threads.sticky` or another viewer's preferences. Anonymous visitors can use personal pinning after normal CSRF, board-view, and target validation. + +Moderator-wide Sticky/Unsticky remains a separate admin operation backed by the canonical `threads.sticky` field. There are no ordinary registered-user accounts or scoped moderator roles; all administrator accounts have instance-wide moderation rights. Anonymous/non-admin requests cannot perform global moderation. Existing POST-only routes and session-scoped admin CSRF remain enforced. + +Active index and catalog default order is **personal pin descending, global sticky descending, bump time descending, thread ID descending**. The filter/order occurs before LIMIT/OFFSET, avoiding repeated pins across pages. Catalog alternative sorts retain pin/sticky precedence and now use thread ID as a deterministic final tie breaker. Unpin returns a thread to its global sticky/bump position. Archive order remains the existing bump-descending/ID-ascending rule; search, feeds, and APIs retain their established non-personal semantics. + +## 7. Report findings and improvements + +Report targets are validated as a complete board/thread/post tuple; stored report metadata and redirect IDs come from the actual post. Thread actions submit the OP ID, which can differ from the thread ID, and reply controls submit the reply ID. Anonymous reporting remains supported; board-view restrictions and bans still apply. + +Reasons remain optional, trim whitespace, and retain the existing 256-Unicode-character truncation behavior. Controls are rejected; native form parsing and a 65,536-byte route limit bound request decoding. Moderator rendering escapes reasons, including script-like text and non-Latin input. Open duplicates remain idempotent, with a distinct “matching report is already open” confirmation instead of claiming a new report. Resolution remains admin-only and permits a later new report; target deletion cascades report rows. + +A dedicated budget permits 20 new reports per reporter in a rolling hour, independently of browsing rate-limit settings. Budget checking and insertion share an immediate transaction; an existing duplicate is accepted without consuming another slot. Exceeding the budget returns a themed 429 and `Retry-After: 3600`. The existing reporter identity/retention model is preserved. + +## 8. Authorization and security + +Found a menu-action CSRF gap: the general public validator accepted a correctly signed token generated for a different visitor. Report and preference requests now require cookie binding when a CSRF cookie exists and reject cross-site fetch metadata and mismatched Origin/Referer. Cookie-disabled forms require same-origin evidence. The site's HTML no-referrer policy can make legitimate native POSTs send `Origin: null`; those are accepted only with a bound cookie or same-origin fetch metadata, after rejecting cross-site metadata. Regression tests exercise foreign-token relay, explicit foreign origin, missing evidence, privacy forms, raw CSRF, and admin scope separation. + +Boards are validated completely rather than lossily sanitized; preference actions use a closed enum; IDs must be positive. Protected-board checks and target checks occur under the same write transaction as mutations. Global moderation validates submitted board against the canonical target board before changing anything, logging, or redirecting. A missing/broken board lookup now fails closed rather than committing with a fallback board. Return paths use the existing strict internal-path validator. No menu visibility assumption is an authorization boundary. + +## 9. Database, queries, and cache + +No dependency or preference-schema redesign was needed. Existing preference composite keys, thread/OP indexes, cascade foreign keys, and Boolean-domain triggers remain. Added `idx_reports_reporter_created(reporter_hash, created_at)` through the existing idempotent startup schema initialization, including existing databases. + +An actual browser database's EXPLAIN QUERY PLAN shows active thread/OP indexes and the composite preference key, rather than per-thread preference queries. Personal ordering needs a temporary sort over the board's active rows; adding another global thread index cannot eliminate ordering by a different viewer's preference. The report budget uses its new covering index. Foreign-key checking returned no violations. Evidence is in `output/post-actions-query-plans.log`. + +Index, catalog, and thread ETags now include personal flags and relevant authoritative metadata; catalog signatures include lock, OP content/media, reply counts, and timestamps. Personalized HTML is explicitly private, with revalidation when activity tracking is disabled and no-store when enabled. Hidden management is private/no-store and varies on Cookie. There is no separate action-state cache or optimistic client state requiring rollback. + +## 10. Concurrency, consistency, and failures + +Preference validation, flag updates, and default-row cleanup commit together in an immediate transaction. Reports serialize validation, the abuse budget, and insertion. Global moderation keeps target validation, state, pruning intent where applicable, and canonical audit logging in one immediate transaction, then redirects using the board already resolved inside it. + +Board/thread render loads use read snapshots so counts, posts, flags, and preferences agree; catalog/Hidden reads also recheck board access in their snapshots. Simultaneous pin/hide preserves both flags; concurrent duplicate reports yield one row; deletion races leave no preference/report references. Concurrent Sticky/Unsticky matches the last committed audit record. Repeated inverse/set operations remain idempotent. Archive rejects new personal flags and contradictory lock/unlock/sticky changes while permitting inverse cleanup. + +Failures return existing themed 400/403/404/409/500 responses or the explicit report-budget 429. Native forms navigate to authoritative results; no optimistic disappearance or false success state is introduced. Existing DB contention/error logging, session expiry, post ownership, destructive confirmation, and durable file cleanup mechanisms remain in place. + +## 11. Accessibility + +Triggers retain “Thread actions” accessible names and add a menu relationship, expanded state, and controlled ID. Action items have menu semantics. Arrow keys, Home, and End move through enabled actions; Escape closes and restores focus; Tab closes and follows native focus order; outside focus/click closes. Report opening records the original trigger and its existing dialog returns focus there. Disabled archived actions are skipped. One-open behavior and pagehide/pageshow cleanup prevent stale disclosures. Native no-JS details/forms, keyboard submission, and existing report controls remain usable. + +## 12. Themes and responsive/browser findings + +All nine built-in themes are exercised: forest, blue-sky, deep-orbit, terminal, dorfic, chanclassic, aero, neoncubicle, and fluorogrid. Menu, report input/confirmation, and hidden-management action contrast checks require at least 4.5:1. Screenshots cover 320px and 1280px in every theme; viewport geometry covers 320, 360, 375, 390, 430, 480, 768, 1280, and 1920px, all four edges, clipping ancestors, attached media, short/long content, keyboard interactions, history, and enlarged text. Additional tests use a 900×900 image, 60 replies, first/middle/final post report targets, nested scrolling, actual thread-update insertion, removed triggers, and reduced-motion emulation. The enlarged-text checks caught and fixed initial measurement and late header-layout movement. + +The project's Chromium, WebKit, Firefox, mobile Firefox, mobile WebKit, Firefox no-JS, and Chromium no-JS configurations exercise native persistence/report/moderation paths, including Hide from index/catalog/thread, Back/Forward after Hide, and admin toolbar Sticky/Unsticky. JS-only geometry tests skip the two no-JS configurations intentionally; native workflows run there. Enlarged-text tests check both an already-open menu following a growing fixed header and reopening after layout settles; a Firefox repeat run passed three times. A mobile WebKit geometry/history repeat run also passed three times. Browser checks wait for the existing activity-page refresh after history restoration, then verify the authoritative state on the listing, direct thread, or Hidden management page actually restored by the browser. Screenshots and browser diagnostics are kept under ignored `output/playwright/` directories. + +## 13. Tests and technical debt + +Added Rust HTTP/database regressions in `src/handlers/board/action_tests.rs` for explicit setters, isolation, forged boards/IDs/actions/CSRF, protected boards, GET rejection, 205-thread ordering/pagination, multiple hidden targets and new replies, direct navigation, archive cleanup, constraints/cascades, duplicate and concurrent reports, Unicode bounds, oversized bodies, report budgets, token binding, and deletion races. Added admin regressions for canonical logging, repeated and concurrent Sticky/Unsticky, forged board/session/CSRF, archive safety, missing targets, and lookup rollback. Existing cache/report tests were updated to the new contracts, and a summary-fragment regression checks balanced containers. Browser tests require exactly one action owner per index thread. + +Added the local `tests/e2e/post-actions.spec.ts` and adapted existing local action tests for portaled menu lookup and menuitem semantics. This repository intentionally ignores Playwright tooling; those files remain local and are not force-added. Removed the verified-unused Rust catalog split/sort helper, old menu bounds helper, and all-menu closing/reposition scans. No unrelated cleanup, commits, branches, or dependency changes were made. + +## 14. Validation results + +| Command | Final result | +| --- | --- | +| `cargo fmt --all --check` | Passed | +| `cargo check --workspace --all-targets --all-features` | Passed | +| `cargo clippy --workspace --all-targets --all-features -- -D warnings -D clippy::all -D clippy::pedantic -D clippy::nursery -D clippy::cargo` | Passed, no lint exceptions added | +| `RUST_TEST_THREADS=1 cargo test --workspace --all-features` | Passed: 1,395 tests in each library/binary target, 5 CLI integration tests, 1 fixture integration test, and 1 doc test; 7 existing ignores in each unit-test target | +| `cargo test --workspace --all-features action_tests` | 10 passed in each unit-test target | +| `cargo test --workspace --all-features server::handlers::admin::content::tests` | 15 passed in each unit-test target | +| `cargo test --workspace --all-features server::server::assets::tests` | 4 passed in each unit-test target after the final CSS change | +| `cargo build --workspace --all-features` | Passed | +| Playwright action matrix described below | Passed: 43 cases, 20 intentional configuration skips; 7 browser configurations and all 9 built-in themes | +| `git diff --check` | Passed | +| `node --check static/main.js` | Passed | +| `python3 tools/check_local_tooling.py` | Passed; local browser tooling remains untracked | + +Browser command: `RUSTCHAN_E2E_SKIP_BUILD=1 RUSTCHAN_AUDIT_OUTPUT=output/playwright/post-actions-verified npx playwright test tests/e2e/post-actions.spec.ts tests/e2e/audit-surfaces.spec.ts tests/e2e/audit-nojs-parity.spec.ts --grep 'personal pin/hide|menus follow|menu supports|all built-in themes|catalog.*(report|pin|hide)|public report|poll creation' --workers=2`. + +The 20 browser skips comprise 12 existing no-JS-parity test gates on other JavaScript projects, 6 JavaScript-only geometry/theme cases on the two no-JS projects, and 2 existing surface-test gates on Chromium no-JS. The new native persistence/report/moderation workflow passes in all seven configurations, including both no-JS projects. + +Rust logs are under `output/post-actions-*.log`; the complete serial run is `output/post-actions-tests-serial.log`. Browser results, screenshots, and HTML report are under `output/playwright/post-actions-verified/`, with the command log at `output/post-actions-browser-verified.log`. The earlier default parallel runs exposed the pre-existing shared media fault-injection race noted below; a full earlier parallel pass and isolated PDF test also passed. The final serial command runs every non-ignored test without skipping that test, and concurrency regressions still explicitly issue concurrent operations within their test bodies. + +## 15. Deliberately preserved boundaries and remaining limits + +- Personal preferences retain the existing IP/Tor identity model. Public users sharing an identity/IP can share preferences and a report budget; clearing public-IP browser cookies does not clear SQLite state. Migrating to account/browser-only identity would be a separate product change. +- Hide remains an active-listing preference. It does not censor direct links, search, archives, feeds, API access, individual replies, or moderation records. Valid archived preferences are retained and can be cleared explicitly. +- Report reason truncation remains compatible with existing behavior. The abuse budget counts retained report rows; resolution still counts for the hour, while deletion of target/report records can release that count. No extra historical identity ledger was introduced. +- Existing non-menu posting/voting CSRF compatibility, account/session architecture, self-action expiration, and file cleanup were not redesigned. Menu-triggered report/preference paths are hardened and privileged actions continue using session-scoped protection. +- Browser tests cover the project's desktop/mobile emulation and enlarged text. Physical device pinch-zoom, assistive-technology speech output, and transport outage injection were not manually exercised. Native POST errors and keyboard/focus semantics are covered. +- Seven pre-existing Rust tests are ignored by the repository; no ignore/lint bypass was added for this work. The default parallel runner can expose an existing shared thumbnail-fault-injection race in `primary_upload_accepts_pdf_when_board_enables_pdf`; the isolated test passes. The final complete suite is also run with `RUST_TEST_THREADS=1` to avoid that unrelated fixture race without changing media code. + +## Files changed + +| Area | Files | +| --- | --- | +| Database | `src/db/schema.rs`, `src/db/threads.rs` | +| Personal actions/reporting | `src/handlers/board.rs`, `src/handlers/board/access_preferences.rs`, `src/handlers/board/catalog.rs`, `src/handlers/board/pages.rs`, `src/handlers/board/create_thread.rs`, `src/handlers/board/reports.rs` | +| Rendering and moderation | `src/handlers/render.rs`, `src/handlers/thread.rs`, `src/handlers/admin/content.rs` | +| Templates and assets | `src/templates/board.rs`, `src/templates/thread.rs`, `static/main.js`, `static/style.css` | +| Rust regressions | `src/handlers/board/action_tests.rs`, `src/handlers/board/tests.rs`, `src/server/server/assets.rs` | +| Audit | `docs/post-actions-audit.md` | +| Local ignored browser files | `tests/e2e/post-actions.spec.ts`, `tests/e2e/audit-surfaces.spec.ts`, `tests/e2e/audit-nojs-parity.spec.ts` | diff --git a/docs/posting-lifecycle-audit.md b/docs/posting-lifecycle-audit.md new file mode 100644 index 00000000..9f3fd45b --- /dev/null +++ b/docs/posting-lifecycle-audit.md @@ -0,0 +1,418 @@ +# Posting and thread lifecycle audit + +Audit date: 2026-10-06. Scope: public creation/replies, owner mutations, +moderation, quoting, ordering, retention, retry behavior, SQL/filesystem recovery, +and browser workflows. No dependencies, commits, branches, or broad redesign were +introduced. The starting checkout was clean. + +## 1. Existing architecture + +`POST /{board}` and `POST /{board}/thread/{id}` share the bounded multipart parser +in `handlers/mod.rs` and the posting service in `handlers/posting.rs`. The parser +validates CSRF, UTF-8, control/upload multiplicity and byte budgets, and owns +temporary uploads. The posting service checks identity, bans, access, cooldown, +CAPTCHA, text/poll rules and media policy. Expensive preparation releases its +pooled connection. A policy callback rechecks mutable settings after acquiring +SQLite's writer lock. + +`db/threads.rs` creates the thread/OP or reply in an immediate transaction. This +bundle includes the submission receipt, counters/bump, optional poll, filesystem +publication intent, required media work and, for new threads, durable coalesced +retention work. Search updates use database triggers. Thread/post IDs are distinct +global AUTOINCREMENT identifiers; an OP post ID must not be assumed to equal its +thread ID. Replies display their post IDs rather than a separate reply sequence. + +Owner edit/delete routes check signed ownership grants and time windows. +Administrator routes check session, origin and scoped CSRF. Deletions commit +durable cleanup intents; filesystem operations happen after SQL commit and +recheck shared-file references. Retention workers read current board settings +under their own immediate transaction. Startup and periodic reconciliation +recover outstanding work. + +Server-rendered and upload-enhanced forms share this backend. Native success is +303 followed by GET; upload XHR success is 204 with `X-Rustchan-Redirect`. Handled +XHR posting rejections intentionally use transport 200 with JSON `error` and +`X-Rustchan-Error-Status`; the browser renders an error and does not navigate as +if accepted. Some earlier validation paths return an ordinary error status. +HTML cache signatures include posts, counts, state, media and poll changes; +cookie-sensitive views use the existing Vary/private-cache rules. + +## 2. State model + +| Current state | Public reply | Owner edit/delete | Administrator / retention behavior | +| --- | --- | --- | --- | +| Active, unlocked | Allowed after current policy checks | Allowed within ownership/window rules; OP self-delete requires no replies | Lock, sticky, archive or delete | +| Active, locked | Rejected, including sage and media-only replies | Rejected | Unlock, archive or delete; participates in active retention | +| Active, sticky | Same as its lock state | Same as its lock state | Exempt from active thread cap; global sticky differs from personal pin | +| Archived | Rejected independently of the lock flag | Rejected | Readable at the same URL; deletion/archive retention allowed | +| Deleted / destructively pruned | Missing target; rejected | Missing target | Cascaded content removed; audit history and bounded retry evidence retained | + +Sticky and locked can coexist. Archiving normally sets both archived and locked; +the internal unlock helper cannot make an archived thread publicly replyable. +There is no public move or unarchive posting workflow. Personal pin/hide state +affects presentation, not posting authorization or global retention protection. + +## 3. Enforced invariants + +- A committed creation includes one thread and one matching OP. Every reply has + an existing parent and matching board; foreign keys and domain triggers protect + ownership, and the reply API now explicitly rejects the OP flag. +- Current parent state and mutable posting policy are checked under the same + SQLite write lock as insertion. Closed/missing targets cannot be bypassed by a + stale form or crafted request. +- A successful receipt names one canonical operation and target. Reuse for + another board, actor, operation or reply target cannot create content. +- SQL failure, including commit failure, rolls back posts, counters, bump, + receipts, required jobs and filesystem intents and releases the transaction. +- Successful SQL commit is not subsequently reported as failed because auxiliary + spam scheduling or a redundant post read fails. +- Stored reply counts equal persisted non-OP rows; listing media counts equal + their established primary-file count. Sage increments counters without bumping. +- Deletion preserves intentionally retained audit history and prevents an old + accepted token from resurrecting content during receipt retention. +- SQL and filesystem publication are not a shared transaction: durable intents + make publication/deletion recoverable, and media workers wait for publication. + +These guarantees concern authorized application operations with SQLite foreign +keys/triggers enabled. They do not promise survival of arbitrary external database +corruption, deleted backups, or irrecoverable storage loss. + +## 4. Bugs found and repaired + +| Finding | Repair | +| --- | --- | +| Cascading content deletion erased the only submission receipt, permitting the same new-thread token to create replacement content. | Historical receipt table and BEFORE DELETE triggers on posts, threads and boards. Parent triggers handle cascade ordering. | +| A successful reply retry was rejected after lock/archive before receipt lookup. | Replay lookup precedes early parent-state validation; new submissions still use the authoritative transactional check. | +| Receipts were not bound to operation/target; wrong-scope token collisions became database errors. | Explicit operation/target validation and typed 409 conflicts for scope reuse. | +| A thread removed during preparation became an internal error at insertion. | Missing parent produces typed 404 and staged-media rollback. | +| Required deferred media jobs and state were recorded after posting commit. | Persist media intent and processing state in the creation transaction. | +| Workers could claim a media job before its staged source was published. | Claim query excludes media sources still named by upload-finalize intents. | +| Auxiliary scheduling and a post-commit read could turn accepted posting into apparent failure. | Capture creation time before commit; warn on auxiliary scheduling/notification failures. | +| Ban-and-delete trusted submitted identity and OP/board classification; ban, deletion and audit records could partially succeed. | Resolve and validate the stored target, then commit authorization, ban, deletion and both audit entries together. | +| Administrator deletion could succeed without an audit record or after stale authorization. | One immediate transaction covers session, target, deletion and audit. | +| Owner action board permissions/settings could change between preparation and mutation. | Recheck permission and edit preparation policy inside the mutation transaction. | +| Repeated scalar multipart controls used last-value-wins behavior. | Reject repeated scalar controls; retain intentional repeated poll options and established upload-slot rules. | +| Quotes linked overflowing/nonpositive/Unicode or embedded malformed IDs. | Require positive ASCII i64 IDs, canonical targets, valid boundaries and valid board syntax; invalid references remain escaped text. | + +## 5. Race conditions + +Write transactions define the winner: if a reply commits before a lock/delete/ +prune transaction, the later action operates on that committed state; if the +state transition commits first, insertion sees the closed or missing parent and +rejects. Deletion may legitimately remove an earlier successful reply. A retry +after that deletion reports the missing canonical target without creating another +post. Moderation authorization and audit failure now share this serialization. + +Preparation races with board access, media settings, CAPTCHA policy, filters, +cooldown, bans and session loss are covered by policy revalidation. Concurrent +identical submissions serialize receipt lookup and insertion under the writer +lock. Losing prepared uploads are cleaned rather than published twice. + +## 6. Transaction-boundary changes + +Required audio/video job JSON, media state and response creation time moved into +the post transaction. Administrator delete and inline ban/delete now use RAII +immediate transactions for all required SQL work. Owner actions gained a narrow +validation callback invoked inside their existing immediate transaction. Existing +atomic reply counter/bump, OP/poll, FTS and durable pruning boundaries remain. +Filesystem finalization and expensive media/body preparation stay outside the +SQL write transaction. + +## 7. Thread creation + +Existing empty/media, board-access, size, poll and CSRF checks remain authoritative. +No image-required board rule or separate disabled-board flag exists; password +access and configured media permissions determine availability. Creation now +also atomically records required media work and safely handles deleted receipts +and wrong-operation reuse. Failed media scheduling and deferred commit failure +leave zero new threads/posts/receipts. Prune intent remains a required part of +public creation, so scheduling failure cannot leave permanent unhandled overflow. + +## 8. Replies + +Current board/thread ownership, lock, archive, cooldown and access are enforced +at insertion. Missing targets now return a normal recoverable 404. A reply cannot +be inserted with `is_op=true`. Successful retries return the original post anchor +even if the thread later closes; deleted-target retries cannot recreate a row. +Creation timestamps needed for ownership cookies come from the transaction. + +## 9. Bump and sage + +Existing semantics were preserved. Normal replies bump only when the current +pre-insertion reply count is below `boards.bump_limit`; sage replies never bump. +The checkbox accepts `1`, case-insensitive `on`, or case-insensitive `true`, not +arbitrary name/email text. Media-only replies follow the same rule. Deletion +reduces reply count without rewinding bump time; it can make later replies +eligible to bump again under the established count rule. Edits and moderation +state changes do not bump. Active ordering is sticky, bump time, ID descending; +archive order is bump descending, ID ascending. Existing tie/pagination and +concurrent bump-limit tests remain in the full suite. + +## 10. Quotes + +`>>id` remains a local fragment. `>>>/board/id` resolves through that board's post +redirect and supports explicit same-board cross-thread and cross-board links; +`>>>/board/` links its index. Search rendering scopes otherwise local links. +Parsing now rejects invalid numeric ranges, Unicode numerals, negative/zero IDs, +oversized board names and quote-like embedded suffixes. Leading zeros resolve to +the canonical ID. No eager quote-target database lookup was added: missing or +deleted targets cannot make submission fail, and protected resolver/preview +routes retain access checks. Existing HTML escaping remains intact. Backlinks +are derived in the browser and deduplicated/rebuilt after updates, not stored +foreign-key records. Browser coverage checks repeated quotes, OP quoting, +crosslink routing, malformed references, escaping and reload behavior. + +## 11. Lock and unlock + +Fresh locked replies are rejected regardless of UI, sage or attachment. Current +state is checked again after preparation under the write lock. Unlock permits a +previously rejected token to be retried because rejection did not consume it. +An already accepted token is replayed rather than rejected after locking. Archive +is checked independently, so clearing only `locked` cannot reopen an archive. +The existing HTTP retry suite exercises lock/reject/unlock/corrected retry. + +## 12. Deletion + +Administrator and inline ban/delete actions use the actual database post, board, +thread, OP flag and posting identity. Forged details are rejected before mutation. +Audit insertion failure rolls back bans and content deletion. Owner actions +retain token/window/closed-thread/OP-with-replies restrictions and now recheck +current board permission. Database cascades remove posts, reports, polls/votes, +personal thread preferences and live receipts; FTS triggers remove search rows. +Moderation history remains. Shared media cleanup keeps the durable, reference- +checked deletion pipeline. Historical receipts intentionally have no content +foreign keys. Existing redirect routes and ban/delete reply anchors are preserved. + +## 13. Pruning + +Active `max_threads` applies to non-sticky, non-archived threads. Locked active +threads count; sticky active threads are exempt. Overflow chooses the oldest +bump/ID according to deterministic active ordering. Current archive policy +selects archive versus destructive prune. `max_archived_threads` is a count cap +and includes archived sticky threads. Both limits are validated before destructive +maintenance. Archive retention follows archive ordering. Creation commits a +coalesced durable job; workers/startup reconciliation converge to current caps. +Temporary overflow during deferred maintenance is an existing deliberate behavior. +Tests race 25 creators at cap one, verify one pending intent, then verify exact +retention convergence; existing worker tests exercise real automatic maintenance. + +## 14. Limits and boundaries + +Body limits count Rust Unicode scalar values, not bytes or grapheme clusters: +4096 characters; empty text requires an accepted attachment. Names are trimmed +and truncated to 64 characters, subjects to 128. The multipart parser additionally +limits text fields to 64 KiB, controls to 64, total payload to 512 MiB plus bounded +framing, and validates board/global per-type upload limits. Submission tokens +are bounded to 128 ASCII alphanumeric/underscore/hyphen characters. Existing +boundary/UTF-8/media tests cover rejection and cleanup. Thread cap/tie tests and +concurrent bump-limit tests verify the supported count boundaries. + +There is no configured hard reply cap or image-count cap in this version. +`bump_limit` stops bumps, not replies; inventing a new 500-reply limit would change +the product. Upload byte caps and thread/archive caps remain the applicable limits. + +## 15. Duplicate protection + +Opaque form tokens are bound to actor, board, operation and reply target. +SQLite serializes duplicate lookup/create; no body-text deduplication was added. +Deleted receipts retain accepted IDs for the existing seven-day receipt horizon, +with indexed expiry cleanup on new receipt insertion. Live receipts have the +same opportunistic expiry model. A deleted canonical target returns 404. Failed +validation/commit leaves the token reusable for a corrected submission. Native +PRG, XHR redirects and existing busy-button guards remain. Distinct tokens still +allow intentionally identical content. Legacy empty-token clients remain supported +without an idempotency guarantee. + +## 16. Stale forms + +The database wins over embedded browser state. Lock, archive, delete, changed +board access/media/filter/CAPTCHA policy, bans, cooldown and revoked admin bypass +are evaluated again before insertion. Owner actions recheck board permission at +mutation. Native error recovery supplies escaped copyable drafts; XHR preserves +the current form and renders a rejection. File selections must be made again +after native navigation. Real browser tests submit attachment-backed forms after +lock/archive/delete and assert saved text and zero inserted rows/filesystem intents. + +## 17. Counters + +`threads.reply_count` is stored and changes with transactional reply insert/delete. +Board thread counts and listing `image_count` are computed. The latter is the +legacy count of non-NULL primary `file_path`, including video/PDF/general files; +secondary/audio-only slots do not all represent an additional image. This naming +and display behavior were deliberately retained. New invariant checks compare +stored reply counts to COUNT(non-OP), listing media counts to COUNT(primary file), +and verify one OP, board ownership, foreign keys and transaction release after +every operation. Bursts mix text and image rows and preserve sage ordering. + +## 18. SQLite, recovery and performance + +Production/test pools use file-backed WAL, foreign keys, synchronous FULL, a +one-second busy timeout and bounded pool checkout. Immediate posting/mutation +transactions avoid deferred write-lock upgrades. Busy errors remain retryable +503 with `Retry-After: 1` on the native error path; XHR exposes 503 through its +handled-error contract. A new contention test holds the writer, checks zero +partial writes and then successfully retries the same token. Deferred-FK commit +failure tests verify rollback and subsequent reuse. Existing archive tests cover +process interruption, restart, overlapping workers and maintenance commit failure. + +EXPLAIN QUERY PLAN on an installed current test database showed primary-key index +searches in both receipt UNION branches, the covering `idx_threads_active_order` +for active ordering, and `idx_submission_tombstones_created` for expiry. No eager +quote-resolution N+1 queries or application-wide posting mutex were introduced. +The final review found full receipt scans for board/thread/post deletion. Three +target indexes now support the preservation triggers and foreign-key cascades; +the upgrade test also reconstructs their absence in the previous schema. Final +plans show `SEARCH ... USING INDEX idx_post_submissions_board/thread/post` for +all three target lookups, replacing the observed full-table scans. +The media claim guard scans outstanding publication intents, which are normally +short-lived; persistent storage trouble can delay jobs until reconciliation. +The exact additive schema repair recognizes only the new receipt objects; its +upgrade regression preserves existing live tokens and subsequent deleted history. + +## 19. Security and resource checks + +CSRF, scoped admin CSRF/origin, board password grants, signed ownership cookies, +parameterized SQL and current-state checks remain the boundaries. Repeated scalar +controls and malformed tokens now fail closed. Inline bans cannot select a +different actor via hidden fields or turn a reply into OP deletion. Quotes accept +validated IDs after escaping; malformed input stays text. Existing bounded +multipart/framing/field parsing, UTF-8 rejection, media sniffing, canonical path +validation, upload admission, timeouts, cooldown, CAPTCHA and filesystem reference +checks were reviewed and retained. No new aggressive rate limit was added. +HTTP framing is handled by the existing Axum/HTTP stack and deployment proxy; +this pass does not claim a separate proxy request-smuggling penetration test. + +## 20. UX and compatibility + +Deleted-during-preparation requests now receive a clear missing-target error. +Receipt reuse gives an actionable reload conflict. Accepted posts retain success +when auxiliary background work fails. Existing inline/generic draft recovery, +upload progress, double-submit controls, keyboard/modal behavior, mobile layout, +form field names, element IDs, routes, CSS/JS hooks and no-JS fallbacks are retained. +Browser validation caught and corrected a temporary ban/delete anchor regression. +No broad visual redesign was made. XHR's established handled-error transport +contract is tested explicitly, rather than mistaking transport 200 for acceptance. + +## 21. Added and expanded regression coverage + +New tracked suites: `src/db/threads/lifecycle_tests.rs` (10 tests) and +`src/handlers/posting/lifecycle_tests.rs` (5 tests). Additional regressions cover +schema upgrade, repeated scalar multipart controls, malformed quote grammar, +moderation audit rollback and forged inline-ban targets. An old deletion fallback +test now verifies fail-closed preservation when board lookup breaks. Obsolete +best-effort deletion board fallback helpers/tests were removed. + +The existing local Playwright harness was expanded with +`tests/e2e/posting-lifecycle.spec.ts` (three workflows) and strengthened forged-board +archive assertions. Browser tooling is intentionally ignored and prohibited from +tracking by `tools/check_local_tooling.py`; it was not force-added. Tracked Rust +regressions remain available to CI. Existing posting, database, media, deletion, +archive, authorization, FTS and cache tests also run in the complete Rust suite. + +## 22. Concurrency scenarios exercised + +- Barrier-synchronized mixed text/image sage bursts of 2, 5, 10, 25 and 64 clients: + every accepted ID unique, exact reply/media counts, no bump, valid parent rows. +- Twenty-five simultaneous thread creations at cap one: durable coalesced work, + deterministic newest survivor after pruning, no permanent unmanaged overflow. +- Reply against lock, archive, deletion and pruning: eight repetitions per state + with a barrier; only committed-before-transition or typed rejection outcomes. +- Three fresh HTTP runtimes: eight identical thread submissions, eight identical + replies, eight distinct thread submissions and eight distinct replies per run; + exact canonical redirects, row counts and receipt counts. +- Existing service races include cooldown contention and concurrent identical + uploads; existing DB races cover bump limits, moderation/archive and overlapping + retention workers. Preparation callbacks force stale policy/session/parent + windows without timing-only sleeps. +- A fixed-seed 160-operation sequence checks invariants after lock/unlock, + image/text/sage reply, deletion and sticky changes, then verifies deleted search + state. Fault injection covers required media job failure, auxiliary spam job + failure, commit failure and moderation audit failure. + +## 23. Validation + +Final command results follow. The complete Rust suite includes the dedicated +lifecycle/concurrency suites and the existing posting, database, archive, +filesystem, authorization and recovery regressions. + +| Command | Result | +| --- | --- | +| `cargo fmt --all --check` | Passed | +| `cargo check --workspace --all-targets --all-features` | Passed | +| `cargo clippy --workspace --all-targets --all-features -- -D warnings -D clippy::all -D clippy::pedantic -D clippy::nursery -D clippy::cargo` | Passed | +| `cargo test --workspace --all-features` | Passed: 1,413 tests per lib/binary target; 5 integration, 1 native-render fixture and 1 doctest; 2,833 executed, 14 pre-existing ignored performance/child fixtures | +| `cargo build --locked --bin rustchan-cli` | Passed | +| Playwright posting/thread, dedicated lifecycle, submission idempotency, archive, admin ban/delete and password-board suites; Chromium, Chromium no-JS, mobile WebKit, two workers | Passed: 46 workflows, 8 project-specific skips | +| Final rebuilt-binary lifecycle/archive/CAPTCHA recheck; Chromium, Chromium no-JS, mobile WebKit and desktop WebKit | Passed: 24 checks, 4 CAPTCHA project skips; includes explicit no-JS CAPTCHA contexts in Chromium/WebKit | +| `git diff --check` | Passed | +| `python3 tools/check_local_tooling.py` | Passed | + +Toolchain: Rust/cargo 1.99.0, matching the repository policy. No lints or tests were +disabled to pass validation. Browser project-specific skips distinguish no-JS +modal coverage, mobile-only cookie coverage and engine-independent HTTP races. +The full suite exposed one transient failure in the unrelated configuration-save +lock regression; it passed in isolation and the final complete rerun. New index +upgrade checks initially exposed a missing table-index repair allowlist entry; +that migration was corrected and both legacy/current upgrade regressions pass. +Final Rust output: `/tmp/rustchan-posting-lifecycle-rust-tests.log`. +Final browser output: `/tmp/rustchan-posting-lifecycle-browser-final-pass.log`; +JSON/HTML/traces: `/tmp/rustchan-posting-lifecycle-browser-final-pass/`. +Final schema/CAPTCHA browser output: +`/tmp/rustchan-posting-lifecycle-browser-schema-captcha.log`; artifacts use the +same name without `.log`. + +Browser commands, using the existing local harness and the freshly built binary: + +```sh +RUSTCHAN_E2E_SKIP_BUILD=1 RUSTCHAN_AUDIT_OUTPUT=/tmp/rustchan-posting-lifecycle-browser-final-pass npx playwright test \ + tests/e2e/posting-thread.spec.ts tests/e2e/posting-lifecycle.spec.ts \ + tests/e2e/submission-idempotency.spec.ts tests/e2e/archive-production.spec.ts \ + tests/e2e/admin-ban-delete.spec.ts tests/e2e/password-boards.spec.ts \ + --project=chromium --project=chromium-nojs --project=mobile-webkit --workers=2 +RUSTCHAN_E2E_SKIP_BUILD=1 RUSTCHAN_AUDIT_OUTPUT=/tmp/rustchan-posting-lifecycle-browser-schema-captcha npx playwright test \ + tests/e2e/posting-lifecycle.spec.ts tests/e2e/archive-production.spec.ts tests/e2e/captcha.spec.ts \ + --project=chromium --project=chromium-nojs --project=mobile-webkit --project=webkit --workers=2 +``` + +## 24. Deliberately retained limitations + +- Retention is asynchronous and may temporarily exceed a cap; durable intents + and existing reconciliation converge it. Making creation synchronously delete + content would change established latency and retention behavior. +- No new hard reply/image cap, board-disabled switch, moderator-scoped role, + thread move or public unarchive feature was invented. +- Receipt protection is bounded to seven days with opportunistic cleanup. A + fresh or empty token, browser draft submitted from another independent form, + or a restored backup missing newer receipts is not a global exactly-once + guarantee. Changing identity cannot replay the original receipt and returns + a conflict for a still-retained token. This avoids blocking legitimate identical posts. +- CAPTCHA challenges remain process-local and single-attempt. Simultaneous use + before the first commit may reject the losing attempt; retries after a stored + successful receipt bypass another challenge check. Multi-instance CAPTCHA + coordination was not added. +- Filesystem/SQL atomicity relies on the existing durable journal and recovery. + Processing failure preserves accepted source media with an explicit failed + state; a full background queue retains its established skipped-processing + failure state. Auxiliary spam-job failure is logged after acceptance. +- The additive receipt table, triggers and indexes change the exact baseline + schema. Older unpatched binaries may reject that schema; database rollback + across binary versions needs the project's existing compatible-backup process. +- Local `>>id` scope, computed backlink behavior, legacy media-count meaning, + second-resolution bumps and numbered-page movement under live activity remain. +- Concurrency evidence is from isolated local SQLite WAL runtimes and browser + engines, not a distributed deployment, every possible crash instruction, or + production load benchmark. The existing subprocess archive recovery tests are + included. Browser harness changes remain local by repository policy. + +## Files changed + +Tracked implementation: `src/db/posts.rs`, `src/db/schema.rs`, `src/db/threads.rs`, +`src/handlers/admin/content.rs`, `src/handlers/admin/moderation.rs`, +`src/handlers/mod.rs`, `src/handlers/posting.rs`, `src/handlers/thread.rs`, +`src/utils/sanitize.rs`. + +New tracked regressions: `src/db/threads/lifecycle_tests.rs`, +`src/handlers/posting/lifecycle_tests.rs`. Report: `docs/posting-lifecycle-audit.md`. + +Local ignored browser changes: `tests/e2e/posting-lifecycle.spec.ts`, +`tests/e2e/archive-production.spec.ts`. diff --git a/docs/security-attack-surface.md b/docs/security-attack-surface.md new file mode 100644 index 00000000..ebad3242 --- /dev/null +++ b/docs/security-attack-surface.md @@ -0,0 +1,157 @@ +# RustChan request attack surface — 2026-10-06 + +Inventory completed against the unmodified route composition before implementation changes. This file records existing controls and review targets; the companion audit report records final changes and results. All GET routes also accept HEAD. No RSS, external JSON posting, legacy posting, or separate upload endpoint exists. Unknown routes use the 404 fallback. Board media and public posting are shared paths; no alternate posting parser was found. + +## Shared boundary + +All main-server routes consume bounded headers (64 KiB aggregate, 32 KiB per value), reject Transfer-Encoding, pass through transport/client identity, timeout, response-header and browsing-rate middleware. Client identity is derived from ConnectInfo; forwarding headers are used only with behind_proxy and a CIDR-trusted immediate peer. Cookies are untrusted and parsed by CookieJar. Host, Origin, Referer and fetch metadata are consumed by origin/cookie handling; Range and validators are used by media/static serving; X-Requested-With selects JSON post-error responses. Before this audit there was no explicit service-level URI limit or general streaming form cap beyond Axum's 2 MiB extractor default. + +## Control and cost profiles + +| Profile | Authentication / authorization / CSRF | Parameters and bodies | Database / filesystem / CPU / memory / rate controls / output | +|---|---|---|---| +| Public read | Anonymous; board password and admin bypass where applicable; no mutation CSRF | Board slug, numeric IDs; page query, theme/activity cookies, ETag; empty expected body | Board/post/poll/preference reads; HTML text/attributes and generated URLs; escaped metadata, prepared body_html. Browsing budget; catalog/hidden/thread/archive materialization merits review. | +| Search | Anonymous; board viewing permission | board; q (256 Unicode scalars after trimming), page i64; empty body | Two parameterized FTS prefix queries (12 alphanumeric terms), count plus sort and 20-result page. Browsing budget only before audit; SQL cost independent of result size. Query echo HTML/URL escaped. | +| Posting | Board posting permission, bans, optional CAPTCHA; public CSRF; cooldown and submission-token checks at commit | board / thread id; multipart _csrf, submission_token, name, subject, body, deletion_token, sage, captcha_id/answer, file/audio_file/image_file, poll fields | Stream to RAII temporary files; 512 MiB aggregate data, 516 MiB envelope, 64 fields, 64 KiB text/unknown fields, 64 KiB preamble/part headers, 200-byte boundary; board media limits; 10-minute body deadline. One concurrent public media request. Content sniffing, hashing, decoding/transcoding, thumbnails; staged filesystem publication and short IMMEDIATE transaction; job scheduling. Post limits and escaped metadata/markup. Browsing policy legacy includes writes, reads policy excludes writes. | +| Public form | Anonymous; board access, ownership grants or ban checks as required; CSRF for actions (menu actions also bind cookie/origin) | Form _csrf plus route-specific fields: report post/thread/board/reason; appeal reason; vote poll/option/board; edit body; delete ownership; unlock password/return_to; preference flags/return_to | Axum default 2 MiB unless route sets 64 KiB. Report reason 256 scalars; appeal 512; report limit 20/hour per identity with open-report dedup; one appeal/day; poll unique voter; transactional target validation. Cookie preferences and internal redirects. Unlock Argon2 and failure counters. | +| CAPTCHA | Anonymous; syntactically valid board/id before audit | id 32 hex; query board ASCII identifier; empty body | PNG generation/filtering and challenge map; five-minute retention; browsing budget only; synchronous CPU and full-map expiry scan before audit. Output server-generated PNG. | +| Media | Board view permission including media password gate | wildcard media_path; v; Range, If-* | Checked board-relative canonical containment; no arbitrary filesystem path; ServeFile streaming/ranges and MIME/nosniff; UUID media identifiers; browsing exemption; DB authorization lookup and filesystem metadata. | +| Setup | Uninitialized setup or administrator-authorized reopen; scoped CSRF and pending setup grant | preset/site metadata/board policies/admin credentials/theme, password token, _csrf | Default form cap; configuration validators; password hashing, SQLite setup/config writes and staged settings persistence. Rendering escapes submitted metadata; no unauthenticated reopen. | +| Admin read | Valid expiring admin session except login entry; no CSRF for reads | Dashboard filters/page/domain/search, logs after/max bytes, jobs selection, mod-log page, ip_hash; empty body | DB reads and diagnostic/filesystem/log reads, bounded log tail; private no-store; HTML escapes stored posts/reports/reasons. No distinct moderator role: admin accounts hold full privilege. Update/restart progress endpoints require session. | +| Admin form | Admin session (login accepts anonymous scoped login CSRF) + session-scoped CSRF + same-origin policy (valid CSRF fallback where designed); sensitive account operations reauthenticate | _csrf and typed/allowlisted per-handler fields; board/thread/post IDs; config sections/actions; ban reason/filter/metadata/settings/secrets | Default form cap; config and board validators; parameterized writes/mod log, controlled filesystem/config mutations, restart/update coordinator, maintenance gate; HTML/redirect outputs escaped/encoded. Password hashing and maintenance can be expensive. | +| Admin asset | Admin session originally checked after multipart iteration; scoped CSRF and origin | Multipart _csrf, target board and file/image/icon metadata; favicon 5 MiB body, banner 8 MiB body | Limited bytes per part, image content validation and thumbnail/storage writes; rejected-file cleanup; scalar/part-count behavior merits review. | +| Restore | Admin session and origin checked before streaming; maintenance gate; scoped CSRF before restore mutation | Multipart archive/password/_csrf/target/options; route extractor 20 GiB, actual stream 8 GiB; archive expansion total 8 GiB; bounded manifest and entry paths/counts | RAII temp archive, archive-layout/preflight validation, SQL schema/trust validation, recomputed body_html, staged rollback/persistent recovery; password verification and disk/database replacement expensive. Timeout exemptions deserve review. | +| Admin backup read | Admin session; GET creates downloadable artifact but does not alter board content | board; inline/attachment choice; kind/filename for saved download | Validated backup names and containment; maintenance coordination, SQLite snapshots/archive creation or streaming download; expensive, timeout-exempt where applicable; private cache. | + +## Complete main-router route inventory + +Path parameter names appear in braces. Handler structs are the authoritative per-field schema and the named cost profile supplies the remaining trust-boundary columns above. + +| Method | Path | Handler | Profile | +|---|---|---|---| +| GET | `/healthz` | `observability::healthz` | Public read | +| GET | `/readyz` | `observability::readyz` | Public read | +| GET | `/metrics` | `observability::metrics` | Public read | +| GET | `/favicon.ico` | `crate::handlers::favicon::serve_favicon_ico` | Public read | +| GET | `/favicon-16x16.png` | `crate::handlers::favicon::serve_favicon_16` | Public read | +| GET | `/favicon-32x32.png` | `crate::handlers::favicon::serve_favicon_32` | Public read | +| GET | `/apple-touch-icon.png` | `crate::handlers::favicon::serve_apple_touch_icon` | Public read | +| GET | `/captcha/{id}` | `crate::handlers::captcha::serve_captcha_image` | CAPTCHA | +| GET | `/android-chrome-192x192.png` | `crate::handlers::favicon::serve_android_chrome_192` | Public read | +| GET | `/android-chrome-512x512.png` | `crate::handlers::favicon::serve_android_chrome_512` | Public read | +| POST | `/nsfw/accept` | `crate::handlers::board::accept_nsfw` | Public form | +| GET | `/theme/{theme}` | `crate::handlers::board::set_theme` | Public read | +| POST | `/preferences` | `crate::handlers::board::set_user_preferences` | Public form | +| GET | `/banned` | `crate::handlers::board::banned_page` | Public read | +| GET | `/theme-css/{theme}` | `crate::handlers::board::serve_theme_css` | Public read | +| GET | `/banner/assets/{id}` | `crate::handlers::banner::serve_banner_asset` | Public read | +| GET | `/banner/external/{id}` | `crate::handlers::banner::external_banner_warning_page` | Public read | +| GET | `/banner/external/{id}/continue` | `crate::handlers::banner::external_banner_continue` | Public read | +| GET | `/setup` | `crate::handlers::setup::setup_get` | Setup | +| POST | `/setup/review` | `crate::handlers::setup::setup_review` | Setup | +| POST | `/setup/finish` | `crate::handlers::setup::setup_finish` | Setup | +| GET | `/` | `crate::handlers::board::index` | Public read | +| GET | `/{board}` | `crate::handlers::board::board_index` | Public read | +| POST | `/{board}` | `crate::handlers::board::create_thread` | Posting | +| GET | `/{board}/unlock` | `crate::handlers::board::board_unlock_page` | Public read | +| POST | `/{board}/unlock` | `crate::handlers::board::unlock_board_access` | Public form | +| GET | `/{board}/catalog` | `crate::handlers::board::catalog` | Public read | +| GET | `/{board}/hidden` | `crate::handlers::board::hidden_threads` | Public read | +| POST | `/{board}/thread-preference` | `crate::handlers::board::update_thread_preference` | Public form | +| GET | `/{board}/archive` | `crate::handlers::board::board_archive` | Public read | +| GET | `/{board}/search` | `crate::handlers::board::search` | Search | +| GET | `/{board}/thread/{id}` | `crate::handlers::thread::view_thread` | Public read | +| POST | `/{board}/thread/{id}` | `crate::handlers::thread::post_reply` | Posting | +| GET | `/{board}/post/{id}/edit` | `crate::handlers::thread::edit_post_get` | Public read | +| POST | `/{board}/post/{id}/edit` | `crate::handlers::thread::edit_post_post` | Public form | +| GET | `/{board}/post/{id}/delete` | `crate::handlers::thread::delete_post_get` | Public read | +| POST | `/{board}/post/{id}/delete` | `crate::handlers::thread::delete_own_post` | Public form | +| POST | `/report` | `crate::handlers::board::file_report` | Public form | +| POST | `/appeal` | `crate::handlers::board::submit_appeal` | Public form | +| POST | `/vote` | `crate::handlers::thread::vote_handler` | Public form | +| GET | `/api/post/{board}/{post_id}` | `crate::handlers::board::api_post_preview` | Public read | +| GET | `/{board}/post/{post_id}` | `crate::handlers::board::redirect_to_post` | Public read | +| GET | `/{board}/thread/{id}/updates` | `crate::handlers::thread::thread_updates` | Public read | +| GET | `/boards/{*media_path}` | `crate::handlers::board::serve_board_media` | Media | +| GET | `/admin/restart/status` | `crate::handlers::admin::restart::status` | Admin read | +| POST | `/admin/restart` | `crate::handlers::admin::restart::restart` | Admin form | +| GET | `/admin/updates/status` | `crate::handlers::admin::updates::status` | Admin read | +| POST | `/admin/updates/check` | `crate::handlers::admin::updates::check` | Admin form | +| POST | `/admin/updates/install` | `crate::handlers::admin::updates::install` | Admin form | +| POST | `/admin/network/settings` | `crate::handlers::admin::update_network_settings` | Admin form | +| POST | `/admin/appearance/defaults` | `crate::handlers::admin::update_visitor_defaults` | Admin form | +| POST | `/admin/secrets/rotate` | `crate::handlers::admin::rotate_secret` | Admin form | +| POST | `/admin/accounts/{action}` | `crate::handlers::admin::update_account` | Admin form | +| POST | `/admin/config/{section}` | `crate::handlers::admin::update_runtime_settings` | Admin form | +| GET | `/admin` | `crate::handlers::admin::admin_index` | Admin read | +| POST | `/admin/login` | `crate::handlers::admin::admin_login` | Admin form | +| POST | `/admin/logout` | `crate::handlers::admin::admin_logout` | Admin form | +| GET | `/admin/panel` | `crate::handlers::admin::admin_panel` | Admin read | +| GET | `/admin/site-health/jobs` | `crate::handlers::admin::admin_site_health_jobs` | Admin read | +| POST | `/admin/site-health/jobs/dismiss` | `crate::handlers::admin::dismiss_failed_site_health_jobs` | Admin form | +| GET | `/admin/log/live` | `crate::handlers::admin::admin_live_log` | Admin read | +| POST | `/admin/board/create` | `crate::handlers::admin::create_board` | Admin form | +| POST | `/admin/board/delete` | `crate::handlers::admin::delete_board` | Admin form | +| POST | `/admin/board/settings` | `crate::handlers::admin::update_board_settings` | Admin form | +| POST | `/admin/board/reorder` | `crate::handlers::admin::reorder_board` | Admin form | +| POST | `/admin/site/favicon` | `crate::handlers::admin::update_site_favicon` | Admin asset | +| POST | `/admin/board/favicon` | `crate::handlers::admin::update_board_favicon` | Admin asset | +| POST | `/admin/board/favicon/clear` | `crate::handlers::admin::clear_board_favicon_override` | Admin form | +| POST | `/admin/site/banner` | `crate::handlers::admin::upload_global_banner` | Admin asset | +| POST | `/admin/home/banner` | `crate::handlers::admin::upload_home_banner` | Admin asset | +| POST | `/admin/board/banner` | `crate::handlers::admin::upload_board_banner` | Admin asset | +| POST | `/admin/board/banner/clear` | `crate::handlers::admin::clear_board_banner_override` | Admin form | +| POST | `/admin/banner/update` | `crate::handlers::admin::update_banner_meta` | Admin form | +| POST | `/admin/banner/delete` | `crate::handlers::admin::delete_banner` | Admin form | +| POST | `/admin/banner/move` | `crate::handlers::admin::move_banner` | Admin form | +| POST | `/admin/thread/action` | `crate::handlers::admin::thread_action` | Admin form | +| POST | `/admin/thread/delete` | `crate::handlers::admin::admin_delete_thread` | Admin form | +| POST | `/admin/post/delete` | `crate::handlers::admin::admin_delete_post` | Admin form | +| POST | `/admin/ban/add` | `crate::handlers::admin::add_ban` | Admin form | +| POST | `/admin/ban/remove` | `crate::handlers::admin::remove_ban` | Admin form | +| POST | `/admin/report/resolve` | `crate::handlers::admin::resolve_report` | Admin form | +| GET | `/admin/mod-log` | `crate::handlers::admin::mod_log_page` | Admin read | +| POST | `/admin/filter/add` | `crate::handlers::admin::add_filter` | Admin form | +| POST | `/admin/filter/remove` | `crate::handlers::admin::remove_filter` | Admin form | +| POST | `/admin/site/settings` | `crate::handlers::admin::update_site_settings` | Admin form | +| POST | `/admin/theme/create` | `crate::handlers::admin::create_theme` | Admin form | +| POST | `/admin/theme/update` | `crate::handlers::admin::update_theme` | Admin form | +| POST | `/admin/theme/delete` | `crate::handlers::admin::delete_theme` | Admin form | +| POST | `/admin/db/check` | `crate::handlers::admin::admin_db_check` | Admin form | +| POST | `/admin/media/settings` | `crate::handlers::admin::update_media_settings` | Admin form | +| POST | `/admin/setup/reopen` | `crate::handlers::setup::admin_reopen_setup` | Admin form | +| POST | `/admin/setup/close` | `crate::handlers::setup::admin_close_setup` | Admin form | +| GET | `/admin/db/repair` | `crate::handlers::admin::admin_db_repair_status` | Admin read | +| POST | `/admin/db/repair` | `crate::handlers::admin::admin_db_repair` | Admin form | +| GET | `/admin/db/repair/status` | `crate::handlers::admin::admin_db_repair_status` | Admin read | +| GET | `/admin/db/repair/progress` | `crate::handlers::admin::admin_db_repair_progress_json` | Admin read | +| POST | `/admin/vacuum` | `crate::handlers::admin::admin_vacuum` | Admin form | +| POST | `/admin/ip/report` | `crate::handlers::admin::admin_ip_report` | Admin form | +| GET | `/admin/ip/{ip_hash}` | `crate::handlers::admin::admin_ip_history` | Admin read | +| POST | `/admin/post/ban-delete` | `crate::handlers::admin::admin_ban_and_delete` | Admin form | +| POST | `/admin/appeal/dismiss` | `crate::handlers::admin::dismiss_appeal` | Admin form | +| POST | `/admin/appeal/accept` | `crate::handlers::admin::accept_appeal` | Admin form | +| GET | `/admin/backup` | `crate::handlers::admin::admin_backup` | Admin backup read | +| GET | `/admin/restore` | `inline redirect` | Admin read | +| POST | `/admin/restore` | `crate::handlers::admin::admin_restore` | Restore | +| GET | `/admin/board/backup/{board}` | `crate::handlers::admin::board_backup` | Admin backup read | +| GET | `/admin/board/restore` | `inline redirect` | Admin read | +| POST | `/admin/board/restore` | `crate::handlers::admin::board_restore` | Restore | +| POST | `/admin/backup/create` | `crate::handlers::admin::create_full_backup` | Admin form | +| POST | `/admin/backup/settings` | `crate::handlers::admin::update_full_backup_settings` | Admin form | +| POST | `/admin/board/backup/create` | `crate::handlers::admin::create_board_backup` | Admin form | +| GET | `/admin/backup/download/{kind}/{filename}` | `crate::handlers::admin::download_backup` | Admin backup read | +| GET | `/admin/backup/progress` | `crate::handlers::admin::backup_progress_json` | Admin read | +| POST | `/admin/backup/delete` | `crate::handlers::admin::delete_backup` | Admin form | +| POST | `/admin/backup/restore-saved` | `crate::handlers::admin::restore_saved_full_backup` | Admin form | +| POST | `/admin/backup/extract-board` | `crate::handlers::admin::extract_board_from_full_backup` | Admin form | +| POST | `/admin/board/backup/restore-saved` | `crate::handlers::admin::restore_saved_board_backup` | Admin form | +| GET | `/static/style.css` | `serve_css` | Public read | +| GET | `/static/main.js` | `serve_main_js` | Public read | +| GET | `/static/admin.css` | `serve_admin_css` | Public read | +| GET | `/static/admin.js` | `serve_admin_js` | Public read | +| GET | `/static/theme-init.js` | `serve_theme_init_js` | Public read | + +## Additional listener and fallbacks + +The HTTP-to-HTTPS listener accepts any method on `/{*path}` and builds a permanent redirect using configured public hosts/ACME domains or a loopback fallback, never an arbitrary untrusted Host. It also runs request-boundary validation. It performs no database or filesystem work. Static assets are embedded and expose no directory browsing. ACME protocol handling is managed separately by the configured TLS provider. Health/readiness/metrics expose runtime readiness/aggregate telemetry; metrics authorization policy is recorded in the final report. No third-party endpoint is within this audit scope. diff --git a/docs/security-audit-2026-10-06.md b/docs/security-audit-2026-10-06.md new file mode 100644 index 00000000..33dec402 --- /dev/null +++ b/docs/security-audit-2026-10-06.md @@ -0,0 +1,303 @@ +# RustChan hostile-input security audit — 2026-10-06 + +Defensive source review, implementation fixes, deterministic regressions, and local HTTP/browser/load validation. Testing used an isolated data directory and a listener bound to `127.0.0.1`; no production or third-party system was targeted. No commits or branches were created. No new crate was added; `hooks` and `blob` features were enabled on the existing rusqlite dependency. + +## 1. Attack surface mapped + +[The pre-change inventory](security-attack-surface.md) maps 122 method/path entries, their shared headers, permissions, CSRF, input/body limits, database/filesystem work, output contexts, and expensive-operation profiles. It includes posting, replies, reports, appeals, search, archive/catalog, previews/polling, ownership actions, admin authentication/settings/assets/accounts/moderation, setup, backups/restores, maintenance, update/restart, media, telemetry, embedded assets, and the HTTPS redirect listener. No alternate JSON posting, RSS, legacy posting, or separate anonymous upload route was found. + +The final review followed shared helpers back through every caller, including both public upload routes, all five admin asset routes, full/saved/board restore paths, post previews/search/archive/admin rendering, password reauthentication, and plaintext/static TLS/ACME listener implementations. + +## 2. Confirmed vulnerabilities and weaknesses + +No CRITICAL or HIGH finding was confirmed. Severity reflects the demonstrated boundary and existing defenses, not a hypothetical worst-case chain. + +| ID / severity | Boundary, consequence and cause | Correction | Regression evidence | +|---|---|---|---| +| F1 MEDIUM | Behind a trusted appending proxy, attacker-supplied leftmost X-Forwarded-For or preferred X-Real-IP could control identity. This undermined rate limits, bans and attribution. | Walk XFF from the trusted peer toward the first untrusted hop; authoritative malformed/duplicate XFF falls back to the peer rather than another forwarding header. | Spoofed-prefix, malformed-hop, IPv6 and direct-client tests; actual direct requests with rotating XFF still hit the same posting budget. | +| F2 MEDIUM | Markup transformations ran after hyperlink generation and could put generated tags/quotes inside URL attributes. Cached post HTML could retain the malformed markup. CSP limits script execution but does not make malformed HTML safe. | Apply formatting before generating URL attributes; escape affected cached content as bounded raw text while preserving valid cached formatting/dice rolls. | URL/markup and legacy-cache regressions; real Chromium found zero injected handlers, malicious image nodes or broken URL attributes. | +| F3 MEDIUM | Browsing/authentication/board-password counters and CAPTCHA challenges admitted arbitrary identities without a hard retained-state ceiling. Expiration alone did not bound burst memory. | Fixed-cap atomic rate tables; fixed-size hashed production keys, amortized expiry and fail-closed saturation. CAPTCHA capacity includes generation reservations. | Exact window boundaries, 16,384-entry saturation/recovery, 64 simultaneous reservations, CAPTCHA cap and reservation cleanup. | +| F4 MEDIUM | Concurrent failures could all pass password failure checks before expensive verification; CAPTCHA work could block an async worker. Expensive operations lacked independent admission bounds across entry points. | Reserve attempts before verification; nonqueueing two-worker password/CAPTCHA gates; move work to blocking workers and retain permits inside them. Include setup, unlock, account/secret/update reauthentication and board password creation. | Existing auth/setup/unlock tests plus gate sharing/release and cancelled-request blocking-work regression. | +| F5 MEDIUM | Small search result pages did not bound FTS count, sorting, deep OFFSET or parallel work. Broad prefix queries could process the entire matching corpus. | Query/term/page/result/count bounds, SQLite VM/time progress budgets, two concurrent searches and independent request budget. | Real search burst, query/pagination regressions, interruption/reusable-connection test and existing FTS plan/result tests. | +| F6 MEDIUM | Thread rendering collected every reply; catalog/hidden listings requested unlimited rows. A stored large thread amplified repeated read memory, rendering and response cost. | OP plus 200 replies per indexed cursor page; 1,000-row catalog/hidden/preference cap matching the allowed active-thread maximum. | 450-reply traversal and EXPLAIN assertion; full-router history/quote/maximum-ID tests; real no-JS traversal of 464 replies. | +| F7 MEDIUM | Admin favicon/banner handlers read multipart fields/files before checking the session and did not share public media admission. Unauthenticated parallel bodies could impose allocation/parser cost. | Authenticate before body consumption, bound part names/counts/text/unknown fields, reject duplicates, apply envelope guard and share media admission through processing. | Full-router stalled infinite-body rejection before reads; text limit−1/limit/limit+1, duplicate-part and permit-release tests. | +| F8 MEDIUM | Full restore materialized all post bodies from an up-to-8-GiB snapshot. Merely slicing SQLite TEXT also misses content after embedded NUL bytes. | Page 128 IDs/types at a time; incremental SQLite value reads check byte length before allocation, validate UTF-8 and enforce 4,096 Rust scalars before staged publication; board manifest bodies use the same scalar limit before filesystem work. | 300-row sanitization; 4,095/4,096/4,097 emoji/NUL boundaries, empty body, 65,536-byte NUL-prefixed body and invalid UTF-8. Board restore rejects oversize before extraction/staging/database mutation. Existing rollback/recovery tests remain. | +| F9 MEDIUM | Application middleware only runs after an HTTP head is parsed; native listeners lacked bounded open-socket admission and header-read deadlines. | 256 sockets per listener, transport admission before handshake, 15-second handshake/head limits; HTTP/2 stream/header caps. Request admission remains held through response streaming. | Acceptor saturation/release and streamed-response regressions; actual stalled head closed after 15.003 seconds while health reads succeeded. | +| F10 LOW | Internal return-path validation allowed controls/backslashes elsewhere in a leading-slash path, enabling browser normalization ambiguity or invalid redirect headers. | Reject controls, any backslash, protocol-relative paths and values over 8 KiB in the shared validator. | Internal/external/protocol-relative/control/encoded-query regressions; existing handler redirect tests. | +| F11 LOW | Anonymous upload errors and restore XHR 500 responses could include tool diagnostics or filesystem/internal details. Request tracing retained complete query strings. | Generic public internal/decoder errors with classified 413/415 behavior preserved; bounded path-only tracing and bounded restore header diagnostics. | Error-classification/privacy tests and actual corrupt-upload rejection. | +| F12 LOW | Missing login usernames skipped password verification, providing a cheap account-existence timing signal despite generic errors. | Verify a public dummy hash at current account parameters within the same rate/concurrency gate; a missing user can never yield an account ID. | Dummy matching/nonmatching password regression and parameter parity check; existing successful/failed login tests. | + +Additional HARDENING: 64-KiB default forms, early declared-body rejection, 8-KiB URI limit, Host ambiguity rejection, early mutation origin/fetch checks, public signed-token cookie binding, and bounded startup browsing settings. Local testing also caught reserved form routes inheriting upload body/timeout allowances; the shared classifier and full-router regression now exclude those routes. The second pass corrected maximum SQLite post-ID focus so the newest reply remains reachable without cursor overflow. + +## 3. Request-size protections + +| Input | Final bound / enforcement | +|---|---| +| URI | 8,192 bytes before query parsing/logging; boundary tests at 8,191/8,192/8,193. | +| Headers | Existing 64-KiB aggregate/32-KiB value service bounds, 64-KiB native protocol bounds; duplicate Host and ambiguous transfer framing rejected. | +| Ordinary forms | 65,536 streaming bytes; early Content-Length rejection; 65,535/65,536/65,537 full-router tests. | +| Public posting | Existing 512-MiB aggregate payload / 516-MiB request envelope, board-specific media limits, 64 fields, 64-KiB text/unknown fields, bounded multipart headers/preamble/boundary. | +| Admin assets | Existing favicon 5-MiB/banner 8-MiB body caps, now early header enforcement plus shared parser/admission bounds. | +| Restore | 8-GiB actual upload/expanded archive budgets and existing staged validation; route extractor allowance cannot bypass actual streaming limits. | +| Text | Post body 4,096 Unicode scalars, subject 128, search 256, report 256, appeal 512; usernames/passwords and config fields retain backend validators. Login password additionally bounded at 1,024 bytes before verification. | + +Limits are enforced in Rust, including no-JS forms. Body declarations never choose upload policy; route/method classification does. Existing media-specific and multipart limits remain authoritative inside the larger envelope. + +## 4. Multipart hardening + +The existing streaming public parser already bounds aggregate bytes, field count, duplicate upload/scalar slots, unknown-field discard, header/preamble bytes, rejection recovery, and a shared body deadline. It uses RAII temporary files and retains a bounded recoverable draft. These controls were preserved. + +All admin favicon/banner paths now authenticate before iteration, reject duplicate names including unknown names, bound names to 128 bytes and total parts to 64, read scalar UTF-8 incrementally within 64 KiB, discard unknown fields within existing bounds, and use the existing envelope scanner. A permit covers parsing through blocking image publication. Session authorization is checked again before mutation. + +Actual HTTP cases covered missing boundary, truncation, duplicate body, a 65,537-byte unknown field and invalid raw UTF-8. Existing automated cases cover split envelopes, malformed/oversized preambles/headers, quoted boundaries, duplicate file slots, zero-byte/unselected controls, board-specific exact limits and rejected-file cleanup. + +## 5. Rate-limiting architecture + +Independent fixed-window per-client budgets are charged before parsing or DB work, across all boards/threads: thread creation 6/minute; replies 30; report/appeal requests 30; searches 20; password entry points 20; CAPTCHA 20; other mutations 120. Failed requests also consume admission. GET and HEAD expensive reads both count. + +Each table retains at most 16,384 entries, updates counters atomically, scans expiry at most once per minute, hashes identity plus a fixed action label, and rejects new identities when full. Authentication/unlock counters retain their existing configured failure windows. CAPTCHA holds at most 4,096 challenges/reservations with its existing five-minute TTL. + +The configurable browsing policy, exemptions and posting cooldowns remain. Settings now validate 1–1,000,000 browsing requests and 1–86,400-second windows. Action admission uses the resolved transport/proxy/Tor identity rather than mutable visitor cookies. Existing cookie-bound posting/report identity semantics are preserved behind this independent admission boundary. + +## 6. Posting and flood resistance + +Both thread creation and reply routes share the production parser, transactional posting validation, media admission and independent action budgets. Sage and cross-board replies cannot escape the reply class. Board access, bans, CAPTCHA policy, cooldowns, bump limits, submission-token retry semantics and staged filesystem publication remain. + +Identical legitimate submissions remain separate posts. Local tests accepted six identical new threads, then returned four 429 responses; accepted thirty replies, including sage, then returned five 429 responses. Rejected attempts did not create posts or corrupt reply counts. Existing concurrent posting/rollback tests cover publication and SQLite contention. + +## 7. Report abuse + +The new 30/minute report/appeal request budget covers invalid and duplicate targets before body/DB work. Existing reporting retains 20 new reports/hour per reporting identity, open-report deduplication, bounded reason text/control rejection, bans and board access, authoritative board/post/thread association, and an IMMEDIATE transaction around budget and insertion. Appeals retain their existing daily window. Separate legitimate reports remain possible. + +Full-router tests exercised 30 invalid reports followed by 429. The mixed HTTP workload returned controlled 400/429 for invalid reports while ordinary reads succeeded. Existing duplicate/target/appeal tests passed. + +## 8. Search abuse + +Public search rejects queries above 256 scalars and pages above 500. The DB parser retains at most twelve alphanumeric terms, generates quoted AND/prefix syntax, clamps result requests to 1–100 and offsets to 0–9,999, and caps counts at 10,000. UI results at the cap show `10,000+` and request refinement. + +Count and result operations each have a scoped progress callback: checks every 10,000 SQLite VM instructions, interruption at 200 checks or 500 ms. This is approximately two million VM instructions per operation, not a hard subprocess wall deadline. The callback is removed before returning a pooled connection. Broad queries produce a useful 400; saturation of the two-worker gate produces retryable 503; per-client excess produces 429. No anonymous waiter queue was added. + +Existing FTS CROSS JOIN/index plans, board scoping, ordering, prefixes and ordinary results remain covered. Native FTS work between VM checkpoints can overshoot the cooperative wall allowance; this limitation is explicit. + +## 9. SQL and query safety + +Request-controlled values use bound parameters. Dynamic SQL reviewed consists of source-defined column lists, table/schema maintenance, fixed route options and allowlisted modes; raw search syntax and sort identifiers are not interpolated from a request. FTS input is compiled into a bounded restricted expression and passed as a parameter. + +Malformed numeric IDs, negatives/zero, i64 maxima, overflow and nonnumeric values produce controlled 400/404. Search pages cannot overflow multiplication into a deep query. Thread history uses an indexed post-ID cursor; its EXPLAIN regression asserts `idx_posts_thread_live` and no temporary sorting B-tree. Maximum post-ID focus is tested through the full router. + +## 10. XSS and output encoding + +Reviewed post body, subject/name, filenames, board metadata, report/moderation reasons, search echo, archives, previews, admin output, recovery drafts and errors. Shared template text/attribute escaping and restricted HTTP(S) URL generation remain. Fixed F2 at the formatting order rather than relying on CSP. The cached-HTML compatibility check runs after a cheap byte cap; only affected legacy markup falls back to bounded escaped body text. Valid cached dice results are not recomputed. + +The real browser rendered literal ``, quotes, ampersands and Unicode as text with zero injected image/handler nodes. Existing template/admin/archive tests and new URL/legacy/search-echo regressions passed. Restored HTML is regenerated from bounded raw text; imported cached HTML is not trusted. + +## 11. Unicode + +Text limits deliberately count Rust Unicode scalars, while envelopes, passwords, URI and header limits use bytes. Tests cover combining marks, ZWJ, bidi controls, variation/emoji/non-BMP content, oversized multibyte search and restore bodies, long quote/markup sequences, raw invalid UTF-8 and embedded NUL restore values. No identifier normalization was introduced: board/identifier allowlists retain their existing semantics. Grapheme counting is not used or promised. + +## 12. Paths and filenames + +Public storage uses generated controlled names, not original filenames. Existing filename sanitation/truncation, canonical containment, board-relative validation, symlink defenses, archive entry validation, backup filename checks and durable delete/publication journals remain. Media authorization is applied before serving protected board files. + +A real valid PNG named `../../.png` was accepted and stored at a generated safe path with sanitized metadata. Corrupt traversal-looking, Unicode and misleading-extension filenames did not create attachments. Encoded media traversal returned 400. Existing path/archive/filename tests passed. Intentional allow-any-file boards retain their existing download policy. + +## 13. MIME and media validation + +The existing posting pipeline detects actual content, validates media type against board policy, and does not use the submitted extension/MIME as authorization. Existing image controls include 40-million-pixel admission, a 256-MiB decoder allocation policy and bounded animation frames. HEIF uses coded/output pixel bounds. Audio/container readers have bounded header/packet reads; FFmpeg/tool execution has deadlines, cancellation/process-group cleanup, output-file budgets and 64-KiB diagnostic retention. Existing media regressions passed. + +The audit did not replace that pipeline. Shared admin/public media admission now closes an inconsistent entry point. Actual tests accepted a valid PNG and returned 415 for non-image bytes labeled `image/png` under several filename forms. The PDF policy test now verifies the content/signature of either documented PNG preview or SVG fallback; dedicated fallback tests remain. + +## 14. Headers and trusted proxies + +Forwarding trust requires both explicit behind-proxy mode and a CIDR-trusted immediate socket peer. XFF is resolved right-to-left through trusted hops; X-Real-IP is only a fallback when XFF is absent. Invalid authoritative chains fail back to the peer. Trusted X-Forwarded-Proto uses the last hop. Direct clients cannot choose their effective IP via forwarding headers. Operators must configure trusted CIDRs accurately and have proxies overwrite or correctly append their forwarding metadata, especially X-Real-IP/protocol. + +Host validation rejects duplicates, credentials, whitespace, percent/separator ambiguity and nonnumeric/out-of-range ports while retaining valid IPv4/IPv6 authorities. Security-sensitive HTTPS redirect hosts remain configured. Request Host is not made a canonical site configuration. Content-Length is an early rejection hint; streaming limits remain authoritative. Actual duplicate Host and TE+CL requests returned 400. + +## 15. Redirects + +Shared internal return validation accepts bounded leading-slash paths, rejects protocol-relative/external paths, controls and all backslashes, and preserves encoded data in legitimate query strings. Existing stricter path validation inherits this boundary. Callers retain their existing route-specific fallbacks. + +Configured external banner continuation is intentional external navigation with its existing target validation/warning flow; it is not a generic internal return-to mechanism. No arbitrary request Host was introduced into security-sensitive absolute redirects. + +## 16. CSRF + +Mutation inventory covered posting, report/appeal, polls, owner edit/delete, thread preferences, board unlock, setup, login/logout, all admin/moderation/settings/assets, maintenance, restore and update/restart. Existing admin scope/session binding, token checks, same-origin rules and cookie flags remain. Public signed form tokens must now match a present CSRF cookie; cookie-free signed forms remain supported. Explicit cross-site Origin/Referer or fetch metadata is rejected before body work; supported no-origin/null-origin flows still require handler CSRF. + +GET polling/previews/read pages do not mutate content. Existing theme GET changes only a presentation cookie for the no-JS flow. Authorized backup GET can create a downloadable artifact; existing admin/CSRF controls and maintenance coordination apply. This compatibility behavior was reviewed and deliberately retained. + +## 17. Memory exhaustion + +Fixed retained-state caps, bounded forms/URI/multipart fields, socket/request gates, response windows and incremental restore reads reduce attacker-driven allocation. Thread responses contain at most 201 posts. Admin asset buffering stays within its existing 5/8-MiB envelopes and one shared media permit. CAPTCHA is finite. Searches materialize bounded pages. No allocation trusts Content-Length as capacity. + +The local mixed test observed RSS from 40,144 to 53,152 KiB, including allocator/WAL/media caching; these are snapshots, not sampled peak measurements or a production sizing guarantee. + +## 18. CPU exhaustion + +Password, CAPTCHA and search each have two active worker permits; uploads/assets share the existing one-media-request permit. No unbounded admission waiter queue was added. Blocking closures own their permits so HTTP cancellation cannot release still-running work. Argon2 strength is unchanged; missing accounts now perform comparable password work inside the same bound. Search callbacks bound cooperative SQL work. Existing media subprocess deadlines and bounded parser controls remain. + +## 19. Disk and temporary files + +Existing RAII upload/archive temporary files, staged publication, failure cleanup, pending filesystem intent recovery, converted-output caps and post/thread deletion cleanup remain. Rejected local multipart/media requests left no temporary files. In the measured mixed fixture, the file count stayed at eleven; accepted posts/media and SQLite/WAL explain permitted disk growth. + +Retention and free-space checks reduce growth but do not impose a universal storage quota or hard log-retention budget. Sustained accepted traffic and operator-created backups still require disk capacity planning. No claim of complete distributed disk-DoS prevention is made. + +## 20. SQLite contention and resources + +Public media work remains outside the short posting write transaction. Account reauthentication/new hashing now precede its IMMEDIATE transaction, with session/account/hash rechecks under the lock before mutation. Setup hashes before beginning its write transaction. Login drops the pooled connection before Argon2; update reauthentication does likewise. + +Existing busy timeout/bounded pool checkout and retryable DbBusy responses remain; there is no new spin/retry loop. Search interruption clears pooled state. Restore staging/rollback protects the live DB from partially validated data. Local integrity checks returned `ok` and authoritative reply counts matched stored replies after posting/load; existing transaction rollback, interrupted-job and crash/recovery tests passed. + +## 21. Parser and algorithmic complexity + +Finite field/body/URI bounds precede parsers. Rust regex processing remains bounded by post length; tests exercise many quotes, repeated incomplete spoilers and combining/zero-width text. Post markup generates numeric reference links without a database read or backlink insertion for every marker; client backlink work is confined to the displayed posts. Post-preview requests authorize and load their explicit target separately. Missing in-window quote targets use canonical post redirects. New routing rewrite uses only generated numeric fragment attributes and a visible-ID set. + +Restore pagination is keyset based and reads each body within a strict byte bound. Rate/CAPTCHA expiry scans are amortized rather than full-map scans per hostile request. Panic review found no new request-controlled unwrap/expect/panic; existing source-literal regex invariants and OS-randomness fail-closed termination are intentional. + +## 22. Admin and moderator security + +Admin authentication/authorization, scoped CSRF, typed actions/config validators, parameterized SQL, sensitive reauthentication and escaped stored-content rendering remain. Asset handlers now authorize before data consumption and again before publication. Account transactions recheck credentials/session after expensive work. Restore treats schema, media paths, text and cached HTML as untrusted and publishes only after preflight/staging validation. Board/saved-board/extracted-board restores enforce the same post-body scalar ceiling before staging or extraction; their JSON manifests retain the existing 64-MiB aggregate bound. Update/restart controls remain authenticated and coordinated; no external update installation was performed during this audit. + +## 23. Errors and logging + +Anonymous internal/TLS/restore-XHR 500 responses and decoder failures do not disclose internal paths/tool output. Useful 413/415 classifications and recognized safe malformed-image guidance remain. Wrapped application errors preserve their typed client status, including broad-search rejection. Detailed internal failures remain in server logs. + +Request spans record at most 256 path characters and omit queries/cookies/tokens/bodies. Restore content-type/length diagnostics are bounded. Login username redaction remains. Existing log rotation is preserved; action budgets constrain repeated costly error paths, but there is no added universal byte quota for logs. + +## 24. Security headers + +Existing CSP, nosniff, frame protection, referrer policy, eligible-origin HSTS, media policy and private admin/session caching were reviewed and preserved. No CSP allowance was weakened and no inline script dependency was added. New action-limit responses use private/no-store and Retry-After; existing retryable 503 behavior is reused. Host/proxy changes protect the inputs to secure-cookie/HSTS decisions. + +## 25. Authentication abuse + +Pre-verification reservation closes the concurrent lockout race; retained counters and active hash work are bounded. Password sizes and stored PHC resource ceilings remain enforced. Missing users use the same current Argon2 cost class, with no possible dummy-account session. Generic errors, random sessions, expiry/revocation, logout and HttpOnly/SameSite/Secure policy remain covered by existing tests. + +This removes the cheap missing-user branch; it does not claim exact constant-time end-to-end HTTP behavior across DB state or legacy password parameters. + +## 26. Automated tests added and strengthened + +New assembled-router tests cover legitimate/excessive posting, separate report/search budgets, declared/streamed form and URI boundaries, malformed Host/IDs/origin/fetch metadata, Unicode search/page limits and echo escaping, unauthenticated stalled admin uploads, bounded thread history/quote destinations/max-ID focus, response-lifetime admission, admin asset field boundaries/duplicates and permit release. + +Helper/DB tests cover retained-state saturation/window/concurrent admission, active-work ownership under cancellation, CAPTCHA reservations, SQLite search interruption/reuse, indexed complete history traversal, malicious URL markup and legacy-cache compatibility, Unicode/quote expansion, redirect normalization controls, private upload errors, typed wrapped errors, bounded restore including NUL/invalid UTF-8, dummy authentication safety/parameter parity, capped board preference collection with individual preferences preserved, and board-restore rejection before extraction/filesystem/database mutation. + +Existing multipart, media, CSRF, SQL, path, transactional rollback, cleanup, backup/restore and admin regressions were retained. No tests or lints were disabled. Independent authentication test fixtures now use distinct peer identities instead of accidentally sharing new production attempt reservations. A PDF preview policy test was corrected to validate both real preview outcomes by content; fallback-specific tests were preserved. + +## 27. Local adversarial, browser and load testing + +[Machine-readable results](security-audit-evidence-2026-10-06.json) record actual HTTP, Chromium and mixed-load evidence. + +- Direct-mode rotating forged XFF: six controlled invalid posting responses then two 429s. +- Actual boundary/framing cases: malformed Host 400, URI 414, declared oversized form 413, cross-site action 403, numeric overflow 400, encoded media traversal 400, unauthenticated asset upload 403, malformed multipart/UTF-8 400, oversized unknown field 413, TE+CL and duplicate Host 400. +- Identical thread burst: six 303 / four 429; DB integrity `ok`, thirteen posts/thirteen threads at that checkpoint. +- Eight-worker search burst: seven 200 / thirteen 503 / twenty 429 in 0.013 seconds; normal reads remained usable. 503 is nonqueueing concurrency rejection, distinct from rate rejection. +- Partial HTTP head: connection closed after 15.003 seconds; an independent health request succeeded while it was stalled. +- Reply burst: thirty 303 / five 429; median accepted-reply latency 3.328 ms. Corrupt image bytes labeled PNG returned four 415s; a valid PNG with a traversal-looking filename returned 303. +- Mixed 160 requests with eight workers: eighty normal page reads all succeeded; totals 88×200, 12×503, 30×429, 30×400. Normal-read median 8.293 ms, maximum 22.204 ms; DB integrity and reply counts remained correct, no temporary files remained. +- Chromium posted a hostile-looking benign text thread and a normal sage reply. It found zero injected handlers/malicious image nodes. With JavaScript disabled, 464 replies were reachable exactly once over pages of 201/201/65 posts including the OP; quotes and latest/older navigation worked. + +Multiple identities in the mixed tests were emitted by an explicitly configured local trusted proxy fixture; all sockets still went to 127.0.0.1. Direct untrusted-header rejection was tested separately. History bulk data was inserted into the isolated fixture solely for response/navigation verification. These small controlled runs are resilience/regression evidence, not maximum-throughput benchmarks. + +## 28. Full validation commands and results + +All commands passed on the final implementation using Rust 1.99.0, matching the project toolchain/MSRV. + +| Command | Result | +|---|---| +| `cargo fmt --all --check` | PASS | +| `cargo check --workspace --all-targets --all-features` | PASS | +| `cargo clippy --workspace --all-targets --all-features -- -D warnings -D clippy::all -D clippy::pedantic -D clippy::nursery -D clippy::cargo` | PASS | +| `cargo test --workspace --all-features` | PASS | +| `git diff --check` | PASS | + +The complete test command passed **1,439 tests with zero failures and seven existing ignored cases in each of the library and CLI suites**, plus five integration tests, one native-update fixture test and one doctest. The two main suites exercise the shared code separately; their totals are not independent unique tests. All new security regressions ran in these suites. Targeted restore/cursor checks also passed during implementation. No test/lint was disabled or broadly suppressed. + +Initial validation found an existing PDF preview fixture assuming only SVG despite the documented PNG renderer path; it now validates both outcomes by contents, while dedicated fallback tests remain. A settings-lock test failed once in the baseline and passed in subsequent complete runs without weakening the locking implementation. Earlier implementation/compiler/test-harness failures were repaired; the result table above describes the final source. + +## 29. Remaining known limitations + +- Per-client fixed windows reset on process restart. Distributed sources, Tor stream rotation and shared NAT identities affect per-client fairness/effectiveness; global work/socket/media/state bounds still apply. Saturated identity tables fail closed until expiry cleanup. +- Search deadlines are cooperative; native FTS work between checkpoints can exceed 500 ms. Broad legitimate queries can receive a refinement error. Gate overload is retryable 503; the gate uses the existing DbBusy response contract. +- Native Rust/media-library work is not a process sandbox and cannot universally be forcibly cancelled. Existing decoder/subprocess limits and active-work gates reduce, rather than eliminate, library/allocator/system-level risk. +- 256 open sockets per listener is a capacity bound, not guaranteed availability against a connection-saturating attacker. Network bandwidth, OS limits and distributed traffic remain outside application-only guarantees. Real local plaintext was exercised; static TLS/ACME compile and existing tests were validated without contacting a public ACME service. +- No universal disk quota, hard log byte retention or global lifetime report/post quota was added. Sustained admitted traffic can consume storage. Existing board/archive/media retention and backup policies remain operator controlled. +- Catalog/hidden output and active preference collection cap at 1,000 threads. An oversized legacy/restored board can have additional rows; normal allowed active-board settings fit this cap, and direct/thread/index navigation remains available. +- Scalar limits are not grapheme limits. Legacy malformed cached HTML uses escaped text and may lose its formatting until edited/restored. Oversized/invalid restored bodies now fail before live publication. +- Existing optional offline performance profiles remain explicitly ignored by the ordinary suite; crash-child tests marked ignored are invoked by their parent regressions. Snapshot latency/RSS evidence does not establish production peak resource usage. + +## 30. Deliberately unchanged + +Argon2 strength, board cooldown/CAPTCHA policy, legitimate duplicate-post semantics, report deduplication policy, public route/form/element hooks, configured browsing controls, no-JS flows, upload size/product policy, MIME decoding architecture, media streaming/ranges, staged recovery, and CSP were preserved. No User-Agent blacklist, arbitrary sleep, content censorship, new crate, commit, branch, deployment or external notification was added. The unbounded thread helper remains for offline backup/test consumers; public rendering uses the bounded helper. The final adversarial pass found no alternate unprotected posting/search/asset path. + +## Changed files + +- `Cargo.toml` +- `docs/security-attack-surface.md` +- `docs/security-audit-2026-10-06.md` +- `docs/security-audit-evidence-2026-10-06.json` +- `src/captcha.rs` +- `src/config.rs` +- `src/db/accounts.rs` +- `src/db/mod.rs` +- `src/db/posts.rs` +- `src/db/search_budget.rs` +- `src/db/user_thread_prefs.rs` +- `src/error.rs` +- `src/handlers/admin/accounts.rs` +- `src/handlers/admin/auth.rs` +- `src/handlers/admin/backup/http.rs` +- `src/handlers/admin/backup/restore_board.rs` +- `src/handlers/admin/backup/restore_full.rs` +- `src/handlers/admin/mod.rs` +- `src/handlers/admin/settings.rs` +- `src/handlers/admin/settings/appearance.rs` +- `src/handlers/admin/settings/banners.rs` +- `src/handlers/admin/settings/board.rs` +- `src/handlers/admin/updates.rs` +- `src/handlers/board.rs` +- `src/handlers/board/access_preferences.rs` +- `src/handlers/board/catalog.rs` +- `src/handlers/board/media.rs` +- `src/handlers/captcha.rs` +- `src/handlers/mod.rs` +- `src/handlers/render.rs` +- `src/handlers/setup.rs` +- `src/handlers/thread.rs` +- `src/middleware/csrf.rs` +- `src/middleware/ip.rs` +- `src/middleware/mod.rs` +- `src/middleware/rate_limit.rs` +- `src/middleware/rate_table.rs` +- `src/middleware/state.rs` +- `src/middleware/work_gate.rs` +- `src/server/server.rs` +- `src/server/server/connections.rs` +- `src/server/server/headers.rs` +- `src/server/server/lifecycle.rs` +- `src/server/server/router.rs` +- `src/server/server/router/security_tests.rs` +- `src/server/server/routes.rs` +- `src/templates/board.rs` +- `src/templates/thread.rs` +- `src/test_fixtures.rs` +- `src/utils/redirect.rs` +- `src/utils/sanitize.rs` +- `static/main.js` + + +## Requirement/evidence cross-check + +| Requested phases | Evidence retained | +|---|---| +| 1: route inventory before edits | Pre-change 122-route attack-surface document and shared cost/control profiles. | +| 2–3: sizes and malformed multipart | Native/router boundaries, public/admin parser controls, exact-boundary and stalled-body regressions; actual HTTP rejection results. | +| 4–6: limiter, floods, reports | Atomic capped tables; separate action budgets; identical/sage posting and report simulations; persistent-state checks. | +| 7–8: search and SQL | Restricted bound FTS expression, count/page/progress limits, query-plan/result regressions, malformed ID and SQL-looking search data. | +| 9–10: output contexts and Unicode | Shared rendering review, markup ordering/cache fix, Chromium DOM assertions, text/byte boundaries, NUL and invalid UTF-8 regressions. | +| 11–14: path/filename/MIME/media bombs | Generated storage identifiers, canonical/archive containment, actual traversal-looking uploads, existing decoder/container/subprocess structural/deadline/cleanup tests. | +| 15–18: headers/Host/redirect/CSRF | Direct/proxy chain fixtures, wire framing tests, Host/return-path regressions, mutation inventory and scoped-token/cookie/origin tests. | +| 19–21: expensive parsing/quotes/bots | Bounded markup/FTS/media parsing, no per-marker DB work, independent expensive-action admission and optional CAPTCHA preserved. | +| 22–26: memory/CPU/disk/DB/responses | Finite state/work/socket/body/output bounds, restore incremental reads and board preflight, temporary cleanup, transaction/rollback/DbBusy tests, no-JS complete history traversal. | +| 27–30: logs/panics/leaks/admin | Bounded path-only logs, private error regressions, request-reachable panic review, pre-body authorization, transactional rechecks and safe admin rendering. | +| 31–35: state/defaults/arithmetic/headers/auth | Capped maps/CAPTCHA, startup bounds, cursor/pagination/geometry checks, preserved CSP/cookie/cache policies, password admission and dummy-account verification tests. | +| 36–38: adversarial regression/load matrix | Deterministic helper/full-router regressions plus local wire, Chromium and 160-request mixed workload evidence. | +| 39–41: substantive controls/performance/focused cleanup | Backend controls, retained duplicate/no-JS semantics, measured local read/reply latency, no new crate/lint waiver or unrelated architectural rewrite. | +| Final adversarial pass | Shared/alternate entry-point review; reserved form-route, max-ID, incremental NUL read, board preference and board-restore corrections with regressions. | diff --git a/docs/security-audit-evidence-2026-10-06.json b/docs/security-audit-evidence-2026-10-06.json new file mode 100644 index 00000000..67917162 --- /dev/null +++ b/docs/security-audit-evidence-2026-10-06.json @@ -0,0 +1,260 @@ +{ + "scope": { + "date": "2026-10-06", + "network": "Only 127.0.0.1 local test listener", + "data": "Isolated temporary fixture; no production data", + "identity_note": "Trusted proxy fixture identities used for independent mixed-load budgets; direct-mode XFF spoof test separate" + }, + "direct_forwarded_spoof": { + "case": "direct clients cannot rotate XFF to bypass posting limits", + "statuses": [ + 400, + 400, + 400, + 400, + 400, + 400, + 429, + 429 + ] + }, + "http": [ + { + "case": "normal board", + "status": 200 + }, + { + "case": "bad Host", + "status": 400 + }, + { + "case": "huge URI", + "status": 414 + }, + { + "case": "oversized declared form", + "status": 413 + }, + { + "case": "cross-site mutation", + "status": 403 + }, + { + "case": "numeric overflow", + "status": 400 + }, + { + "case": "SQL syntax as search data", + "status": 200 + }, + { + "case": "encoded media traversal", + "status": 400 + }, + { + "case": "unauthenticated admin upload", + "status": 403 + }, + { + "case": "cross-site fetch metadata", + "status": 403 + }, + { + "case": "missing boundary", + "status": 400 + }, + { + "case": "truncated multipart", + "status": 400 + }, + { + "case": "duplicate body", + "status": 400 + }, + { + "case": "unknown oversized field", + "status": 413 + }, + { + "case": "invalid UTF8", + "status": 400 + }, + { + "case": "identical-post burst", + "statuses": { + "303": 6, + "429": 4 + } + }, + { + "case": "concurrent search burst", + "statuses": { + "200": 7, + "503": 13, + "429": 20 + }, + "elapsed_seconds": 0.013 + }, + { + "case": "TE+CL", + "status": 400 + }, + { + "case": "duplicate Host", + "status": 400 + }, + { + "case": "stalled request head", + "elapsed_seconds": 15.003, + "closed": true + }, + { + "case": "database consistency", + "integrity": "ok", + "posts": 13, + "threads": 13 + } + ], + "mixed_load": { + "reply_burst": { + "303": 30, + "429": 5 + }, + "reply_latency_median_ms": 3.328, + "valid_png_traversal_filename_status": 303, + "corrupt_image_statuses": [ + 415, + 415, + 415, + 415 + ], + "mixed_requests": 160, + "mixed_statuses": { + "200": 88, + "503": 12, + "429": 30, + "400": 30 + }, + "mixed_elapsed_seconds": 0.13, + "normal_read_median_ms": 8.293, + "normal_read_max_ms": 22.204, + "database_integrity": "ok", + "authoritative_reply_count": 63, + "resources_before": { + "rss_kib": 40144, + "file_count": 11, + "disk_bytes": 4590154, + "temporary_files": [] + }, + "resources_after": { + "rss_kib": 53152, + "file_count": 11, + "disk_bytes": 4593370, + "temporary_files": [] + }, + "resource_note": "Snapshot RSS, not a sampled peak; SQLite/WAL and accepted post/media growth included." + }, + "browser_sage_reply": { + "url": "/audit/thread/1", + "posts": 2, + "quoteHref": "#p1", + "bodyText": "Browser audit\nAnonymous\nID: 2JQIV2O9\n10/06/26(Tue)09:35:22\nNo.1\n>>14\nBenign \" & é 😀 https://example.invalid/safe\nreport\nAnonymous\nID: 2JQIV2O9\n10/06/26(Tue)09:45:05\nNo.14\n(You)\n>>1(OP) Normal reply é 😀 [b]bold[/b]\nreport", + "injectedHandlers": 0 + }, + "browser_no_javascript_history": { + "initial": { + "posts": 201, + "quoteHref": "/audit/post/77", + "injectedHandlers": 0 + }, + "javaScriptDisabled": true, + "historyPages": 3, + "historyPagePostCounts": [ + 201, + 201, + 65 + ], + "uniqueReplies": 464, + "quoteFocus": true, + "latestNavigation": true + }, + "validation": { + "status": "passed", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "commands": [ + { + "command": "cargo fmt --all --check", + "exit_code": 0, + "elapsed_seconds": 2.081, + "log": "/tmp/rustchan-release-validation-0.log" + }, + { + "command": "cargo check --workspace --all-targets --all-features", + "exit_code": 0, + "elapsed_seconds": 7.379, + "log": "/tmp/rustchan-release-validation-1.log" + }, + { + "command": "cargo clippy --workspace --all-targets --all-features -- -D warnings -D clippy::all -D clippy::pedantic -D clippy::nursery -D clippy::cargo", + "exit_code": 0, + "elapsed_seconds": 12.071, + "log": "/tmp/rustchan-release-validation-2.log" + }, + { + "command": "cargo test --workspace --all-features", + "exit_code": 0, + "elapsed_seconds": 118.838, + "log": "/tmp/rustchan-release-validation-3.log" + }, + { + "command": "git diff --check", + "exit_code": 0, + "elapsed_seconds": 0.024, + "log": "/tmp/rustchan-release-validation-4.log" + } + ], + "suites": [ + { + "suite": "library", + "passed": 1439, + "failed": 0, + "ignored": 7 + }, + { + "suite": "CLI binary", + "passed": 1439, + "failed": 0, + "ignored": 7 + }, + { + "suite": "integration", + "passed": 5, + "failed": 0, + "ignored": 0 + }, + { + "suite": "native-update fixture", + "passed": 1, + "failed": 0, + "ignored": 0 + }, + { + "suite": "doctest", + "passed": 1, + "failed": 0, + "ignored": 0 + } + ], + "additional_targeted_checks": [ + "maximum SQLite post-ID focus", + "incremental restore Unicode/NUL/invalid UTF-8/empty-body bounds", + "board restore rejection before filesystem/database mutation" + ], + "local_fixture_cleanup": "Named browser and test server closed; isolated data directory removed." + }, + "notes": [ + "Latencies are local fixture observations, not capacity benchmarks.", + "RSS measurements are snapshots, not sampled peaks.", + "HTTP/browser tests ran the built audit binary; later restore/authentication/metadata regressions are verified in the final full Rust suite." + ] +} diff --git a/docs/strict-lint-exceptions.tsv b/docs/strict-lint-exceptions.tsv index aa34f802..58c2e3e8 100644 --- a/docs/strict-lint-exceptions.tsv +++ b/docs/strict-lint-exceptions.tsv @@ -341,7 +341,6 @@ src/workers/mod.rs 1380 expect clippy::cognitive_complexity source validation an src/workers/mod.rs 1551 expect clippy::too_many_arguments the finalisation boundary carries the durable job and all validated filesystem identities fn transcode_video_finalise src/workers/mod.rs 1593 expect clippy::too_many_arguments output installation and atomic DB completion require the full validated transcode identity fn persist_transcoded_webm src/workers/mod.rs 1934 expect clippy::too_many_arguments the finalisation boundary explicitly carries all validated paths and DB identity fn waveform_finalise -src/workers/mod.rs 2115 expect clippy::cognitive_complexity archive, prune, and filesystem finalization remain one consistency operation fn prune_threads src/workers/mod.rs 2561 expect clippy::cognitive_complexity,clippy::too_many_lines the bounded legacy-state matrix is intentionally explicit fn reconcile_media_job_states src/workers/mod.rs 3050 expect clippy::expect_used test database queries stop locally with fixture-specific failure context fn file_hash_count src/workers/mod.rs 3227 expect clippy::expect_used fixture setup failures should stop this queue durability test fn required_thread_prune_bypasses_capacity_and_survives_queue_recreation diff --git a/docs/thread-archive-audit.md b/docs/thread-archive-audit.md new file mode 100644 index 00000000..c5dc5de9 --- /dev/null +++ b/docs/thread-archive-audit.md @@ -0,0 +1,255 @@ +# Thread/archive production-readiness audit + +## Architecture before changes + +Threads remain in `threads` with an `archived` boolean; posts, polls, submission +tokens and media references stay in place. There is no separate archive table, +archive timestamp, time-based expiry, or public unarchive action. The internal +unarchive helper preserves the lock until explicitly unlocked. + +Public thread creation commits the thread, OP, optional poll, file publication +intent and coalesced durable `thread_prune` job in one immediate transaction. +Replies serialize thread state, reply count and bump-limit checks under SQLite's +write lock. Sage replies increment the count without bumping. Reaching the bump +limit stops future bumps; it does not itself archive a thread. Deleting replies +reduces the reply count but preserves bump time. Deleting the OP deletes its thread. + +`workers::prune_threads` loads current board settings and archives overflow +non-sticky active threads by descending bump time. Sticky active threads do not +consume `max_threads`; locked threads do. Archiving atomically sets archived and +locked. `allow_archive || CONFIG.archive_before_prune` decides whether active +overflow is archived or deleted. The global safety net defaults to true. +Startup and periodic bounded reconciliation schedule any outstanding overflow; +creation and moderation can temporarily exceed limits pending maintenance. + +Archive retention is a count cap, `max_archived_threads`, not a duration. All +archived threads, including manually archived sticky threads, participate; sticky +is not an archive preservation flag. Retention keeps most recently bumped +content, not most recently manually archived content. Deletion cascades database +references and commits a durable file-delete intent after checking all remaining +references. Filesystem replay checks references again and resumes after restart. +Archiving itself neither moves nor deletes media. Active-media pruning excludes +archives, and the normal media reconciliation pipeline remains authoritative. + +`/{board}/archive` lists 20 threads per numbered page. It paginates threads before +aggregating post/media counts. `/{board}/thread/{id}` remains the stable URL across +archival and shares the active renderer, quoting, escaping, thumbnails and CSP. +Thread loads check the actual board. Board-password gates apply to archive, +thread, search and media routes. Catalog, index and unread history exclude +archives; board FTS and admin searches include archived posts. Replies and owner +edit/delete controls reject archived content under the write lock. + +Administrators can archive and delete threads using session, origin and CSRF +checks. No moderator-scoped role or preservation/unarchive UI exists. Board +limits are exposed through established board settings; global archive policy +uses TOML/environment configuration. Workers validate limits before deletion. + +## Findings and implementation evidence + +| Problem and root cause | Repair | Regression coverage | +| --- | --- | --- | +| Equal bump timestamps had no explicit retention tie-breaker, allowing planner choice to disagree with visible ordering. | Active retention uses `bumped_at DESC, id DESC`; archive retention uses the existing archive order, `bumped_at DESC, id ASC`. | `archive_threshold_ties_sticky_locked_and_idempotency`, `live_deletion_ties_keep_the_visible_newest_threads`, `archive_pages_and_retention_share_tie_order_and_search_survives` | +| Maintenance read policy before acquiring its write lock, then archived and pruned in separate transactions. A failed prune could leave a partially completed maintenance pass. | Read current policy, archive/live-prune, archive-cap pruning and file intents under one immediate transaction; finalize files after commit. | `archive_and_prune_worker_rolls_back_as_one_operation`, `combined_retention_failure_rolls_back_archive_transition`, existing `delayed_prune_uses_only_current_limits` and `delayed_prune_uses_current_archived_limit` | +| Overflow recovery built an unbounded SQLite parameter list. Large restored/backlogged boards could fail maintenance. | Archive using a parameterized selection subquery; batch file collection and deletion into at most 500 IDs per statement. | `archive_recovery_handles_more_than_sqlite_parameter_limit` uses 33,000 threads. | +| Hand-written archive/prune transactions relied on individual rollback/commit paths. | RAII transactions roll back on body or commit failure and release the write lock. | `failed_archive_commit_releases_the_lock_and_rolls_back`, `archive_busy_failure_and_restart_are_recoverable`, `process_interruption_during_archive_recovers_without_content_loss` | +| Invalid administrator retention values were clamped to one or defaulted to 150, potentially deleting content; omitted fields reset existing caps. Direct DB callers also lacked destructive-limit validation. | Reject invalid explicit values with HTTP 400; preserve omitted caps; validate again at the archive/prune DB boundary. | `retention_settings_reject_invalid_values_and_preserve_omitted_limits`, `invalid_retention_http_settings_cannot_reduce_saved_caps`, `destructive_limits_fail_closed_at_database_boundary`; browser settings rejection assertions | +| Existing/manual/safety-net archives became inaccessible through the archive index when `allow_archive` was false. Navigation used the same incorrect condition. | Archive access and navigation are independent of automatic overflow policy. | `archive_index_remains_accessible_when_overflow_archiving_is_disabled`; browser automatic/manual archival scenarios | +| Archived polls still offered voting and accepted mutations, despite the thread being read-only. | Render poll results and `[archived]`; enforce `threads.archived=0` inside the vote INSERT itself. | `archived_poll_is_read_only_in_thread_and_vote_route`, extended vote membership/expiry regression, browser poll rejection assertions | +| Manual moderation could succeed without an audit record, and archive logs used the supplied board name. Manual archive/unsticky changes lacked an immediate durable retention request. | Session authorization, state mutation, canonical archive log and retention intent commit together. | `manual_archive_rolls_back_when_audit_logging_fails`; browser anonymous/CSRF rejection and forged-board audit/redirect checks | +| Archive counts and rows were separate snapshots; access preflight could become stale during a password change. | Revalidate board access and read board/count/page under one SQLite read snapshot. | `archived_pages_enforce_board_view_passwords` checks protected archive and stable thread URLs, forged cookies and post-password viewability; pagination regressions | +| Archived replies were reported as merely locked because that check ran first. | Prioritize archive status during preflight and the authoritative reply write check. | Extended `reply_rechecks_thread_state_after_independent_preparation`; browser checks the precise archived error. | + +These changes preserve background enforcement: the active cap is authoritative +when maintenance runs, not a synchronous cap on thread creation. Sticky active +threads remain exempt, locked active threads remain eligible, and archived sticky +threads remain subject to archive retention. No time-based expiry, preservation +flag, separate archive table, or reopening interface was introduced. + +## Database, concurrency and recovery + +The only schema change replaces `idx_threads_board_sticky_bumped` with: + +```sql +CREATE INDEX idx_threads_active_order +ON threads(board_id, sticky DESC, bumped_at DESC, id DESC) +WHERE archived = 0; +``` + +The existing strict schema-repair path recognizes the exact legacy index, +installs the replacement and removes the redundant old index in its migration +transaction. The existing upgrade regression reconstructs that legacy schema and +verifies data survives repair. No columns, foreign keys or dependency versions +changed. The existing archive index continues to support archive pagination and +retention order. + +File-backed WAL tests race creators, replies and maintenance, and race +stickying/locking/deleting against archival. SQLite write transactions determine +the outcome; no application-wide mutex was added. Two overlapping worker passes +converge to the same caps. Tests also exercise lock contention, failed DELETE +triggers, deferred-foreign-key commit failure, reopening the database, and a +subprocess killed after its archive UPDATE but before commit. The latter verifies +the original active state returns intact and maintenance can safely retry. + +New concurrency tests are +`overlapping_creators_and_archive_passes_preserve_every_thread`, +`concurrent_oldest_reply_and_archive_obey_the_committed_bump`, and +`concurrent_moderation_and_archive_are_serialized_by_sqlite`. +`sage_bump_limit_and_deleted_replies_preserve_archive_order` verifies sage, +bump-limit and reply-deletion behavior, including rejection of archived replies +even if an administrator clears the lock flag. + +## Media, routing, search and security + +Archive rendering uses the existing shared post renderer. The new +`archived_media_and_markup_use_the_same_safe_post_renderer` compares active and +archived reply output for image, video, audio, PDF, other-file and legacy media, +including companion audio, quotes, spoilers and hostile names/filenames. +`archived_media_survives_and_shared_paths_are_never_pruned` verifies archival +preserves references and retention never schedules deletion of a shared original, +thumbnail or companion audio path. + +The unchanged media subsystem already tests missing-thumbnail fallback, pruned +originals, processing failures, HTTP validators/ranges, legacy redirects, +reference rechecks and interrupted durable cleanup. Those regressions are +included in the full test suite. Board restore rerenders `body_html` from escaped +raw body through the shared sanitizer; it does not trust archived HTML supplied +by a board-backup manifest. No separate archive rendering or conversion pipeline +was added. + +Stable thread URLs remain valid after archival; wrong-board, negative, +nonexistent and pruned IDs fail through the existing route checks. Board archive +and archived thread pages retain view-password protection. Poll voting is now +read-only at the SQL write boundary. Administrator actions retain session, +origin and CSRF enforcement and now fail atomically when audit logging fails. +Archive requests remain parameterized, user display fields stay escaped, media +delivery retains path validation, and the existing CSP is unchanged. + +Search still includes archives and uses the existing FTS index. Index, catalog +and unread activity still exclude archives. Pruning removes searchable posts +through existing cascades/triggers. No cache of archive rows was introduced; +cookie-dependent responses retain `Vary: Cookie`. + +## Archive UI and administration + +Archive navigation is always discoverable from board/index/catalog views. Rows +show thread number, state, reply and attachment counts, creation and last-bump +times. The page states its actual retention cap; empty-state copy is clearer. +Decorative thumbnails use empty alternative text and archive rows have visible +theme-aware keyboard focus outlines. Existing routes, numbered pagination, +element hooks, progressive enhancement and no-JS browsing are preserved. + +Administrator guidance now explicitly says lowering the archive cap permanently +prunes overflow, archived sticky threads are included, and the global safety net +can override the automatic-archive checkbox. `SETUP.md` documents creation, +sage/bump limits, count retention, media cleanup, manual moderation, search and +live versus restart-required policy changes. The generated TOML comment correctly +describes in-place archival rather than a nonexistent archive table. + +## Performance evidence + +The explicit `archive_query_profile` test uses 10,000 threads and 210,000 posts, +including 9,900 archived threads. Both index candidates receive `ANALYZE` before +comparison. These local Rust 1.99/SQLite debug-build measurements are illustrative, +not production latency guarantees. + +| Operation | Evidence | +| --- | --- | +| 200 active-retention selections before | 13.903 ms; `SEARCH threads USING INDEX idx_threads_archived (board_id=? AND archived=?)`, then `USE TEMP B-TREE FOR LAST TERM OF ORDER BY` | +| 200 selections after | 6.169 ms; `SEARCH threads USING COVERING INDEX idx_threads_active_order (board_id=? AND sticky=?)`; no temporary sort, approximately 2.25× faster | +| Archive page, offset 0 | 4.328 ms for 20 loads, approximately 0.216 ms/page | +| Archive page, offset 5,000 | 52.855 ms for 20 loads, approximately 2.643 ms/page | +| Archive page, offset 9,880 | 99.045 ms for 20 loads, approximately 4.952 ms/page | +| Archived thread and posts | 0.404 ms | +| FTS search including archives | 11.893 ms | +| Archive plus retention deleting 950 threads/19,950 posts | 268.596 ms | + +Deep OFFSET cost increases, but the validated archive cap is 10,000 and this +measured cost does not justify replacing established numbered URLs with cursors. +The page-first aggregation remains in place. Separately, the 33,000-thread test +proves backlog maintenance avoids the SQLite bind-parameter scaling trap. + +Reproduce profiling with: + +```sh +RUSTCHAN_ARCHIVE_EVIDENCE=/tmp/rustchan-archive-performance.txt \ + cargo test --locked --lib db::threads::archive_tests::archive_query_profile \ + --all-features -- --ignored --exact +``` + +## Files changed + +- `src/db/threads.rs` and new `src/db/threads/archive_tests.rs`: transitions, deterministic pruning, batching, boundary validation and lifecycle tests. +- `src/db/schema.rs`: partial active-order index and legacy upgrade regression. +- `src/db/posts.rs`: atomic archived-poll vote rejection. +- `src/workers/mod.rs`: atomic current-policy retention and worker rollback regression. +- `src/handlers/admin/content.rs`: atomic audited moderation and retention scheduling. +- `src/handlers/admin/settings/board.rs`: safe retention parsing and handler regressions. +- `src/handlers/board/catalog.rs` and `src/handlers/board/tests.rs`: archive accessibility, consistent authorized snapshot and routing regressions. +- `src/handlers/posting.rs` and `src/handlers/thread.rs`: precise archived reply/vote errors. +- `src/templates/board.rs`, `src/templates/thread.rs`, `src/templates/admin.rs`, `static/style.css`: archive metadata, navigation, read-only polls, administrator guidance and focus styling. +- `src/config/template.rs`, `SETUP.md`, `docs/thread-archive-audit.md`: corrected policy/operational documentation and audit evidence. +- `docs/strict-lint-exceptions.tsv`: remove the worker complexity expectation made obsolete by the focused simplification. + +The new `tests/e2e/archive-production.spec.ts` stays local, as required by the +repository's existing Playwright ignore/check policy. No browser harness, +package manifest or tooling is force-added to Git. + +## Validation results + +Validated on macOS with `rustc 1.99.0 (b940084d7 2026-09-28)`. + +| Command | Result | +| --- | --- | +| `cargo fmt --all --check` | Passed | +| `cargo check --locked --workspace --all-targets --all-features` | Passed | +| `cargo clippy --workspace --all-targets --all-features -- -D warnings -D clippy::all -D clippy::pedantic -D clippy::nursery -D clippy::cargo` | Passed; no new lint allowances | +| `cargo test --workspace --all-features` | Passed: 1,376 unit tests in each library/binary target, six integration tests and one doctest; zero failures. Seven tests are explicitly ignored in each unit target. The archive profiler was run separately; the interrupted child helper runs through its parent regression. | +| `cargo test --locked --lib db::threads::archive_tests --all-features` | Passed: 14 regressions; profiler and subprocess helper intentionally ignored | +| Explicit ignored archive profiler command above | Passed; before/after evidence recorded in this report | +| `RUSTDOCFLAGS='-D warnings' cargo doc --locked --workspace --all-features --no-deps` | Passed | +| `cargo deny --locked check` | Passed advisories, bans, licenses and sources; existing missing-license metadata for `captcha` and duplicate-version warnings remain. Dependencies were unchanged. | +| `python3 tools/check_local_tooling.py` | Passed; local browser tooling remains untracked | +| `git diff --check` | Passed | + +Browser validation used the existing local isolated-runtime Playwright harness: + +```sh +RUSTCHAN_E2E_SKIP_BUILD=1 RUSTCHAN_AUDIT_OUTPUT=output/playwright/archive-final \ + npx playwright test tests/e2e/archive-production.spec.ts --workers=2 +RUSTCHAN_E2E_SKIP_BUILD=1 RUSTCHAN_AUDIT_OUTPUT=output/playwright/archive-existing-final \ + npx playwright test tests/e2e/settings.spec.ts --grep 'bump/archive|thread sage' \ + --project=chromium --workers=2 +RUSTCHAN_E2E_SKIP_BUILD=1 RUSTCHAN_AUDIT_OUTPUT=output/playwright/archive-backlinks-final \ + npx playwright test tests/e2e/archive-production.spec.ts --grep 'automatic archival' \ + --workers=2 +``` + +Results: 14/14 new scenarios across Chromium, WebKit, Firefox, mobile Firefox, +mobile WebKit, Firefox without JS and Chromium without JS; 2/2 unchanged existing +archive scenarios; 7/7 final automatic-archive scenarios after adding explicit +backlink navigation. Coverage includes real image uploads, automatic archival, +stable URLs, quotes/backlinks, media loading, search, pagination, restart, denied +replies/votes, manual moderation authorization/CSRF/canonical audit logs and +invalid settings. All nine built-in themes were checked at 1,280 and 390 pixels, +with overflow/focus assertions and visual screenshot review. No existing test or +diagnostic assertion was weakened. + +One earlier parallel full-suite run hit a transient lock-acquisition failure in +the unchanged `config::admin::tests::atomic_save_validates_before_mutation_and_does_not_change_active_config` +test. The complete default-parallelism rerun passed. This was recorded rather +than changing unrelated configuration behavior or suppressing that test. + +## Remaining limitations + +- Creation can temporarily exceed the active cap while its durable maintenance + job is pending. Failure/retry/reconciliation remain operational concerns. +- Archives are capped by count and bump order. There is no permanent/protected + archive mode, archive timestamp, time retention or public reopening control. +- File removal follows the committed database transition through durable cleanup + and reference checks. It cannot be one atomic transaction across SQLite and + the filesystem; failures remain queued and logged for replay. +- The interrupted-process regression tests SQLite recovery from process death, + not physical power loss or every storage failure. Invalid retention policy + fails closed; this audit does not invent an automatic corrupted-database repair. +- Timing evidence is from this machine and debug fixtures. Deployments should + monitor maintenance backlog and SQLite contention under their own workloads. diff --git a/docs/thumbnail-evidence/browser-baseline.json b/docs/thumbnail-evidence/browser-baseline.json new file mode 100644 index 00000000..93baa40e --- /dev/null +++ b/docs/thumbnail-evidence/browser-baseline.json @@ -0,0 +1,433 @@ +{ + "browser": "153.0.8010.12", + "thumbnail_bytes": 33016, + "unique_thumbnail_requests_after_full_scroll": 65, + "samples": [ + { + "posts_with_images": 0, + "sample": 0, + "state": "cold-browser-cache", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13.5, + "navigation_dom_content_loaded_ms": 12.800000011920929, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 1, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 10.700000047683716, + "navigation_dom_content_loaded_ms": 9.600000023841858, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 2, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13.100000023841858, + "navigation_dom_content_loaded_ms": 12.100000023841858, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 3, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13.099999964237213, + "navigation_dom_content_loaded_ms": 12, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 4, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13.199999988079071, + "navigation_dom_content_loaded_ms": 12.100000023841858, + "resource_duration_ms": [] + }, + { + "posts_with_images": 4, + "sample": 0, + "state": "cold-browser-cache", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 4, + "cached_media_responses": 0, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 133264, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 135536, + "navigation_load_event_end_ms": 20.899999976158142, + "navigation_dom_content_loaded_ms": 19.600000023841858, + "resource_duration_ms": [ + 4.899999976158142, + 2.099999964237213, + 2.299999952316284, + 2.100000023841858 + ] + }, + { + "posts_with_images": 4, + "sample": 1, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 14.900000035762787, + "navigation_dom_content_loaded_ms": 13.700000047683716, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 4, + "sample": 2, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 14.899999976158142, + "navigation_dom_content_loaded_ms": 13.600000023841858, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 4, + "sample": 3, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 15.800000011920929, + "navigation_dom_content_loaded_ms": 14.5, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 4, + "sample": 4, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 18.80000001192093, + "navigation_dom_content_loaded_ms": 17.100000023841858, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 0, + "state": "cold-browser-cache", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 25, + "cached_media_responses": 0, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 832900, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 847100, + "navigation_load_event_end_ms": 52, + "navigation_dom_content_loaded_ms": 45.09999996423721, + "resource_duration_ms": [ + 5, + 3.100000023841858, + 5.199999988079071, + 3.599999964237213, + 4.899999976158142, + 5, + 5.099999964237213, + 5.099999964237213, + 6.299999952316284, + 7.600000023841858, + 7.400000035762787, + 7.5, + 7.800000011920929, + 7, + 7.800000011920929, + 9.800000011920929, + 9.800000011920929, + 9.899999976158142, + 9.400000035762787, + 9.800000011920929, + 9.600000023841858, + 10.600000023841858, + 11.400000035762787, + 11.100000023841858, + 11.300000011920929 + ] + }, + { + "posts_with_images": 128, + "sample": 1, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 48.19999998807907, + "navigation_dom_content_loaded_ms": 41.19999998807907, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 2, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 48.39999997615814, + "navigation_dom_content_loaded_ms": 41.39999997615814, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 3, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 47.09999996423721, + "navigation_dom_content_loaded_ms": 40, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 4, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 51.19999998807907, + "navigation_dom_content_loaded_ms": 40.80000001192093, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "limitations": "Debug server, local loopback HTTP, synthetic 1024x768 gradient source, 250px lossless WebP preview, 128 posts share 65 unique URLs. Initial viewport measured for 500ms after load. Browser setup/CDP overhead; cold browser cache, warm server/OS cache. Resource Timing adds an estimated 300-byte header cost; CDP records transfer separately. Load-event timings are observations, not isolated proof of frontend speedup. Original loaded only after click; missing/corrupt fallback and missing-file recovery asserted. Raw output remains in local output/playwright and /tmp." +} diff --git a/docs/thumbnail-evidence/browser-final.json b/docs/thumbnail-evidence/browser-final.json new file mode 100644 index 00000000..f6bcefdc --- /dev/null +++ b/docs/thumbnail-evidence/browser-final.json @@ -0,0 +1,433 @@ +{ + "browser": "153.0.8010.12", + "thumbnail_bytes": 33016, + "unique_thumbnail_requests_after_full_scroll": 65, + "samples": [ + { + "posts_with_images": 0, + "sample": 0, + "state": "cold-browser-cache", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13.099999964237213, + "navigation_dom_content_loaded_ms": 12.299999952316284, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 1, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13.300000011920929, + "navigation_dom_content_loaded_ms": 12.400000035762787, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 2, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 14.199999988079071, + "navigation_dom_content_loaded_ms": 13.199999988079071, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 3, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 15, + "navigation_dom_content_loaded_ms": 14, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 4, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 12.799999952316284, + "navigation_dom_content_loaded_ms": 11.799999952316284, + "resource_duration_ms": [] + }, + { + "posts_with_images": 4, + "sample": 0, + "state": "cold-browser-cache", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 4, + "cached_media_responses": 0, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 133264, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 135536, + "navigation_load_event_end_ms": 20.399999976158142, + "navigation_dom_content_loaded_ms": 19.19999998807907, + "resource_duration_ms": [ + 5.300000011920929, + 2.400000035762787, + 2.900000035762787, + 3 + ] + }, + { + "posts_with_images": 4, + "sample": 1, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 10, + "navigation_dom_content_loaded_ms": 9.199999988079071, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 4, + "sample": 2, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13, + "navigation_dom_content_loaded_ms": 11.800000011920929, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 4, + "sample": 3, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 9.399999976158142, + "navigation_dom_content_loaded_ms": 8.5, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 4, + "sample": 4, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 9.800000011920929, + "navigation_dom_content_loaded_ms": 8.899999976158142, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 0, + "state": "cold-browser-cache", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 25, + "cached_media_responses": 0, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 832900, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 847100, + "navigation_load_event_end_ms": 52.80000001192093, + "navigation_dom_content_loaded_ms": 46.10000002384186, + "resource_duration_ms": [ + 4.699999988079071, + 3.300000011920929, + 4.699999988079071, + 3.700000047683716, + 5, + 5.100000023841858, + 5.100000023841858, + 6.199999988079071, + 7.399999976158142, + 7.599999964237213, + 7.699999988079071, + 7.799999952316284, + 7.799999952316284, + 7.800000011920929, + 9, + 9.600000023841858, + 10, + 9.900000035762787, + 10, + 9.199999988079071, + 11, + 11.5, + 11.300000011920929, + 11.400000035762787, + 11.5 + ] + }, + { + "posts_with_images": 128, + "sample": 1, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 44.10000002384186, + "navigation_dom_content_loaded_ms": 37.19999998807907, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 2, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 35.89999997615814, + "navigation_dom_content_loaded_ms": 30, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 3, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 36.30000001192093, + "navigation_dom_content_loaded_ms": 30, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 4, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 51.39999997615814, + "navigation_dom_content_loaded_ms": 40.89999997615814, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "limitations": "Debug server, local loopback HTTP, synthetic 1024x768 gradient source, 250px lossless WebP preview, 128 posts share 65 unique URLs. Initial viewport measured for 500ms after load. Browser setup/CDP overhead; cold browser cache, warm server/OS cache. Resource Timing adds an estimated 300-byte header cost; CDP records transfer separately. Load-event timings are observations, not isolated proof of frontend speedup. Original loaded only after click; missing/corrupt fallback and missing-file recovery asserted. Raw output remains in local output/playwright and /tmp." +} diff --git a/docs/thumbnail-evidence/http-baseline.json b/docs/thumbnail-evidence/http-baseline.json new file mode 100644 index 00000000..a8b30287 --- /dev/null +++ b/docs/thumbnail-evidence/http-baseline.json @@ -0,0 +1,346 @@ +{ + "limitations": "Debug build, loopback HTTP/1.1, warm OS cache, 16 concurrent requests, current-thread runtime; middleware/browser cache excluded. DB checkouts measured; SQL count and syscall count require separate instrumentation. ps CPU cumulative; RSS sampled, not peak. Original removed before requests.", + "samples": [ + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 33439, + "latency": { + "max_us": 27858, + "p50_us": 2232, + "p95_us": 25280 + }, + "process_cpu_time_and_rss_kib": "0:00.72 47952", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 25510, + "latency": { + "max_us": 5208, + "p50_us": 2804, + "p95_us": 3498 + }, + "process_cpu_time_and_rss_kib": "0:00.76 48064", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 32583, + "latency": { + "max_us": 6700, + "p50_us": 3604, + "p95_us": 4722 + }, + "process_cpu_time_and_rss_kib": "0:00.80 48432", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 28545, + "latency": { + "max_us": 6059, + "p50_us": 3001, + "p95_us": 4957 + }, + "process_cpu_time_and_rss_kib": "0:00.84 48544", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 30023, + "latency": { + "max_us": 6610, + "p50_us": 3430, + "p95_us": 4088 + }, + "process_cpu_time_and_rss_kib": "0:00.88 48656", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 27221, + "latency": { + "max_us": 5912, + "p50_us": 2920, + "p95_us": 3883 + }, + "process_cpu_time_and_rss_kib": "0:00.92 48656", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 31531, + "latency": { + "max_us": 6433, + "p50_us": 3621, + "p95_us": 4483 + }, + "process_cpu_time_and_rss_kib": "0:00.96 48656", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 27485, + "latency": { + "max_us": 6402, + "p50_us": 2961, + "p95_us": 3613 + }, + "process_cpu_time_and_rss_kib": "0:01.00 48784", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 30575, + "latency": { + "max_us": 6260, + "p50_us": 3503, + "p95_us": 4425 + }, + "process_cpu_time_and_rss_kib": "0:01.04 48800", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 25194, + "latency": { + "max_us": 5474, + "p50_us": 2680, + "p95_us": 3471 + }, + "process_cpu_time_and_rss_kib": "0:01.08 48800", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 29966, + "latency": { + "max_us": 6182, + "p50_us": 3401, + "p95_us": 4376 + }, + "process_cpu_time_and_rss_kib": "0:01.12 48800", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 25804, + "latency": { + "max_us": 5505, + "p50_us": 2757, + "p95_us": 5041 + }, + "process_cpu_time_and_rss_kib": "0:01.15 48800", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 28657, + "latency": { + "max_us": 6732, + "p50_us": 3242, + "p95_us": 4174 + }, + "process_cpu_time_and_rss_kib": "0:01.19 48832", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 25678, + "latency": { + "max_us": 5548, + "p50_us": 2810, + "p95_us": 3251 + }, + "process_cpu_time_and_rss_kib": "0:01.22 48832", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 31150, + "latency": { + "max_us": 4532, + "p50_us": 3649, + "p95_us": 4131 + }, + "process_cpu_time_and_rss_kib": "0:01.27 48832", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 26708, + "latency": { + "max_us": 5658, + "p50_us": 2863, + "p95_us": 4983 + }, + "process_cpu_time_and_rss_kib": "0:01.30 48832", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 29845, + "latency": { + "max_us": 6263, + "p50_us": 3341, + "p95_us": 5603 + }, + "process_cpu_time_and_rss_kib": "0:01.34 48960", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 25653, + "latency": { + "max_us": 5442, + "p50_us": 2716, + "p95_us": 3586 + }, + "process_cpu_time_and_rss_kib": "0:01.38 48976", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 31436, + "latency": { + "max_us": 4213, + "p50_us": 3703, + "p95_us": 4012 + }, + "process_cpu_time_and_rss_kib": "0:01.42 48976", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 27288, + "latency": { + "max_us": 5716, + "p50_us": 2897, + "p95_us": 4989 + }, + "process_cpu_time_and_rss_kib": "0:01.46 48976", + "requests": 128 + } + } + ], + "thumbnail_bytes": 556 +} \ No newline at end of file diff --git a/docs/thumbnail-evidence/http-final-repeat.json b/docs/thumbnail-evidence/http-final-repeat.json new file mode 100644 index 00000000..2dcfb02f --- /dev/null +++ b/docs/thumbnail-evidence/http-final-repeat.json @@ -0,0 +1,346 @@ +{ + "limitations": "Debug build, loopback HTTP/1.1, warm OS cache, 16 concurrent requests, current-thread runtime; middleware/browser cache excluded. DB checkouts measured; SQL count and syscall count require separate instrumentation. ps CPU cumulative; RSS sampled, not peak. Original removed before requests.", + "samples": [ + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 34380, + "latency": { + "max_us": 28665, + "p50_us": 1811, + "p95_us": 17779 + }, + "process_cpu_time_and_rss_kib": "0:00.70 48064", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 17250, + "latency": { + "max_us": 3026, + "p50_us": 1807, + "p95_us": 2812 + }, + "process_cpu_time_and_rss_kib": "0:00.74 48224", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22986, + "latency": { + "max_us": 4204, + "p50_us": 2517, + "p95_us": 3220 + }, + "process_cpu_time_and_rss_kib": "0:00.79 48352", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18741, + "latency": { + "max_us": 3649, + "p50_us": 1953, + "p95_us": 3238 + }, + "process_cpu_time_and_rss_kib": "0:00.83 48496", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 21114, + "latency": { + "max_us": 4138, + "p50_us": 2277, + "p95_us": 3365 + }, + "process_cpu_time_and_rss_kib": "0:00.88 48608", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 17841, + "latency": { + "max_us": 3543, + "p50_us": 1845, + "p95_us": 2505 + }, + "process_cpu_time_and_rss_kib": "0:00.93 48608", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22351, + "latency": { + "max_us": 4437, + "p50_us": 2493, + "p95_us": 3324 + }, + "process_cpu_time_and_rss_kib": "0:00.98 48672", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 19291, + "latency": { + "max_us": 3764, + "p50_us": 2001, + "p95_us": 3560 + }, + "process_cpu_time_and_rss_kib": "0:01.02 48800", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 21414, + "latency": { + "max_us": 3477, + "p50_us": 2478, + "p95_us": 2745 + }, + "process_cpu_time_and_rss_kib": "0:01.07 48816", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18079, + "latency": { + "max_us": 3742, + "p50_us": 1869, + "p95_us": 3372 + }, + "process_cpu_time_and_rss_kib": "0:01.11 48816", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22742, + "latency": { + "max_us": 4190, + "p50_us": 2655, + "p95_us": 3413 + }, + "process_cpu_time_and_rss_kib": "0:01.16 48816", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18891, + "latency": { + "max_us": 3638, + "p50_us": 1987, + "p95_us": 3186 + }, + "process_cpu_time_and_rss_kib": "0:01.20 48816", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22022, + "latency": { + "max_us": 4206, + "p50_us": 2505, + "p95_us": 3632 + }, + "process_cpu_time_and_rss_kib": "0:01.24 48816", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18390, + "latency": { + "max_us": 3722, + "p50_us": 1890, + "p95_us": 3401 + }, + "process_cpu_time_and_rss_kib": "0:01.28 48816", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22432, + "latency": { + "max_us": 3022, + "p50_us": 2606, + "p95_us": 2897 + }, + "process_cpu_time_and_rss_kib": "0:01.33 48816", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18502, + "latency": { + "max_us": 3709, + "p50_us": 1924, + "p95_us": 3381 + }, + "process_cpu_time_and_rss_kib": "0:01.37 48816", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 20991, + "latency": { + "max_us": 4568, + "p50_us": 2301, + "p95_us": 3353 + }, + "process_cpu_time_and_rss_kib": "0:01.41 48832", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 17826, + "latency": { + "max_us": 3552, + "p50_us": 1839, + "p95_us": 2386 + }, + "process_cpu_time_and_rss_kib": "0:01.46 48832", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 21886, + "latency": { + "max_us": 3624, + "p50_us": 2505, + "p95_us": 2930 + }, + "process_cpu_time_and_rss_kib": "0:01.50 48832", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 19120, + "latency": { + "max_us": 3870, + "p50_us": 1983, + "p95_us": 3374 + }, + "process_cpu_time_and_rss_kib": "0:01.55 48832", + "requests": 128 + } + } + ], + "thumbnail_bytes": 556 +} \ No newline at end of file diff --git a/docs/thumbnail-evidence/http-final.json b/docs/thumbnail-evidence/http-final.json new file mode 100644 index 00000000..611f842e --- /dev/null +++ b/docs/thumbnail-evidence/http-final.json @@ -0,0 +1,346 @@ +{ + "limitations": "Debug build, loopback HTTP/1.1, warm OS cache, 16 concurrent requests, current-thread runtime; middleware/browser cache excluded. DB checkouts measured; SQL count and syscall count require separate instrumentation. ps CPU cumulative; RSS sampled, not peak. Original removed before requests.", + "samples": [ + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 29652, + "latency": { + "max_us": 27912, + "p50_us": 1705, + "p95_us": 25540 + }, + "process_cpu_time_and_rss_kib": "0:00.73 47888", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 17498, + "latency": { + "max_us": 3436, + "p50_us": 1802, + "p95_us": 2880 + }, + "process_cpu_time_and_rss_kib": "0:00.77 47984", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22750, + "latency": { + "max_us": 4217, + "p50_us": 2596, + "p95_us": 3475 + }, + "process_cpu_time_and_rss_kib": "0:00.82 48240", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18747, + "latency": { + "max_us": 3712, + "p50_us": 1938, + "p95_us": 2577 + }, + "process_cpu_time_and_rss_kib": "0:00.86 48368", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 21775, + "latency": { + "max_us": 3444, + "p50_us": 2514, + "p95_us": 2940 + }, + "process_cpu_time_and_rss_kib": "0:00.90 48416", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18058, + "latency": { + "max_us": 3776, + "p50_us": 1870, + "p95_us": 3274 + }, + "process_cpu_time_and_rss_kib": "0:00.95 48432", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22328, + "latency": { + "max_us": 4932, + "p50_us": 2488, + "p95_us": 3292 + }, + "process_cpu_time_and_rss_kib": "0:00.99 48480", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18924, + "latency": { + "max_us": 3689, + "p50_us": 1983, + "p95_us": 3355 + }, + "process_cpu_time_and_rss_kib": "0:01.03 48608", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22027, + "latency": { + "max_us": 3941, + "p50_us": 2530, + "p95_us": 3217 + }, + "process_cpu_time_and_rss_kib": "0:01.08 48624", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18363, + "latency": { + "max_us": 3765, + "p50_us": 1898, + "p95_us": 2403 + }, + "process_cpu_time_and_rss_kib": "0:01.13 48624", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22624, + "latency": { + "max_us": 4185, + "p50_us": 2542, + "p95_us": 3624 + }, + "process_cpu_time_and_rss_kib": "0:01.17 48624", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 19327, + "latency": { + "max_us": 3793, + "p50_us": 2003, + "p95_us": 3194 + }, + "process_cpu_time_and_rss_kib": "0:01.21 48624", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22308, + "latency": { + "max_us": 4124, + "p50_us": 2506, + "p95_us": 3108 + }, + "process_cpu_time_and_rss_kib": "0:01.26 48656", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18789, + "latency": { + "max_us": 3693, + "p50_us": 1930, + "p95_us": 2603 + }, + "process_cpu_time_and_rss_kib": "0:01.30 48656", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22698, + "latency": { + "max_us": 4602, + "p50_us": 2609, + "p95_us": 2880 + }, + "process_cpu_time_and_rss_kib": "0:01.35 48656", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18904, + "latency": { + "max_us": 4054, + "p50_us": 1963, + "p95_us": 2493 + }, + "process_cpu_time_and_rss_kib": "0:01.39 48656", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 21806, + "latency": { + "max_us": 3040, + "p50_us": 2521, + "p95_us": 2851 + }, + "process_cpu_time_and_rss_kib": "0:01.43 48656", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18195, + "latency": { + "max_us": 3699, + "p50_us": 1878, + "p95_us": 2438 + }, + "process_cpu_time_and_rss_kib": "0:01.47 48672", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22505, + "latency": { + "max_us": 4205, + "p50_us": 2589, + "p95_us": 3549 + }, + "process_cpu_time_and_rss_kib": "0:01.52 48672", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 19186, + "latency": { + "max_us": 3865, + "p50_us": 1996, + "p95_us": 2664 + }, + "process_cpu_time_and_rss_kib": "0:01.56 48672", + "requests": 128 + } + } + ], + "thumbnail_bytes": 556 +} \ No newline at end of file diff --git a/docs/thumbnail-pipeline-audit.md b/docs/thumbnail-pipeline-audit.md new file mode 100644 index 00000000..59c0b2ea --- /dev/null +++ b/docs/thumbnail-pipeline-audit.md @@ -0,0 +1,214 @@ +# Thumbnail pipeline audit — 2026-10-04 + +Audited from `c7e4be7` on branch `1.7.0-staging`. No dependencies, schema, +thumbnail encoding, templates, CSS, post URLs, or production JavaScript changed. + +**Lifecycle and existing behavior** + +- Multipart uploads stream to temporary files. MIME/structure, board policy, + size and decoded-image limits are validated before persistence. The existing + `MediaUploadGate` permits one public media upload at a time, including parsing + and processing. Decoder validation remains mandatory even for deduplicated + uploads. Originals are capped at 40 million decoded pixels and decoder + allocations at 256 MiB. +- `handlers::process_primary_upload` hashes the upload and consults `file_hashes`. + A same-board hit reuses the stored original and thumbnail when both exist; + missing outputs cause upload-time processing again. Empty thumbnail paths are + valid for generic files. No content decode is performed on thumbnail GETs. +- `utils/files/storage.rs` stages UUID-named files, invokes + `MediaProcessor::process_upload`, and records the actual thumbnail output path. + Image conversion happens first; thumbnail generation decodes the persisted + result. JPEG orientation and metadata stripping retain their existing behavior. + Validation, conversion and thumbnailing can therefore decode a fresh image + separately; that work happens on upload, never on page or thumbnail delivery. +- `media/thumbnail.rs` fits still previews into the configured square (default + 250 × 250), preserving aspect ratio and avoiding enlargement of small images. + Triangle filtering and the `image` crate's lossless WebP encoding remain + unchanged; there is no lossy quality setting. Animated media uses a still first + composited frame for its preview; the original conversion preserves animation. + PDF first pages use Hayro; video frames use optional FFmpeg then the same Rust + WebP encoder. Failures can produce generated 250 × 250 SVG placeholders. Audio + initially uses SVG, then durable workers publish waveform PNGs. Background + workers are capped at four; queued job claiming/completion is transactional. +- Upload finalization persists post/file-hash paths together with filesystem + recovery intent. Referenced thumbnails survive cache eviction. Ordinary + page loads do not regenerate previews. Generation failure can leave an original + without a thumbnail; pending/failed/pruned media has dedicated template states. +- Board, catalog, archive, thread and preview/update APIs render stored paths + into `/boards/{board}/thumbs/{filename}` URLs. Rendering does not stat or decode + thumbnail files. Catalog/index/archive images are lazy with async decoding; + thread OPs are eager and replies lazy. CSS limits post previews to 150 px and + replies to 80 px (with responsive overrides). Posts store no thumbnail + width/height metadata, so post images have no intrinsic-size HTML hints. +- `static/main.js` installs one load/error fallback per image; an already complete + image with no natural size may call `decode()` before showing the fallback. + There is no custom thumbnail download queue or original-image prefetch. The + full image's `src` is assigned from `data-src` on expansion; videos/audio use + `preload="none"`, PDFs use `about:blank` until opened. Native browser request + scheduling and caching handle concurrency and repeated URLs. Cross-board hover + previews also deduplicate in-flight HTML fetches and retain a per-page HTML + cache; pending media is refreshed through the existing post-update endpoint. No-JS links still + open the original. Sensitive-content/NSFW, quote-preview and media hooks remain + intact. +- `/boards/{*media_path}` checks live board permissions before every origin + request, including conditional GETs. Path validation canonicalizes root and + target, checks every path component for symlinks, verifies a regular file and + rejects multiple hardlinks on Unix. The leaf is inspected during the component + walk and again after canonicalization; the file service then obtains metadata + from its opened file. Those checks were preserved. `ServeFile` streams only the + requested file in bounded chunks (default 64 KiB). It supports HEAD, byte + ranges, Last-Modified/If-Modified-Since and metadata-derived + ETag/If-None-Match; ETags do not hash/read image contents. GET/304 still opens + the file and reads metadata; 304 does not read a payload. WebP/PNG are not + HTTP-compressed again. Generated SVGs retain restrictive CSP/nosniff headers. +- Public UUID media already had `public, max-age=31536000, immutable`. + View-password media uses `private, no-cache, must-revalidate`; unversioned + replaceable board favicons use a one-hour TTL. Revalidated private media still + checks access. Missing SVG waveforms redirect temporarily to their completed + PNGs; old MP4/MKV URLs redirect permanently to WebM siblings. Corrupt persisted + thumbnail bytes receive an ordinary file response and the browser displays a + fallback; serving never decodes or repairs them. + +**Findings and implemented changes** + +The browser baseline did not expose duplicate downloads, eager original loading, +far-offscreen unique thumbnail floods, or unnecessary warm-cache media requests. +The worthwhile changes were on the server request path: + +1. Synchronous canonicalization, per-component filesystem checks and legacy + sibling resolution previously ran on the async executor after DB preflight. + They now run in that same blocking preflight. The DB connection is released + before filesystem work. A rare replacement/open race also resolves its legacy + sibling on a blocking worker. Security checks and their syscall count remain + intact; the improvement is executor responsiveness and overlapping work. +2. A public/post-password request carrying an admin cookie previously performed a + session SELECT despite already having view access. Session lookup now occurs + only when an unresolved view-password gate requires administrator access. + Valid board unlock cookies also avoid that extra lookup. Board access is still + loaded from the DB each time; no authorization cache was introduced. Public + cookie-bearing requests go from two application SELECTs to one (code-path + accounting; not a SQL trace). Pool validation SQL is separate. +3. Media errors now carry private no-store headers. Previously an initial 404 had + no explicit cache policy, and a failed open after successful validation could + inherit the one-year immutable policy. A missing/generated-later file or a + temporary denial must not be retained as immutable content. This is a + correctness improvement established from the response path; no negative-cache + performance win is claimed. Successful 200/206/304 policies remain unchanged. +4. Response metadata uses the validated file path, avoiding the earlier redundant + root/path construction. No large image buffers or application cache were added. + +**Measured evidence** + +The tracked ignored Rust workload runs the production handler and `ServeFile` +through real loopback HTTP/1.1: 128 pre-generated 250 px WebP files, concurrency +16, five batches for each visitor/admin-cookie and full/conditional combination. +The original is deleted before requests, proving delivery is independent of +source decoding/resizing. It measures complete-response latency, body bytes and +DB checkouts, with cumulative CPU/end-of-batch RSS observations from `ps`. + +Apple M2 Pro, arm64 macOS, Rust 1.99.0, debug build, current-thread Tokio harness, +warm filesystem/server cache. These are local workload results, not a release +capacity claim. Moving blocking calls can have a different timing effect on a +multi-thread production runtime or slower storage. + +| 128-request batch | Before median ms | After median ms | Before request p50/p95 ms | After request p50/p95 ms | +| --- | ---: | ---: | ---: | ---: | +| Visitor, full responses | 30.023 | 22.027 | 3.341 / 4.425 | 2.514 / 3.108 | +| Visitor, conditional 304 | 25.653 | 18.195 | 2.804 / 3.498 | 1.878 / 2.603 | +| Admin cookie, full responses | 31.436 | 22.624 | 3.621 / 4.376 | 2.589 / 3.475 | +| Admin cookie, conditional 304 | 27.288 | 18.924 | 2.897 / 4.983 | 1.983 / 2.664 | + +An independent optimized repeat recorded 21.414, 17.841, 22.432 and 18.891 ms, +respectively. Every full batch transferred 71,168 body bytes before and after; +304 batches transferred zero body bytes. All origin batches checked out 128 DB +connections. No reduction in filesystem calls, total CPU or RSS is established. +`ps` includes client/setup/instrumentation and is not a peak-memory or isolated +server-CPU measurement. Syscalls and SQL statements were not traced on this host. + +The local browser workload extends `tests/e2e/helpers.ts` and diagnostics rather +than introducing a framework. It uploads a real 1024 × 768 gradient, then seeds +0, 4 and 128 image-post cases. The many-post page has 65 unique thumbnails, +including repeated URLs. CDP observes cache/response events without request +routing (routing would disable browser cache); Resource Timing observes bytes. +Five visits per size include a cleared browser cache followed by four warm visits. + +| Browser case | Before / after unique origin thumbnail responses | Before / after Resource Timing transfer bytes | +| --- | ---: | ---: | +| No images, cold or warm | 0 / 0 | 0 / 0 | +| Four images, cold | 4 / 4 | 133,264 / 133,264 | +| Four images, warm | 0 / 0 | 0 / 0 | +| 128 posts, initial cold viewport | 25 / 25 | 832,900 / 832,900 | +| 128 posts, warm viewport | 0 / 0 | 0 / 0 | +| 128 posts, complete scroll | 65 / 65 total unique URLs | No duplicate URL requests | + +Browser WebP previews were 33,016 bytes in both runs. The initial many-post +viewport transferred 825,400 body bytes plus Resource Timing's estimated header +cost. Some far-offscreen repeated elements obtain natural dimensions from shared +cached URLs; that does not represent additional network downloads. Cold load-event +completion was 52.0 ms before and 52.8 ms after. Warm navigation timings varied, +including the no-image control, so they do not establish a frontend speedup. +Opening the image loaded the correct 1024 px original; missing/corrupt previews +showed fallbacks, and a missing file created later recovered on navigation. + +Raw HTTP samples: [baseline](thumbnail-evidence/http-baseline.json), +[optimized](thumbnail-evidence/http-final.json), +[repeat](thumbnail-evidence/http-final-repeat.json). Browser aggregate/per-resource +observations: [baseline](thumbnail-evidence/browser-baseline.json), +[optimized](thumbnail-evidence/browser-final.json). Full browser artifacts remain +under the repository's ignored `output/playwright/thumbnails/` directory. The new +browser spec remains local under ignored `tests/e2e/`, following `.gitignore`'s +existing policy; the Rust benchmark and evidence are versionable. + +**Validation and reproduction** + +```sh +cargo fmt --all --check +cargo clippy --workspace --all-targets --all-features -- -D warnings -D clippy::all -D clippy::pedantic -D clippy::nursery -D clippy::cargo +cargo test --workspace --all-features +cargo test --workspace --all-features -- --test-threads=1 +RUSTCHAN_THUMB_EVIDENCE=/tmp/thumbnail-http.json cargo test --locked --lib server::handlers::board::media::performance::thumbnail_http_workload -- --ignored --exact +RUSTCHAN_E2E_SKIP_BUILD=1 RUSTCHAN_AUDIT_OUTPUT=output/playwright/thumbnails/check npx playwright test tests/e2e/thumbnail-loading.spec.ts --project=chromium --workers=1 +RUSTCHAN_E2E_SKIP_BUILD=1 npx playwright test tests/e2e/media.spec.ts tests/e2e/password-boards.spec.ts tests/e2e/phase3-media-runtime.spec.ts tests/e2e/non-video-media.spec.ts --project=chromium --project=webkit --project=chromium-nojs --workers=2 +``` + +Build the current server with `cargo build --locked --bin rustchan-cli` before +using `RUSTCHAN_E2E_SKIP_BUILD=1`. Browser setup also generates its native update +fixture using the existing global setup. + +Formatting and strict Clippy passed. New request regressions cover ETag/date 304, +HEAD, byte ranges, original independence, missing-file recovery/no-store, +permission changes, unlock retirement, session revocation, and legacy redirects. +The browser matrix passed 29 tests with 10 existing engine/capability skips; +the many-image benchmark passed before and after. Full parallel Rust runs exposed +shared global configuration-save locking and thumbnail-failure-injection +interference with a PDF test. The complete serial run passed: 1,353 library +tests, 1,353 binary tests, five CLI runtime tests, one native UI fixture test and +one doc test (2,713 passes total; 10 intentional ignores across library/binary). +No unrelated test or production configuration logic was modified. + +**Deliberately deferred** + +- No stronger public caching, application thumbnail cache, static-route bypass, + or permission cache: existing warm visits already generate zero media origin + work, while private media must recheck authorization. +- No lossy codecs, smaller previews or extra responsive variants: preview quality, + compatibility and existing high-DPI presentation take precedence; network bytes + did not regress. Lossless gradient previews can be relatively large. +- No runtime dimension probing. Intrinsic dimension metadata would require a + coordinated schema/upload/restore/template change; guessing square dimensions + would alter portrait/landscape layout. Profile layout shifts before doing that. +- No shared decoded-image upload pipeline, new generation queue, or single-flight + cache: upload processing is already gated, requests never generate thumbnails, + and fusing validation/conversion would broaden this change into media semantics. +- No reduction of symlink/hardlink/canonicalization checks, permanent root trust + cache, or alternate file-server implementation: the checks protect untrusted + stored paths. A descriptor-based race-hardening design is separate work. +- Public immutable retention and cached corrupt-file repair at the same URL retain + their existing semantics; replacing immutable assets should use a new URL. + +**Files changed** + +- `src/handlers/board/media.rs`: delivery preflight, error cache policy and request regressions. +- `src/handlers/board/media/performance.rs`: opt-in actual HTTP workload. +- `docs/thumbnail-pipeline-audit.md` and `docs/thumbnail-evidence/{http-baseline,http-final,http-final-repeat,browser-baseline,browser-final}.json`: audit and measured evidence. +- Local ignored `tests/e2e/thumbnail-loading.spec.ts`: browser workload and recovery assertions. diff --git a/src/captcha.rs b/src/captcha.rs index 669f9c71..b4c8d712 100644 --- a/src/captcha.rs +++ b/src/captcha.rs @@ -2,8 +2,9 @@ use ::captcha::{ filters::{Dots, Grid, Noise, Wave}, Captcha, }; -use dashmap::DashMap; +use parking_lot::Mutex; use sha2::{Digest as _, Sha256}; +use std::collections::HashMap; use std::sync::LazyLock; use subtle::ConstantTimeEq as _; @@ -22,10 +23,49 @@ const CAPTCHA_CHARSET: &[char] = &[ ]; /// In-memory, single-use challenge store keyed by challenge identifier. -static CAPTCHA_CHALLENGES: LazyLock> = - LazyLock::new(DashMap::new); +static CAPTCHA_CHALLENGES: LazyLock> = LazyLock::new(Mutex::default); +/// Hard cap includes outstanding generation reservations. +const CAPTCHA_MAX_CHALLENGES: usize = 4096; + +#[derive(Default)] +/// Bounded challenge state and amortized expiration scheduling. +struct ChallengeStore { + /// Single-use challenges awaiting a response. + challenges: HashMap, + /// Slots reserved before expensive generation starts. + generating: usize, + /// Last whole-store expiry scan. + last_pruned: Option, +} + +/// A capacity reservation reclaimed on every failed generation path. +struct ChallengeReservation<'a> { + /// Store that owns the reservation. + store: &'a Mutex, +} + +impl Drop for ChallengeReservation<'_> { + fn drop(&mut self) { + let mut store = self.store.lock(); + store.generating = store.generating.saturating_sub(1); + } +} + +/// Reserve a slot atomically before PNG generation; never queue at capacity. +fn reserve_challenge( + store: &Mutex, + now: i64, +) -> Result, CaptchaImageError> { + let mut state = store.lock(); + prune_store(&mut state, now); + if state.challenges.len().saturating_add(state.generating) >= CAPTCHA_MAX_CHALLENGES { + return Err(CaptchaImageError::Busy); + } + state.generating = state.generating.saturating_add(1); + drop(state); + Ok(ChallengeReservation { store }) +} -#[derive(Clone)] /// Hashed server-side state for one outstanding CAPTCHA challenge. struct StoredCaptchaChallenge { /// Board short name that owns the challenge. @@ -43,6 +83,8 @@ pub enum CaptchaImageError { InvalidRequest, /// The CAPTCHA backend failed to encode an image. GenerationFailed, + /// Outstanding challenges have exhausted the bounded store. + Busy, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -91,7 +133,7 @@ pub fn generate_captcha_image( return Err(CaptchaImageError::InvalidRequest); } - prune_expired(chrono::Utc::now().timestamp()); + let _reservation = reserve_challenge(&CAPTCHA_CHALLENGES, chrono::Utc::now().timestamp())?; let mut captcha = Captcha::new(); let _configured = captcha @@ -139,7 +181,7 @@ pub fn verify_captcha( let now = chrono::Utc::now().timestamp(); prune_expired(now); - let Some((_, challenge)) = CAPTCHA_CHALLENGES.remove(captcha_id) else { + let Some(challenge) = CAPTCHA_CHALLENGES.lock().challenges.remove(captcha_id) else { return Err(CaptchaValidationError::Expired); }; @@ -163,7 +205,7 @@ pub fn verify_captcha( /// Hash and store one challenge when its generated answer is valid. fn store_challenge(board_short: &str, captcha_id: &str, answer: &str, now: i64) { if let Some(answer_hash) = answer_hash(board_short, captcha_id, answer) { - let _previous_value = CAPTCHA_CHALLENGES.insert( + let _previous_value = CAPTCHA_CHALLENGES.lock().challenges.insert( captcha_id.to_owned(), StoredCaptchaChallenge { board_short: board_short.to_owned(), @@ -176,7 +218,21 @@ fn store_challenge(board_short: &str, captcha_id: &str, answer: &str, now: i64) /// Remove challenges whose expiration time is at or before `now`. fn prune_expired(now: i64) { - CAPTCHA_CHALLENGES.retain(|_, challenge| challenge.expires_at > now); + prune_store(&mut CAPTCHA_CHALLENGES.lock(), now); +} + +/// Expire in one bounded scan at most once per minute, including under saturation. +fn prune_store(store: &mut ChallengeStore, now: i64) { + if store + .last_pruned + .is_some_and(|last| now.saturating_sub(last) < 60) + { + return; + } + store + .challenges + .retain(|_, challenge| challenge.expires_at > now); + store.last_pruned = Some(now); } /// Compute the secret-bound answer hash for a scoped challenge. @@ -232,7 +288,7 @@ pub mod testing { let Some(answer_hash) = super::answer_hash(board_short, captcha_id, answer) else { return; }; - let _previous_value = super::CAPTCHA_CHALLENGES.insert( + let _previous_value = super::CAPTCHA_CHALLENGES.lock().challenges.insert( captcha_id.to_owned(), super::StoredCaptchaChallenge { board_short: board_short.to_owned(), @@ -244,7 +300,10 @@ pub mod testing { /// Return whether a challenge identifier is currently stored. pub fn challenge_exists_for_test(captcha_id: &str) -> bool { - super::CAPTCHA_CHALLENGES.contains_key(captcha_id) + super::CAPTCHA_CHALLENGES + .lock() + .challenges + .contains_key(captcha_id) } } @@ -340,4 +399,20 @@ mod tests { Err(CaptchaValidationError::Missing) ); } + #[test] + fn challenge_capacity_includes_concurrent_reservations_and_recovers_on_drop() { + let store = Mutex::new(ChallengeStore::default()); + let reservations = (0..CAPTCHA_MAX_CHALLENGES) + .map(|_| reserve_challenge(&store, 100)) + .collect::>(); + assert!(reservations.iter().all(Result::is_ok)); + assert!(matches!( + reserve_challenge(&store, 100), + Err(CaptchaImageError::Busy) + )); + assert_eq!(store.lock().generating, CAPTCHA_MAX_CHALLENGES); + drop(reservations); + assert_eq!(store.lock().generating, 0); + assert!(reserve_challenge(&store, 100).is_ok()); + } } diff --git a/src/config.rs b/src/config.rs index d98771c2..135efc7f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1570,6 +1570,13 @@ impl Config { const MAX_VIDEO_MIB: usize = 2048; const MAX_AUDIO_MIB: usize = 512; self.operator.validate()?; + if !(1..=1_000_000).contains(&self.rate_limit_gets) + || !(1..=86_400).contains(&self.rate_limit_window) + { + anyhow::bail!( + "CONFIG ERROR: browsing request allowance must be 1..=1000000 and window 1..=86400 seconds." + ); + } // cookie_secret is hex-encoded: 64 hex chars = 32 bytes of entropy. if self.cookie_secret.len() < 64 { anyhow::bail!( diff --git a/src/config/admin.rs b/src/config/admin.rs index 047704c3..0fac2273 100644 --- a/src/config/admin.rs +++ b/src/config/admin.rs @@ -1161,11 +1161,24 @@ mod tests { } } +/// Owns the settings lock until the configuration operation completes. +pub(crate) struct SettingsLease(std::fs::File); + +impl Drop for SettingsLease { + fn drop(&mut self) { + // Closing only this descriptor can leave a Unix lock held by a duplicate + // inherited during concurrent process creation. Release it explicitly. + if let Err(error) = self.0.unlock() { + tracing::error!(%error, "failed to release settings lease"); + } + } +} + /// Lock the fixed configuration resource across web saves and supervisor transactions. /// /// # Errors /// Rejects unsafe lock paths and concurrent saves, restarts or configuration recovery. -pub(crate) fn settings_lease(path: &Path) -> anyhow::Result { +pub(crate) fn settings_lease(path: &Path) -> anyhow::Result { let parent = path .parent() .context("settings path needs a parent")? @@ -1219,7 +1232,7 @@ pub(crate) fn settings_lease(path: &Path) -> anyhow::Result { file.try_lock().context( "Another configuration save, update or restart is in progress. Retry after it completes.", )?; - Ok(file) + Ok(SettingsLease(file)) } /// Compare effective startup controls against the running process, excluding live controls. @@ -1363,6 +1376,20 @@ mod restart_tests { ); Ok(()) } + /// A retained descriptor must not extend the owning settings operation's lock. + #[test] + fn settings_lease_release_is_not_delayed_by_a_retained_descriptor() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("settings.toml"); + let lease = settings_lease(&path)?; + let retained_descriptor = lease.0.try_clone()?; + drop(lease); + let next_lease = settings_lease(&path)?; + drop(next_lease); + drop(retained_descriptor); + Ok(()) + } + /// A supervisor's lease rejects all configuration writers until it releases the resource. #[test] fn supervisor_lease_rejects_config_save_without_partial_mutation() -> anyhow::Result<()> { diff --git a/src/config/template.rs b/src/config/template.rs index 4bc7a92a..c3ac91ab 100644 --- a/src/config/template.rs +++ b/src/config/template.rs @@ -200,7 +200,8 @@ db_warn_threshold_mb = 2048 job_queue_capacity = 1000 # When true, threads that would be hard-deleted by the prune worker are instead -# moved to the archive table, even on boards where archiving is disabled. +# marked archived in place, even on boards where archiving is disabled. +# The board archive count cap still applies; this does not guarantee permanent retention. archive_before_prune = true # Maximum total size (MiB) of all thumbnail/waveform cache files across all boards. diff --git a/src/db/accounts.rs b/src/db/accounts.rs index 5d12892f..95aac1aa 100644 --- a/src/db/accounts.rs +++ b/src/db/accounts.rs @@ -43,15 +43,26 @@ pub fn manage_account( "password is too long" ); crate::utils::crypto::validate_password(new_password)?; - let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; - let session = super::get_session(&tx, session)?.context("administrator session expired")?; + let authorized = super::get_session(conn, session)?.context("administrator session expired")?; let actor = - super::get_admin_name_by_id(&tx, session.admin_id)?.context("administrator missing")?; - let user = super::get_admin_by_username(&tx, &actor)?.context("administrator missing")?; + super::get_admin_name_by_id(conn, authorized.admin_id)?.context("administrator missing")?; + let user = super::get_admin_by_username(conn, &actor)?.context("administrator missing")?; if !crate::utils::crypto::verify_password(current_password, &user.password_hash)? { return Ok(false); } + // Password work runs before taking SQLite's single writer lock. Recheck + // session and credentials under that lock to close the authorization race. let hash = crate::utils::crypto::hash_password(new_password)?; + let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let current_session = + super::get_session(&tx, session)?.context("administrator session expired")?; + let current_user = + super::get_admin_by_username(&tx, &actor)?.context("administrator missing")?; + ensure!( + current_session.admin_id == authorized.admin_id + && current_user.password_hash == user.password_hash, + "administrator authorization changed; retry after signing in again" + ); match action { AccountAction::Create => { ensure!( diff --git a/src/db/mod.rs b/src/db/mod.rs index c3d16b84..d98f1489 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -25,6 +25,8 @@ mod pool; pub mod posts; /// Baseline schema installation, repair, and verification. mod schema; +/// Scoped `SQLite` progress limits for public full-text searches. +mod search_budget; /// First-run setup state and completion markers. pub mod setup; /// Built-in and custom theme persistence. diff --git a/src/db/posts.rs b/src/db/posts.rs index 7cd2dcdf..407ba2bc 100644 --- a/src/db/posts.rs +++ b/src/db/posts.rs @@ -180,6 +180,48 @@ pub fn get_posts_for_thread(conn: &rusqlite::Connection, thread_id: i64) -> Resu Ok(posts) } +/// Load at most 200 replies and the opening post, with a cursor for older replies. +/// Numeric cursors use the thread/id index, without deep OFFSET scans. +/// +/// # Errors +/// Returns an error if the database query fails. +pub fn get_thread_post_window( + conn: &rusqlite::Connection, + thread_id: i64, + before: Option, +) -> Result<(Vec, Option)> { + let mut stmt = conn.prepare_cached(&format!( + "SELECT {POST_SELECT_COLUMNS} FROM posts + WHERE thread_id = ?1 AND is_op = 0 AND id <= ?2 + ORDER BY id DESC LIMIT 201" + ))?; + let mut posts = stmt + .query_map( + params![ + thread_id, + before.map_or(i64::MAX, |id| id.saturating_sub(1)) + ], + map_post, + )? + .collect::>>()?; + let has_older = posts.len() > 200; + posts.truncate(200); + let older_before = has_older + .then(|| posts.last().map(|post| post.id)) + .flatten(); + let op = conn + .prepare_cached(&format!( + "SELECT {POST_SELECT_COLUMNS} FROM posts WHERE thread_id = ?1 AND is_op = 1 LIMIT 1" + ))? + .query_row(params![thread_id], map_post) + .optional()?; + if let Some(op) = op { + posts.push(op); + } + posts.reverse(); + Ok((posts, older_before)) +} + /// Fetch posts in `thread_id` whose id is strictly greater than `since_id`. /// Returns them oldest-first. Used by the thread auto-update polling endpoint. /// @@ -365,7 +407,10 @@ pub fn get_post_submission( Ok(conn .query_row( "SELECT thread_id, post_id, is_thread - FROM post_submissions + FROM ( + SELECT * FROM post_submissions + UNION ALL SELECT * FROM post_submission_tombstones + ) WHERE submission_token = ?1 AND ip_hash = ?2 AND board_id = ?3 @@ -399,6 +444,14 @@ pub fn record_post_submission( return Ok(()); } + anyhow::ensure!( + !conn.query_row( + "SELECT EXISTS(SELECT 1 FROM post_submission_tombstones WHERE submission_token = ?1)", + [submission_token], + |row| row.get::<_, bool>(0), + )?, + "submission token was already accepted and deleted" + ); conn.execute( "INSERT INTO post_submissions (submission_token, ip_hash, board_id, thread_id, post_id, is_thread) @@ -422,6 +475,12 @@ pub fn record_post_submission( .context("Failed to prune expired post submission tokens") .map(|_affected_rows| ())?; + conn.execute( + "DELETE FROM post_submission_tombstones WHERE created_at < unixepoch() - 604800", + [], + ) + .context("Failed to prune expired deleted submission receipts") + .map(|_affected_rows| ())?; Ok(()) } @@ -522,7 +581,7 @@ pub fn delete_post( } /// Delete a non-opening post inside the caller's transaction. -fn delete_post_reply_in_tx( +pub(crate) fn delete_post_reply_in_tx( conn: &rusqlite::Connection, post_id: i64, ) -> crate::error::Result { @@ -665,12 +724,24 @@ pub fn self_delete_post( post_id: i64, token: &str, delete_window_secs: i64, +) -> crate::error::Result<(SelfDeleteOutcome, Option)> { + self_delete_post_with_validation(conn, post_id, token, delete_window_secs, |_| Ok(())) +} + +/// Recheck mutable request policy under the same write lock as deletion. +pub(crate) fn self_delete_post_with_validation( + conn: &rusqlite::Connection, + post_id: i64, + token: &str, + delete_window_secs: i64, + validate: impl FnOnce(&rusqlite::Connection) -> crate::error::Result<()>, ) -> crate::error::Result<(SelfDeleteOutcome, Option)> { conn.execute_batch("BEGIN IMMEDIATE") .context("Failed to begin self_delete_post transaction")?; let result: crate::error::Result<(SelfDeleteOutcome, Option)> = (|| { + validate(conn)?; let row: Option<(i64, bool, String, i64, bool, bool)> = conn .query_row( "SELECT p.thread_id, p.is_op, p.deletion_token, p.created_at, @@ -761,6 +832,27 @@ pub fn edit_post( new_body: &str, new_body_html: &str, edit_window_secs: i64, +) -> Result { + edit_post_with_validation( + conn, + post_id, + token, + new_body, + new_body_html, + edit_window_secs, + |_| Ok(()), + ) +} + +/// Recheck mutable request policy under the same write lock as editing. +pub(crate) fn edit_post_with_validation( + conn: &rusqlite::Connection, + post_id: i64, + token: &str, + new_body: &str, + new_body_html: &str, + edit_window_secs: i64, + validate: impl FnOnce(&rusqlite::Connection) -> crate::error::Result<()>, ) -> Result { let window = if edit_window_secs <= 0 { 60 @@ -774,6 +866,7 @@ pub fn edit_post( .context("Failed to begin IMMEDIATE transaction for edit_post")?; let result: Result = (|| { + validate(conn)?; let row: Option<(String, i64, bool, bool)> = conn .query_row( "SELECT p.deletion_token, p.created_at, t.locked, t.archived @@ -858,7 +951,7 @@ fn search_terms(query: &str) -> Vec { let mut terms = Vec::new(); let mut current = String::new(); - for ch in query.chars() { + for ch in query.chars().take(crate::models::SEARCH_QUERY_MAX_CHARS) { if ch.is_alphanumeric() { for lower in ch.to_lowercase() { current.push(lower); @@ -906,6 +999,8 @@ pub fn search_posts( offset: i64, ) -> Result> { let _timing = super::diagnostics::QueryTiming::start("search_posts"); + let limit = limit.clamp(1, 100); + let offset = offset.clamp(0, 9_999); let Some(fts_query) = to_fts_query(query) else { return Ok(Vec::new()); }; @@ -922,11 +1017,14 @@ pub fn search_posts( WHERE posts.board_id = ?1 AND posts_fts MATCH ?2 ORDER BY posts.created_at DESC, posts.id DESC LIMIT ?3 OFFSET ?4"; - let mut stmt = conn.prepare_cached(sql)?; - let posts = stmt - .query_map(params![board_id, fts_query, limit, offset], map_post)? - .collect::>>()?; - Ok(posts) + super::search_budget::with_search_budget(conn, || { + let mut stmt = conn.prepare_cached(sql)?; + let posts = stmt + .query_map(params![board_id, fts_query, limit, offset], map_post)? + .collect::>>()?; + Ok(posts) + }) + .map_err(anyhow::Error::from) } /// # Errors @@ -940,13 +1038,17 @@ pub fn count_search_results( let Some(fts_query) = to_fts_query(query) else { return Ok(0); }; - Ok(conn - .prepare_cached( - "SELECT COUNT(*) - FROM posts_fts CROSS JOIN posts ON posts.id = posts_fts.rowid - WHERE posts.board_id = ?1 AND posts_fts MATCH ?2", - )? - .query_row(params![board_id, fts_query], |r| r.get(0))?) + super::search_budget::with_search_budget(conn, || { + Ok(conn + .prepare_cached( + "SELECT COUNT(*) FROM ( + SELECT 1 FROM posts_fts CROSS JOIN posts ON posts.id = posts_fts.rowid + WHERE posts.board_id = ?1 AND posts_fts MATCH ?2 LIMIT 10000 + )", + )? + .query_row(params![board_id, fts_query], |r| r.get(0))?) + }) + .map_err(anyhow::Error::from) } // File deduplication @@ -1110,7 +1212,7 @@ pub fn get_poll_for_thread( /// /// This returns false for two distinct cases: /// 1. The voter has already voted (UNIQUE constraint fires INSERT OR IGNORE) -/// 2. The option does not belong to the poll, or the poll has expired +/// 2. The option does not belong to the poll, or the poll has expired/archived /// /// Callers that need to distinguish these cases should call `cast_vote` and, on /// false, separately query whether the IP has voted on this poll. A future @@ -1129,7 +1231,9 @@ pub fn cast_vote( SELECT ?1, ?2, ?3 FROM poll_options AS po JOIN polls AS p ON p.id = po.poll_id - WHERE po.id = ?2 + JOIN threads AS t ON t.id = p.thread_id + WHERE t.archived = 0 + AND po.id = ?2 AND po.poll_id = ?1 AND p.expires_at > unixepoch()", params![poll_id, option_id, ip_hash], @@ -1613,6 +1717,16 @@ pub fn claim_next_job(conn: &rusqlite::Connection) -> Result Result<()> { + let conn = test_conn()?; + let op_id = seed_search_post(&conn, "window", "opening post")?; + let op = get_post(&conn, op_id)?.context("opening post exists")?; + conn.execute( + "WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<450) + INSERT INTO posts(thread_id,board_id,body,body_html,deletion_token,created_at) + SELECT ?1,?2,'reply','reply','delete',1700000000+x FROM n", + params![op.thread_id, op.board_id], + ) + .map(|_count| ())?; + let mut before = None; + let mut seen = std::collections::HashSet::new(); + loop { + let (posts, older) = super::get_thread_post_window(&conn, op.thread_id, before)?; + anyhow::ensure!( + posts.len() <= 201 && posts.first().is_some_and(|post| post.id == op_id) + ); + for post in posts.iter().filter(|post| !post.is_op) { + anyhow::ensure!(seen.insert(post.id), "reply appeared twice across pages"); + } + if older.is_none() { + break; + } + before = older; + } + anyhow::ensure!(seen.len() == 450); + let plan = conn.prepare("EXPLAIN QUERY PLAN SELECT id FROM posts WHERE thread_id=?1 AND is_op=0 AND id(3))? + .collect::>>()?.join(" "); + anyhow::ensure!( + plan.contains("idx_posts_thread_live") && !plan.contains("TEMP B-TREE"), + "window must use keyset index: {plan}" + ); + Ok(()) + } + #[test] #[expect( clippy::panic_in_result_fn, @@ -4110,6 +4262,11 @@ mod tests { !cast_vote(&conn, expired_poll, expired_option, "late")?, "an expired poll should reject the vote in the write statement" ); + crate::db::set_thread_archived(&conn, thread_id, true)?; + assert!( + !cast_vote(&conn, open_poll, open_option, "archived-voter")?, + "archival must reject votes in the same write statement" + ); Ok(()) } } diff --git a/src/db/schema.rs b/src/db/schema.rs index 99040aa0..f9b93907 100644 --- a/src/db/schema.rs +++ b/src/db/schema.rs @@ -272,10 +272,64 @@ const BASE_SCHEMA_SQL: &str = " ); "; +/// Retain bounded retry receipts after content deletion, including cascade/prune. +/// These historical IDs intentionally have no foreign keys: deletion must not +/// erase the evidence that a submission was already accepted. +const SUBMISSION_TOMBSTONE_SQL: &str = " + CREATE TABLE IF NOT EXISTS post_submission_tombstones ( + submission_token TEXT PRIMARY KEY, + ip_hash TEXT NOT NULL, + board_id INTEGER NOT NULL, + thread_id INTEGER NOT NULL, + post_id INTEGER NOT NULL, + is_thread INTEGER NOT NULL CHECK (is_thread IN (0, 1)), + created_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_submission_tombstones_created + ON post_submission_tombstones(created_at); + CREATE INDEX IF NOT EXISTS idx_post_submissions_board + ON post_submissions(board_id); + CREATE INDEX IF NOT EXISTS idx_post_submissions_thread + ON post_submissions(thread_id); + CREATE INDEX IF NOT EXISTS idx_post_submissions_post + ON post_submissions(post_id); + CREATE TRIGGER IF NOT EXISTS posts_submission_tombstone + BEFORE DELETE ON posts BEGIN + INSERT OR IGNORE INTO post_submission_tombstones + SELECT submission_token, ip_hash, board_id, thread_id, post_id, is_thread, created_at + FROM post_submissions WHERE post_id = OLD.id; + END; + CREATE TRIGGER IF NOT EXISTS threads_submission_tombstone + BEFORE DELETE ON threads BEGIN + INSERT OR IGNORE INTO post_submission_tombstones + SELECT submission_token, ip_hash, board_id, thread_id, post_id, is_thread, created_at + FROM post_submissions WHERE thread_id = OLD.id; + END; + CREATE TRIGGER IF NOT EXISTS boards_submission_tombstone + BEFORE DELETE ON boards BEGIN + INSERT OR IGNORE INTO post_submission_tombstones + SELECT submission_token, ip_hash, board_id, thread_id, post_id, is_thread, created_at + FROM post_submissions WHERE board_id = OLD.id; + END; +"; + +/// Exactly recognized objects added for retry receipts on existing databases. +const SUBMISSION_TOMBSTONE_OBJECTS: [&str; 9] = [ + "post_submission_tombstones", + "sqlite_autoindex_post_submission_tombstones_1", + "idx_submission_tombstones_created", + "idx_post_submissions_board", + "idx_post_submissions_thread", + "idx_post_submissions_post", + "posts_submission_tombstone", + "threads_submission_tombstone", + "boards_submission_tombstone", +]; + /// Complete baseline secondary-index definitions. const INDEX_SCHEMA_SQL: &str = " - CREATE INDEX IF NOT EXISTS idx_threads_board_sticky_bumped - ON threads(board_id, sticky DESC, bumped_at DESC); + CREATE INDEX IF NOT EXISTS idx_threads_active_order + ON threads(board_id, sticky DESC, bumped_at DESC, id DESC) WHERE archived = 0; CREATE INDEX IF NOT EXISTS idx_posts_thread ON posts(thread_id, created_at ASC); CREATE INDEX IF NOT EXISTS idx_posts_thread_live @@ -304,6 +358,8 @@ const INDEX_SCHEMA_SQL: &str = " ON background_jobs(status, priority DESC, created_at ASC); CREATE INDEX IF NOT EXISTS idx_reports_status ON reports(status, created_at DESC); + CREATE INDEX IF NOT EXISTS idx_reports_reporter_created + ON reports(reporter_hash, created_at); CREATE UNIQUE INDEX IF NOT EXISTS idx_reports_open_unique ON reports(post_id, reporter_hash) WHERE status = 'open'; @@ -340,7 +396,8 @@ const INDEX_SCHEMA_SQL: &str = " /// Obsolete theme index accepted only during the known legacy repair path. const LEGACY_THEME_SORT_INDEX: &str = "idx_themes_enabled_sort"; /// Additive indexes introduced after the first package-version baseline. -const ADDITIVE_BASELINE_INDEXES: [&str; 11] = [ +const ADDITIVE_BASELINE_INDEXES: [&str; 12] = [ + "idx_threads_active_order", "idx_posts_thread_live", "idx_posts_board_ip_created", "idx_posts_file_path", @@ -354,7 +411,11 @@ const ADDITIVE_BASELINE_INDEXES: [&str; 11] = [ "idx_ban_appeals_ip_created", ]; /// Redundant indexes removed when the additive index set is installed. -const REDUNDANT_LEGACY_INDEXES: [&str; 2] = ["idx_file_hashes", "idx_posts_thread_id"]; +const REDUNDANT_LEGACY_INDEXES: [&str; 3] = [ + "idx_file_hashes", + "idx_posts_thread_id", + "idx_threads_board_sticky_bumped", +]; /// Board columns whose historical default was zero instead of the baseline value. const LEGACY_BOARD_ZERO_DEFAULT_COLUMNS: [&str; 4] = [ "allow_editing", @@ -678,6 +739,7 @@ fn install_baseline_schema_in_transaction(conn: &rusqlite::Connection) -> Result /// Create every table, index, search object, and invariant in the baseline. fn create_baseline_schema_objects(conn: &rusqlite::Connection) -> Result<()> { create_base_tables(conn)?; + conn.execute_batch(SUBMISSION_TOMBSTONE_SQL)?; create_indexes(conn)?; ensure_posts_search_index(conn)?; ensure_post_invariants(conn)?; @@ -930,7 +992,8 @@ fn can_repair_known_legacy_baseline_drift(expected: &SchemaShape, actual: &Schem fn schema_objects_are_legacy_repairable(expected: &SchemaShape, actual: &SchemaShape) -> bool { for (name, expected_object) in &expected.objects { let Some(actual_object) = actual.objects.get(name) else { - if ADDITIVE_BASELINE_INDEXES.contains(&name.as_str()) + if SUBMISSION_TOMBSTONE_OBJECTS.contains(&name.as_str()) + || ADDITIVE_BASELINE_INDEXES.contains(&name.as_str()) || (expected_object.kind == "trigger" && is_additive_domain_trigger(name)) { continue; @@ -978,6 +1041,8 @@ fn schema_objects_are_legacy_repairable(expected: &SchemaShape, actual: &SchemaS /// Return whether SQL exactly describes a removed redundant index. fn is_redundant_legacy_index(name: &str, sql: &str) -> bool { let expected = match name { + "idx_threads_board_sticky_bumped" => + "CREATE INDEX idx_threads_board_sticky_bumped ON threads(board_id, sticky DESC, bumped_at DESC)", "idx_file_hashes" => "CREATE INDEX idx_file_hashes ON file_hashes(sha256)", "idx_posts_thread_id" => "CREATE INDEX idx_posts_thread_id ON posts(thread_id)", _ => return false, @@ -996,6 +1061,9 @@ fn is_legacy_theme_sort_index(sql: &str) -> bool { fn tables_are_legacy_repairable(expected: &SchemaShape, actual: &SchemaShape) -> bool { for (table, expected_table) in &expected.tables { let Some(actual_table) = actual.tables.get(table) else { + if table == "post_submission_tombstones" { + continue; + } return false; }; let repairable = match table.as_str() { @@ -1020,7 +1088,8 @@ fn table_is_additive_index_repairable(expected: &TableShape, actual: &TableShape for (name, expected_index) in &expected.indexes { match actual.indexes.get(name) { Some(actual_index) if actual_index == expected_index => {} - None if ADDITIVE_BASELINE_INDEXES.contains(&name.as_str()) => {} + None if ADDITIVE_BASELINE_INDEXES.contains(&name.as_str()) + || SUBMISSION_TOMBSTONE_OBJECTS.contains(&name.as_str()) => {} Some(_) | None => return false, } } @@ -1077,9 +1146,11 @@ fn apply_additive_schema_repairs(conn: &rusqlite::Connection) -> Result<()> { /// Install additive indexes and domain triggers while the caller owns a transaction. fn apply_additive_schema_repairs_in_transaction(conn: &rusqlite::Connection) -> Result<()> { + conn.execute_batch(SUBMISSION_TOMBSTONE_SQL)?; conn.execute_batch( "DROP INDEX IF EXISTS idx_themes_enabled_sort; DROP INDEX IF EXISTS idx_file_hashes; + DROP INDEX IF EXISTS idx_threads_board_sticky_bumped; DROP INDEX IF EXISTS idx_posts_thread_id;", ) .context("Remove obsolete indexes failed")?; @@ -2484,7 +2555,8 @@ mod tests { conn.execute_batch(&format!("DROP INDEX {index}"))?; } conn.execute_batch( - "CREATE INDEX idx_file_hashes ON file_hashes(sha256); + "CREATE INDEX idx_threads_board_sticky_bumped ON threads(board_id, sticky DESC, bumped_at DESC); + CREATE INDEX idx_file_hashes ON file_hashes(sha256); CREATE INDEX idx_posts_thread_id ON posts(thread_id); CREATE TABLE schema_version (version TEXT NOT NULL PRIMARY KEY);", )?; @@ -3471,4 +3543,31 @@ mod tests { ); Ok(()) } + #[test] + fn submission_receipt_upgrade_preserves_live_tokens_and_deleted_history() -> Result<()> { + let conn = rusqlite::Connection::open_in_memory()?; + install_or_migrate_schema(&conn)?; + conn.execute_batch("INSERT INTO boards(id,short_name,name) VALUES(1,'life','Life'); + INSERT INTO threads(id,board_id) VALUES(1,1); + INSERT INTO posts(id,thread_id,board_id,body,body_html,deletion_token,is_op) VALUES(1,1,1,'body','body','token',1); + INSERT INTO post_submissions(submission_token,ip_hash,board_id,thread_id,post_id,is_thread) VALUES('receipt','actor',1,1,1,1); + DROP TRIGGER posts_submission_tombstone; + DROP TRIGGER threads_submission_tombstone; + DROP TRIGGER boards_submission_tombstone; + DROP INDEX idx_post_submissions_board; + DROP INDEX idx_post_submissions_thread; + DROP INDEX idx_post_submissions_post; + DROP TABLE post_submission_tombstones;")?; + install_or_migrate_schema(&conn)?; + let receipt = crate::db::get_post_submission(&conn, "receipt", "actor", 1)? + .context("preserved receipt")?; + anyhow::ensure!(receipt.post_id == 1); + crate::db::delete_thread(&conn, 1).map(|_cleanup| ())?; + anyhow::ensure!( + crate::db::get_post_submission(&conn, "receipt", "actor", 1)? == Some(receipt) + ); + verify_database_schema(&conn)?; + install_or_migrate_schema(&conn)?; + Ok(()) + } } diff --git a/src/db/search_budget.rs b/src/db/search_budget.rs new file mode 100644 index 00000000..d6476bd5 --- /dev/null +++ b/src/db/search_budget.rs @@ -0,0 +1,74 @@ +//! Cooperative SQLite work budgets for anonymous search count/sort operations. + +use crate::error::{AppError, Result}; +use rusqlite::{Connection, ErrorCode}; +use std::time::{Duration, Instant}; + +/// Progress checks are amortized over this many SQLite VM instructions. +const CHECK_INTERVAL: i32 = 10_000; +/// Approximate combined VM instruction budget for a count and result page. +const MAX_CHECKS: usize = 200; +/// Cooperative wall-clock allowance, checked at each VM progress checkpoint. +const SEARCH_DEADLINE: Duration = Duration::from_millis(500); + +/// Removes the callback before a pooled connection can serve other requests. +struct SearchBudget<'a> { + /// Connection whose callback belongs exclusively to the scoped search. + conn: &'a Connection, +} + +impl Drop for SearchBudget<'_> { + fn drop(&mut self) { + if let Err(error) = self.conn.progress_handler(0, None:: bool>) { + tracing::error!(%error, "could not clear SQLite search progress handler"); + } + } +} + +/// Run bounded search work and translate budget exhaustion into a useful client error. +pub(super) fn with_search_budget( + conn: &Connection, + work: impl FnOnce() -> Result, +) -> Result { + let started = Instant::now(); + let mut checks = 0usize; + conn.progress_handler( + CHECK_INTERVAL, + Some(move || { + checks = checks.saturating_add(1); + checks >= MAX_CHECKS || started.elapsed() >= SEARCH_DEADLINE + }), + )?; + let guard = SearchBudget { conn }; + let result = work(); + drop(guard); + match result { + Err(AppError::Internal(error)) if error.chain().any(|cause| matches!( + cause.downcast_ref::(), + Some(rusqlite::Error::SqliteFailure(code, _)) if code.code == ErrorCode::OperationInterrupted + )) => Err(AppError::BadRequest("This search is too broad. Please use more specific words.".into())), + other => other, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn expensive_queries_stop_and_pooled_connection_is_reusable() -> anyhow::Result<()> { + let conn = Connection::open_in_memory()?; + let result = with_search_budget(&conn, || { + conn.query_row("WITH RECURSIVE work(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM work WHERE n<10000000) SELECT SUM(n) FROM work", [], |row| row.get::<_, i64>(0)).map_err(AppError::from) + }); + anyhow::ensure!(matches!(result, Err(AppError::BadRequest(_)))); + let value: i64 = conn.query_row("SELECT 1", [], |row| row.get(0))?; + anyhow::ensure!(value == 1); + let count: i64 = with_search_budget(&conn, || { + conn.query_row("SELECT 7", [], |row| row.get(0)) + .map_err(AppError::from) + })?; + anyhow::ensure!(count == 7); + Ok(()) + } +} diff --git a/src/db/threads.rs b/src/db/threads.rs index 5dd525e0..41337f99 100644 --- a/src/db/threads.rs +++ b/src/db/threads.rs @@ -76,7 +76,7 @@ fn map_thread(row: &rusqlite::Row<'_>) -> rusqlite::Result { /// Collect all file paths (`file_path`, `thumb_path`, `audio_file_path`) for every /// post in the given set of thread ids. Returns a flat Vec of non-null paths. /// -/// Uses a single JOIN query instead of one query per thread. +/// Uses bounded batches instead of one query per thread. /// /// Call before deleting the thread rows; cascading deletion removes the posts /// that supply these paths. @@ -88,35 +88,37 @@ fn collect_thread_file_paths( return Ok(Vec::new()); } - // Build WHERE thread_id IN (?, ?, ...) dynamically. - let placeholders: String = thread_ids - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i.saturating_add(1))) - .collect::>() - .join(", "); - let sql = format!( - "SELECT file_path, thumb_path, audio_file_path + let mut paths = Vec::new(); + for batch in thread_ids.chunks(500) { + // Bound parameters so restored or backlogged boards cannot exceed SQLite limits. + let placeholders: String = batch + .iter() + .enumerate() + .map(|(i, _)| format!("?{}", i.saturating_add(1))) + .collect::>() + .join(", "); + let sql = format!( + "SELECT file_path, thumb_path, audio_file_path FROM posts WHERE thread_id IN ({placeholders})" - ); + ); - let mut stmt = conn.prepare(&sql)?; - let rows: Vec<(Option, Option, Option)> = stmt - .query_map(rusqlite::params_from_iter(thread_ids), |r| { - Ok((r.get(0)?, r.get(1)?, r.get(2)?)) - })? - .collect::>()?; + let mut stmt = conn.prepare(&sql)?; + let rows: Vec<(Option, Option, Option)> = stmt + .query_map(rusqlite::params_from_iter(batch), |r| { + Ok((r.get(0)?, r.get(1)?, r.get(2)?)) + })? + .collect::>()?; - let mut paths = Vec::new(); - for (f, t, a) in rows { - if let Some(p) = f { - paths.push(p); - } - if let Some(p) = t { - paths.push(p); - } - if let Some(p) = a { - paths.push(p); + for (f, t, a) in rows { + if let Some(p) = f { + paths.push(p); + } + if let Some(p) = t { + paths.push(p); + } + if let Some(p) = a { + paths.push(p); + } } } Ok(paths) @@ -154,19 +156,44 @@ pub fn get_threads_for_board( /// Paginate threads before computing reply aggregates; exclude missing OPs first. pub(super) fn thread_page_sql(archived: bool) -> String { + thread_listing_sql(archived, false) +} + +/// Build the shared page query, applying personal visibility before pagination. +fn thread_listing_sql(archived: bool, personalized: bool) -> String { // Preserve the legacy aggregate's tie order for each listing, including // deterministic boundaries when many replies bump in the same second. - let order = if archived { + let base_order = if archived { "t.bumped_at DESC, t.id ASC" } else { "t.sticky DESC, t.bumped_at DESC, t.id DESC" }; + let order = if personalized && !archived { + format!("t.viewer_pinned DESC, {base_order}") + } else { + base_order.to_owned() + }; + let page_order = if personalized && !archived { + format!("COALESCE(pref.pinned, 0) DESC, {base_order}") + } else { + base_order.to_owned() + }; + let (columns, join, visibility) = if personalized { + ( + ", COALESCE(pref.pinned, 0) AS viewer_pinned", + "LEFT JOIN user_thread_preferences pref ON pref.thread_id = t.id AND pref.user_hash = ?5", + "AND COALESCE(pref.hidden, 0) = ?6", + ) + } else { + ("", "", "") + }; format!( "WITH page AS MATERIALIZED ( - SELECT t.* FROM threads t + SELECT t.*{columns} FROM threads t {join} WHERE t.board_id = ?1 AND t.archived = ?4 + {visibility} AND EXISTS (SELECT 1 FROM posts op WHERE op.thread_id=t.id AND op.is_op=1) - ORDER BY {order} LIMIT ?2 OFFSET ?3 + ORDER BY {page_order} LIMIT ?2 OFFSET ?3 ) SELECT t.id, t.board_id, t.subject, t.created_at, t.bumped_at, t.locked, t.sticky, t.reply_count, @@ -179,6 +206,52 @@ pub(super) fn thread_page_sql(archived: bool) -> String { ) } +/// List a visitor's visible or hidden active threads in canonical personal order. +/// +/// Personal pins precede sticky threads, then bump time and descending thread ID. +/// Filtering and sorting occur before LIMIT/OFFSET, so pages never repeat pins. +/// +/// # Errors +/// Returns an error if the indexed listing cannot be read. +pub fn get_threads_for_viewer( + conn: &rusqlite::Connection, + board_id: i64, + user_hash: &str, + hidden: bool, + limit: i64, + offset: i64, +) -> Result> { + let sql = thread_listing_sql(false, true); + let mut stmt = conn.prepare_cached(&sql)?; + let threads = stmt + .query_map( + params![board_id, limit, offset, 0_i32, user_hash, i32::from(hidden)], + map_thread, + )? + .collect::>>()?; + Ok(threads) +} + +/// Count active, renderable threads with the same visibility filter as listing. +/// +/// # Errors +/// Returns an error if the count cannot be read. +pub fn count_threads_for_viewer( + conn: &rusqlite::Connection, + board_id: i64, + user_hash: &str, + hidden: bool, +) -> Result { + Ok(conn.query_row( + "SELECT COUNT(*) FROM threads t + LEFT JOIN user_thread_preferences pref ON pref.thread_id = t.id AND pref.user_hash = ?2 + WHERE t.board_id = ?1 AND t.archived = 0 AND COALESCE(pref.hidden, 0) = ?3 + AND EXISTS (SELECT 1 FROM posts op WHERE op.thread_id = t.id AND op.is_op = 1)", + params![board_id, user_hash, i32::from(hidden)], + |row| row.get(0), + )?) +} + /// Execute an indexed page and its bounded per-thread aggregate probes. fn load_thread_page( conn: &rusqlite::Connection, @@ -247,7 +320,7 @@ pub fn get_thread(conn: &rusqlite::Connection, thread_id: i64) -> Result { /// The transaction created and committed a new post bundle. - Created(T), + Created(T, i64), /// The token already names a canonical committed post. Replayed(super::PostSubmissionRecord), } @@ -263,6 +336,8 @@ pub(crate) struct PostFilesystemCommit<'a, F = fn(&rusqlite::Connection) -> Resu schedule_thread_prune: bool, /// Posting policy checked after token replay and while holding the write lock. validate: F, + /// Board name for media work that must persist alongside the new post. + media_job_board: Option<&'a str>, } impl<'a> PostFilesystemCommit<'a> { @@ -289,16 +364,114 @@ impl<'a, F> PostFilesystemCommit<'a, F> { deduplicated_paths: &'a [&'a str], schedule_thread_prune: bool, validate: F, + ) -> Self { + Self::new_with_media_validation( + pending_fs_op, + deduplicated_paths, + schedule_thread_prune, + None, + validate, + ) + } + + /// Include required media work in the atomic creation bundle. + pub(crate) const fn new_with_media_validation( + pending_fs_op: Option<&'a crate::pending_fs::PendingFsOpInsert>, + deduplicated_paths: &'a [&'a str], + schedule_thread_prune: bool, + media_job_board: Option<&'a str>, + validate: F, ) -> Self { Self { pending_fs_op, deduplicated_paths, schedule_thread_prune, validate, + media_job_board, } } } +/// Persist media intent before commit; a full queue leaves an explicit failure. +fn persist_new_post_metadata( + conn: &rusqlite::Connection, + post: &super::NewPost, + post_id: i64, + media_board: Option<&str>, +) -> Result { + if let Some(board_short) = media_board { + let source = post + .file_path + .as_ref() + .context("pending media requires a source")?; + let (job_type, job_tag) = match post.media_type.as_deref() { + Some("video") => ("video_transcode", "VideoTranscode"), + Some("audio") => ("audio_waveform", "AudioWaveform"), + _ => anyhow::bail!("pending processing requires audio or video"), + }; + let capacity = crate::config::CONFIG.job_queue_capacity; + let full = capacity > 0 && u64::try_from(super::pending_job_count(conn)?)? >= capacity; + if full { + super::set_post_media_processing_state( + conn, + post_id, + Some(super::MEDIA_PROCESSING_FAILED), + Some("Background media queue is full; deferred processing was skipped."), + )?; + } else { + let payload = serde_json::json!({ + "t": job_tag, "d": { "post_id": post_id, "file_path": source, "board_short": board_short } + }).to_string(); + super::enqueue_job(conn, job_type, &payload).map(|_job_id| ())?; + super::set_post_media_processing_state( + conn, + post_id, + Some(super::MEDIA_PROCESSING_PENDING), + None, + )?; + } + } + Ok(conn.query_row( + "SELECT created_at FROM posts WHERE id = ?1", + [post_id], + |row| row.get(0), + )?) +} + +/// Reject reuse of a receipt for another operation or reply target. +pub(crate) fn validate_submission_target( + record: super::PostSubmissionRecord, + thread_id: Option, +) -> Result<()> { + if record.is_thread != thread_id.is_none() + || thread_id.is_some_and(|target| target != record.thread_id) + { + return Err(crate::error::AppError::Conflict( + "This submission token belongs to a different posting form. Reload the page.".into(), + ) + .into()); + } + Ok(()) +} + +/// A token accepted for another actor/board must never create new content. +fn reject_conflicting_submission(conn: &rusqlite::Connection, token: &str) -> Result<()> { + if !token.is_empty() + && conn.query_row( + "SELECT EXISTS(SELECT 1 FROM post_submissions WHERE submission_token=?1) + OR EXISTS(SELECT 1 FROM post_submission_tombstones WHERE submission_token=?1)", + [token], + |row| row.get::<_, bool>(0), + )? + { + return Err(crate::error::AppError::Conflict( + "This submission token has already been used. Reload the posting form.".into(), + ) + .into()); + } + Ok(()) +} + /// Create a thread, its OP post, and an optional poll atomically. /// /// # Errors @@ -321,7 +494,7 @@ pub fn create_thread_with_optional_poll( poll, PostFilesystemCommit::new(pending_fs_op, &[], false), )? { - PostCreationOutcome::Created(ids) => Ok(ids), + PostCreationOutcome::Created(ids, _) => Ok(ids), PostCreationOutcome::Replayed(existing) => Ok((existing.thread_id, existing.post_id, None)), } } @@ -355,10 +528,12 @@ pub(crate) fn create_thread_submission( if let Some(existing) = super::posts::get_post_submission(conn, submission_token, ip_hash, board_id)? { + validate_submission_target(existing, None)?; return Ok(PostCreationOutcome::Replayed(existing)); } } + reject_conflicting_submission(conn, submission_token)?; (filesystem.validate)(conn)?; validate_deduplicated_paths(conn, filesystem.deduplicated_paths)?; @@ -406,7 +581,16 @@ pub(crate) fn create_thread_submission( .map(|_operation_summary| ())?; } - Ok(PostCreationOutcome::Created((thread_id, post_id, poll_id))) + let created_at = persist_new_post_metadata( + conn, + &post_with_thread, + post_id, + filesystem.media_job_board, + )?; + Ok(PostCreationOutcome::Created( + (thread_id, post_id, poll_id), + created_at, + )) })(); match result { @@ -443,7 +627,7 @@ pub fn create_reply_with_thread_update( should_bump, PostFilesystemCommit::new(pending_fs_op, &[], false), )? { - PostCreationOutcome::Created(post_id) => Ok(post_id), + PostCreationOutcome::Created(post_id, _) => Ok(post_id), PostCreationOutcome::Replayed(existing) => Ok(existing.post_id), } } @@ -468,39 +652,22 @@ pub(crate) fn create_reply_submission( if let Some(existing) = super::posts::get_post_submission(conn, submission_token, ip_hash, post.board_id)? { + validate_submission_target(existing, Some(post.thread_id))?; return Ok(PostCreationOutcome::Replayed(existing)); } } + if post.is_op { + return Err(crate::error::AppError::BadRequest( + "A reply cannot be a thread starter.".into(), + ) + .into()); + } + reject_conflicting_submission(conn, submission_token)?; (filesystem.validate)(conn)?; validate_deduplicated_paths(conn, filesystem.deduplicated_paths)?; - let flags = conn - .query_row( - "SELECT locked, archived - FROM threads - WHERE id = ?1 AND board_id = ?2", - params![post.thread_id, post.board_id], - |row| { - Ok(( - row.get::<_, i32>(0)? != 0_i32, - row.get::<_, i32>(1)? != 0_i32, - )) - }, - ) - .optional()?; - let Some((locked, archived)) = flags else { - anyhow::bail!( - "Thread id {} not found while creating reply", - post.thread_id - ); - }; - if locked { - return Err(anyhow::Error::new(ThreadClosed::Locked)); - } - if archived { - return Err(anyhow::Error::new(ThreadClosed::Archived)); - } + ensure_replyable(conn, post.board_id, post.thread_id)?; let post_id = super::posts::create_post_inner(conn, post)?; let updated = if should_bump { @@ -541,7 +708,9 @@ pub(crate) fn create_reply_submission( false, )?; } - Ok(PostCreationOutcome::Created(post_id)) + let created_at = + persist_new_post_metadata(conn, post, post_id, filesystem.media_job_board)?; + Ok(PostCreationOutcome::Created(post_id, created_at)) })(); match result { @@ -559,6 +728,38 @@ pub(crate) fn create_reply_submission( } } +/// Authoritative current parent-state check, performed under the posting write lock. +fn ensure_replyable(conn: &rusqlite::Connection, board_id: i64, thread_id: i64) -> Result<()> { + let flags = conn + .query_row( + "SELECT locked, archived + FROM threads + WHERE id = ?1 AND board_id = ?2", + params![thread_id, board_id], + |row| { + Ok(( + row.get::<_, i32>(0)? != 0_i32, + row.get::<_, i32>(1)? != 0_i32, + )) + }, + ) + .optional()?; + let Some((locked, archived)) = flags else { + return Err(crate::error::AppError::NotFound( + "Thread not found. It may have been deleted or pruned.".into(), + ) + .into()); + }; + if archived { + return Err(anyhow::Error::new(ThreadClosed::Archived)); + } + if locked { + return Err(anyhow::Error::new(ThreadClosed::Locked)); + } + + Ok(()) +} + /// Revalidate a prior dedup cache hit after the post transaction owns the write lock. fn validate_deduplicated_paths(conn: &rusqlite::Connection, paths: &[&str]) -> Result<()> { for path in paths { @@ -710,243 +911,125 @@ pub fn delete_thread( } // Archive / prune -/// Archive oldest non-sticky threads that exceed the board's `max_threads` limit. -/// -/// Archived threads are locked and marked read-only; their content remains -/// accessible via `/{board}/archive`. Returns the count of threads archived -/// (no file deletion occurs). +/// Archive overflow non-sticky active threads, keeping the newest bump/ID pairs. /// -/// ID selection and the bulk update share a transaction so a concurrent bump -/// cannot change the ordering between those operations. -/// -/// Note: LIMIT -1 OFFSET ? is a SQLite-specific idiom for "skip the first -/// max rows, return everything else". It is not standard SQL. The LIMIT -1 -/// means "no upper bound on the result set after the offset is applied". +/// Archiving locks threads and preserves all posts and media references. +/// Sticky active threads do not count against the limit; locked threads do. /// /// # Errors -/// Returns an error if the database operation fails. +/// Returns an error for invalid limits or a failed transaction. pub fn archive_old_threads(conn: &rusqlite::Connection, board_id: i64, max: i64) -> Result { - conn.execute_batch("BEGIN IMMEDIATE") - .context("Failed to begin archive_old_threads transaction")?; - - let result: Result = (|| { - // Collect inside the transaction to prevent races with concurrent bumps. - let ids: Vec = { - let mut stmt = conn.prepare_cached( - "SELECT id FROM threads - WHERE board_id = ?1 AND sticky = 0 AND archived = 0 - ORDER BY bumped_at DESC LIMIT -1 OFFSET ?2", - )?; - // Bind `collected` explicitly so `stmt` is dropped before the - // block ends — the MappedRows iterator borrows `stmt`, and the - // compiler requires the borrow to end before the binding goes out - // of scope at the closing `}`. - let collected = stmt - .query_map(params![board_id, max], |r| r.get(0))? - .collect::>>()?; - collected - }; - - let count = ids.len(); - if count == 0 { - return Ok(0); - } - - // Single bulk UPDATE instead of N individual statements. - let placeholders: String = ids - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i.saturating_add(1))) - .collect::>() - .join(", "); - let sql = - format!("UPDATE threads SET archived = 1, locked = 1 WHERE id IN ({placeholders})"); - conn.execute(&sql, rusqlite::params_from_iter(&ids)) - .context("Failed to bulk archive threads") - .map(|_affected_rows| ())?; - - Ok(count) - })(); + let tx = rusqlite::Transaction::new_unchecked(conn, rusqlite::TransactionBehavior::Immediate)?; + let count = archive_old_threads_in_tx(&tx, board_id, max)?; + tx.commit()?; + Ok(count) +} - match result { - Ok(0) => { - // Nothing to archive — roll back the (empty) transaction cleanly. - drop(conn.execute_batch("ROLLBACK")); - Ok(0) - } - Ok(count) => { - super::commit_transaction(conn, "Failed to commit archive_old_threads transaction")?; - Ok(count) - } - Err(e) => { - drop(conn.execute_batch("ROLLBACK")); - Err(e) - } - } +/// Archive overflow while the caller holds the SQLite write transaction. +pub(crate) fn archive_old_threads_in_tx( + conn: &rusqlite::Transaction<'_>, + board_id: i64, + max: i64, +) -> Result { + validate_retention_limit(max, false)?; + // A subquery avoids building an unbounded parameter list for backlog recovery. + conn.execute( + "UPDATE threads SET archived = 1, locked = 1 + WHERE id IN ( + SELECT id FROM threads + WHERE board_id = ?1 AND sticky = 0 AND archived = 0 + ORDER BY bumped_at DESC, id DESC LIMIT -1 OFFSET ?2 + )", + params![board_id, max], + ) + .context("Failed to archive overflow threads") } -/// Hard-delete oldest non-sticky, non-archived threads that exceed `max_threads`. -/// Used when a board has archiving disabled — threads are permanently removed. +/// Hard-delete overflow non-sticky active threads when archiving is disabled. /// -/// Returns the on-disk paths that are now safe to delete (i.e. no longer -/// referenced by any remaining post after the prune). The caller is responsible -/// for actually removing these files from disk. -/// -/// ID selection, bulk deletion, and the final path-reference check share a -/// transaction so a concurrent insert cannot make a returned path live again. -/// File paths are collected with one joined query. -/// -/// Note: LIMIT -1 OFFSET ? is a SQLite-specific idiom — see `archive_old_threads`. +/// Returns safe file paths and a durable cleanup intent for filesystem replay. /// /// # Errors -/// Returns an error if the database operation fails. +/// Returns an error for invalid limits or a failed transaction. pub fn prune_old_threads( conn: &rusqlite::Connection, board_id: i64, max: i64, ) -> Result { - conn.execute_batch("BEGIN IMMEDIATE") - .context("Failed to begin prune_old_threads transaction")?; - - let result: Result = (|| { - // Collect ids inside the transaction to prevent concurrent bumps from - // changing the ordering between the SELECT and the DELETE. - let ids: Vec = { - let mut stmt = conn.prepare_cached( - "SELECT id FROM threads - WHERE board_id = ?1 AND sticky = 0 AND archived = 0 - ORDER BY bumped_at DESC LIMIT -1 OFFSET ?2", - )?; - let collected = stmt - .query_map(params![board_id, max], |r| r.get(0))? - .collect::>>()?; - collected - }; - - if ids.is_empty() { - return Ok(crate::db::DeletePathsResult { - paths: Vec::new(), - pending_fs_op_id: None, - }); - } - - // Collect all file paths in a single query BEFORE the DELETEs. - let candidates = collect_thread_file_paths(conn, &ids)?; - - // Single bulk DELETE instead of N individual statements. - let placeholders: String = ids - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i.saturating_add(1))) - .collect::>() - .join(", "); - let sql = format!("DELETE FROM threads WHERE id IN ({placeholders})"); - conn.execute(&sql, rusqlite::params_from_iter(&ids)) - .context("Failed to bulk delete pruned threads") - .map(|_affected_rows| ())?; - - // Determine safe paths INSIDE the transaction so the check sees the - // post-delete state before any concurrent writer can insert new references. - let safe = super::paths_safe_to_delete(conn, candidates)?; - let pending_fs_op = super::build_delete_files_pending_op(&safe)?; - if let Some(op) = pending_fs_op.as_ref() { - super::insert_pending_fs_op(conn, op)?; - } - Ok(crate::db::DeletePathsResult { - paths: safe, - pending_fs_op_id: pending_fs_op.map(|op| op.id), - }) - })(); - - match result { - Ok(result) => { - super::commit_transaction(conn, "Failed to commit prune_old_threads transaction")?; - Ok(result) - } - Err(e) => { - drop(conn.execute_batch("ROLLBACK")); - Err(e) - } - } + let tx = rusqlite::Transaction::new_unchecked(conn, rusqlite::TransactionBehavior::Immediate)?; + let result = prune_old_threads_in_tx(&tx, board_id, max, false)?; + tx.commit()?; + Ok(result) } -/// Hard-delete oldest archived threads that exceed the archive retention cap. +/// Hard-delete archived overflow, keeping the first entries in archive order. /// -/// Returns the on-disk paths that are now safe to remove. As with live-thread -/// pruning, the caller is responsible for deleting those files from disk. -/// -/// The ordering uses `bumped_at DESC`, matching the archive page and ensuring -/// we keep the most recently-active archived threads. +/// Retention uses bump time, not time of archival. Archived sticky threads are +/// subject to this cap too. File cleanup is durably scheduled before commit. /// /// # Errors -/// Returns an error if the transaction cannot be opened or committed, if the -/// candidate threads cannot be queried, or if the bulk delete/safe-path -/// calculation fails. +/// Returns an error for invalid limits or a failed transaction. pub fn prune_old_archived_threads( conn: &rusqlite::Connection, board_id: i64, max: i64, ) -> Result { - conn.execute_batch("BEGIN IMMEDIATE") - .context("Failed to begin prune_old_archived_threads transaction")?; - - let result: Result = (|| { - let ids: Vec = { - let mut stmt = conn.prepare_cached( - "SELECT id FROM threads - WHERE board_id = ?1 AND archived = 1 - ORDER BY bumped_at DESC LIMIT -1 OFFSET ?2", - )?; - let collected = stmt - .query_map(params![board_id, max], |r| r.get(0))? - .collect::>>()?; - collected - }; - - if ids.is_empty() { - return Ok(crate::db::DeletePathsResult { - paths: Vec::new(), - pending_fs_op_id: None, - }); - } - - let candidates = collect_thread_file_paths(conn, &ids)?; - - let placeholders: String = ids - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i.saturating_add(1))) - .collect::>() - .join(", "); - let sql = format!("DELETE FROM threads WHERE id IN ({placeholders})"); - conn.execute(&sql, rusqlite::params_from_iter(&ids)) - .context("Failed to bulk delete archived threads") - .map(|_affected_rows| ())?; + let tx = rusqlite::Transaction::new_unchecked(conn, rusqlite::TransactionBehavior::Immediate)?; + let result = prune_old_threads_in_tx(&tx, board_id, max, true)?; + tx.commit()?; + Ok(result) +} - let safe = super::paths_safe_to_delete(conn, candidates)?; - let pending_fs_op = super::build_delete_files_pending_op(&safe)?; - if let Some(op) = pending_fs_op.as_ref() { - super::insert_pending_fs_op(conn, op)?; - } - Ok(crate::db::DeletePathsResult { - paths: safe, - pending_fs_op_id: pending_fs_op.map(|op| op.id), - }) - })(); +/// Reject invalid destructive limits even when called outside the worker. +fn validate_retention_limit(max: i64, archived: bool) -> Result<()> { + let upper = if archived { 10_000 } else { 1_000 }; + anyhow::ensure!( + (1..=upper).contains(&max), + "invalid retention limit {max}; expected 1..={upper}; preserving content" + ); + Ok(()) +} - match result { - Ok(result) => { - super::commit_transaction( - conn, - "Failed to commit prune_old_archived_threads transaction", - )?; - Ok(result) - } - Err(e) => { - drop(conn.execute_batch("ROLLBACK")); - Err(e) - } +/// Select, delete, reference-check and journal files in the caller's transaction. +pub(crate) fn prune_old_threads_in_tx( + conn: &rusqlite::Transaction<'_>, + board_id: i64, + max: i64, + archived: bool, +) -> Result { + validate_retention_limit(max, archived)?; + // Archive ties historically ascend by ID. Active ties descend by ID, exactly + // matching the board index. Never let query planner choice decide retention. + let sql = if archived { + "SELECT id FROM threads WHERE board_id = ?1 AND archived = 1 + ORDER BY bumped_at DESC, id ASC LIMIT -1 OFFSET ?2" + } else { + "SELECT id FROM threads WHERE board_id = ?1 AND sticky = 0 AND archived = 0 + ORDER BY bumped_at DESC, id DESC LIMIT -1 OFFSET ?2" + }; + let ids = conn + .prepare_cached(sql)? + .query_map(params![board_id, max], |row| row.get::<_, i64>(0))? + .collect::>>()?; + let candidates = collect_thread_file_paths(conn, &ids)?; + for batch in ids.chunks(500) { + let placeholders = vec!["?"; batch.len()].join(", "); + conn.execute( + &format!("DELETE FROM threads WHERE id IN ({placeholders})"), + rusqlite::params_from_iter(batch), + ) + .context("Failed to delete overflow threads") + .map(|_affected_rows| ())?; } + let safe = super::paths_safe_to_delete(conn, candidates)?; + let pending_fs_op = super::build_delete_files_pending_op(&safe)?; + if let Some(op) = pending_fs_op.as_ref() { + super::insert_pending_fs_op(conn, op)?; + } + Ok(crate::db::DeletePathsResult { + paths: safe, + pending_fs_op_id: pending_fs_op.map(|op| op.id), + }) } // Archive listing @@ -1707,3 +1790,10 @@ mod tests { Ok(()) } } + +#[cfg(test)] +mod archive_tests; + +#[cfg(test)] +/// Full posting state-machine and WAL concurrency regressions. +mod lifecycle_tests; diff --git a/src/db/threads/archive_tests.rs b/src/db/threads/archive_tests.rs new file mode 100644 index 00000000..687268f7 --- /dev/null +++ b/src/db/threads/archive_tests.rs @@ -0,0 +1,574 @@ +//! Archive lifecycle regressions using real schema, transactions and file-backed WAL. + +use super::*; +use anyhow::ensure; +use rusqlite::Connection; + +fn fixture(conn: &Connection) -> Result { + crate::db::schema::install_or_migrate_schema(conn)?; + crate::db::create_board(conn, "archive", "Archive", "", false) +} + +fn thread(conn: &Connection, board: i64, bump: i64) -> Result { + let id = conn.query_row( + "INSERT INTO threads(board_id, subject, bumped_at) VALUES(?1, 'café ").to_owned(), + file_name: Some(concat!("unsafe\".webp").to_owned()), + body_html: crate::utils::sanitize::render_post_body( + &crate::utils::sanitize::escape_html(concat!( + ">quote\n>>1 [spoiler]hidden[/spoiler] alert(1)" + )), + false, + ), + audio_file_path: Some("test/companion.opus".to_owned()), + audio_file_name: Some("companion.opus".to_owned()), + audio_mime_type: Some("audio/ogg".to_owned()), + ..sample_post() + }; + let render = |archived| { + render_post( + &post, + "test", + "csrf", + RenderPostOpts { + show_delete: false, + is_admin: false, + admin_csrf_token: None, + show_media: true, + allow_editing: false, + allow_self_delete: false, + owned_post_controls: None, + show_poster_ids: false, + collapse_greentext: false, + thread_state: Some((false, archived, archived)), + thread_op_id: Some(1), + video_audio_muted: false, + }, + 0, + ) + }; + let active = render(false); + let archived = render(true); + assert_eq!( + active, archived, + "archive state must preserve reply media and markup" + ); + assert!( + !archived.contains(""), + false, + ); + assert!( + valid.contains("data-pid=\"12\""), + "leading zeros must resolve to a canonical post ID" + ); + assert!( + valid.contains("data-pid=\"9223372036854775807\""), + "largest supported post ID must remain representable" + ); + assert!( + !valid.contains("