diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c2ec06..ae1026b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,10 @@ concurrency: permissions: contents: read +env: + # cargo-generate-rpm 0.21.0 has prebuilts for both x86_64 and aarch64. + PACKAGING_TOOLS: cargo-deb@3.8.0,cargo-generate-rpm@0.21.0 + jobs: clippy_check: name: Run clippy check @@ -34,15 +38,10 @@ jobs: components: clippy - name: Cache cargo + target - uses: actions/cache@v4 + uses: Swatinem/rust-cache@v2 with: - key: clippy-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - clippy-${{ runner.os }}- - path: | - ~/.cargo/registry - ~/.cargo/git - ./target + workspaces: . -> target + key: clippy # Frontend dist/ must exist for the server's include_dir! to compile. # Reuses the artifact produced by the `build_frontend` job. @@ -58,6 +57,7 @@ jobs: rustfmt: name: Check style + needs: [image_meta] runs-on: ubuntu-latest steps: - name: Checkout @@ -69,17 +69,37 @@ jobs: components: rustfmt - name: Run cargo fmt - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check + run: cargo fmt --all -- --check image_meta: - name: Compute image tag + name: Validate version and compute image tag runs-on: ubuntu-latest outputs: image_tag: ${{ steps.vars.outputs.image_tag }} image_name: ${{ steps.vars.outputs.image_name }} + package_version: ${{ steps.version.outputs.package_version }} steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Validate release version + id: version + run: | + python3 - <<'PY' + import os + import tomllib + + with open("privaxy/Cargo.toml", "rb") as manifest: + version = tomllib.load(manifest)["package"]["version"] + ref = os.environ["GITHUB_REF"] + if ref.startswith("refs/tags/v"): + tag_version = ref.removeprefix("refs/tags/v") + if tag_version != version: + print(f"::error::Release tag v{tag_version} does not match privaxy/Cargo.toml version {version}") + raise SystemExit(1) + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + print(f"package_version={version}", file=output) + PY + - name: Get image tag id: vars run: | @@ -135,6 +155,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Log in to GitHub Container Registry + if: github.event_name == 'push' uses: docker/login-action@v3 with: registry: ghcr.io @@ -147,7 +168,7 @@ jobs: with: images: ${{ needs.image_meta.outputs.image_name }} - - name: Build and push by digest + - name: Build image id: build uses: docker/build-push-action@v6 with: @@ -161,15 +182,18 @@ jobs: annotations: ${{ steps.meta.outputs.annotations }} cache-from: type=gha,scope=${{ matrix.cache_scope }} cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }} - outputs: type=image,name=${{ needs.image_meta.outputs.image_name }},push-by-digest=true,name-canonical=true,push=true + # PRs validate the image build without publishing to the registry. + outputs: ${{ github.event_name == 'push' && format('type=image,name={0},push-by-digest=true,name-canonical=true,push=true', needs.image_meta.outputs.image_name) || 'type=cacheonly' }} - name: Export digest + if: github.event_name == 'push' run: | mkdir -p /tmp/digests digest="${{ steps.build.outputs.digest }}" touch "/tmp/digests/${digest#sha256:}" - name: Upload digest + if: github.event_name == 'push' uses: actions/upload-artifact@v4 with: name: image-digest-${{ matrix.cache_scope }} @@ -180,6 +204,7 @@ jobs: image_merge: name: Merge multi-arch manifest needs: [image_meta, image_build] + if: github.event_name == 'push' runs-on: ubuntu-latest permissions: contents: read @@ -237,7 +262,7 @@ jobs: - name: Cache cargo + wasm target uses: Swatinem/rust-cache@v2 with: - workspaces: web_frontend -> target + workspaces: . -> target shared-key: build-frontend - name: Install node @@ -315,19 +340,17 @@ jobs: - name: Checkout uses: actions/checkout@v6 - - name: Cache build artifacts - uses: actions/cache@v3 - with: - key: ${{ matrix.os }}-${{ matrix.target }}-artifacts - path: | - ./target - ~/.cargo - - name: Install rust uses: dtolnay/rust-toolchain@stable with: targets: ${{ matrix.target }} + - name: Cache build artifacts + uses: Swatinem/rust-cache@v2 + with: + workspaces: . -> target + key: native-${{ matrix.target }} + - name: Install cross build dependencies if: matrix.target == 'aarch64-unknown-linux-gnu' run: sudo apt-get update && sudo apt-get install -y gcc-aarch64-linux-gnu libc6-dev-arm64-cross @@ -352,14 +375,21 @@ jobs: CC_x86_64_unknown_linux_musl: musl-gcc CC_aarch64_unknown_linux_musl: musl-gcc run: cargo build --release --target ${{ matrix.target }} --bin privaxy --target-dir target + + - name: Install packaging tools + if: ${{ !endsWith(matrix.target, '-musl') }} + uses: taiki-e/install-action@v2 + with: + tool: ${{ env.PACKAGING_TOOLS }} + - name: Build packages if: ${{ !endsWith(matrix.target, '-musl') }} run: | # target/ is cached; discard packages from previous versions. rm -f target/${{ matrix.target }}/release/privaxy_*.deb \ target/${{ matrix.target }}/release/privaxy-*.rpm - cargo install cargo-deb && cargo deb -p privaxy --target ${{ matrix.target }} -o target/${{ matrix.target }}/release - cargo install cargo-generate-rpm && cargo generate-rpm -p privaxy --target ${{ matrix.target }} -o target/${{ matrix.target }}/release + cargo deb --no-build -p privaxy --target ${{ matrix.target }} -o target/${{ matrix.target }}/release + cargo generate-rpm -p privaxy --target ${{ matrix.target }} -o target/${{ matrix.target }}/release - uses: actions/upload-artifact@v4 if: ${{ !endsWith(matrix.target, '-musl') }} with: @@ -401,17 +431,15 @@ jobs: - name: Checkout uses: actions/checkout@v6 - - name: Cache build artifacts - uses: actions/cache@v3 - with: - key: ${{ matrix.os }}-${{ matrix.target }}-artifacts - path: | - ./target - ~/.cargo - - name: Install rust uses: dtolnay/rust-toolchain@nightly + - name: Cache build artifacts + uses: Swatinem/rust-cache@v2 + with: + workspaces: . -> target + key: mips-${{ matrix.target }}-${{ hashFiles('Cross.toml') }} + - name: Install cross uses: taiki-e/install-action@v2 with: @@ -432,6 +460,12 @@ jobs: - name: Build server run: cross build --release --target ${{ matrix.target }} --bin privaxy --target-dir target + - name: Install packaging tools + if: ${{ endsWith(matrix.target, '-linux-gnu') }} + uses: taiki-e/install-action@v2 + with: + tool: ${{ env.PACKAGING_TOOLS }} + - name: Build packages if: ${{ endsWith(matrix.target, '-linux-gnu') }} run: | @@ -439,8 +473,8 @@ jobs: rm -f target/${{ matrix.target }}/release/privaxy_*.deb \ target/${{ matrix.target }}/release/privaxy-*.rpm variant=$(echo "${{ matrix.target }}" | cut -d- -f1) - cargo install cargo-deb && cargo deb --no-build --no-strip --variant "$variant" -p privaxy --target ${{ matrix.target }} -o target/${{ matrix.target }}/release - cargo install cargo-generate-rpm && cargo generate-rpm -p privaxy --target ${{ matrix.target }} -o target/${{ matrix.target }}/release + cargo deb --no-build --no-strip --variant "$variant" -p privaxy --target ${{ matrix.target }} -o target/${{ matrix.target }}/release + cargo generate-rpm -p privaxy --target ${{ matrix.target }} -o target/${{ matrix.target }}/release - uses: actions/upload-artifact@v4 if: ${{ endsWith(matrix.target, '-linux-gnu') }} @@ -462,7 +496,7 @@ jobs: release: name: Create GitHub Release - needs: [ci, ci_mips, image_merge] + needs: [image_meta, ci, ci_mips, image_merge] runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/v') permissions: @@ -477,10 +511,9 @@ jobs: path: ./artifacts pattern: privaxy-* merge-multiple: false - - name: Set package version from tag - run: | - echo "PKG_VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV - name: Flatten artifacts for release + env: + PKG_VERSION: ${{ needs.image_meta.outputs.package_version }} run: | mkdir ./release-assets for dir in ./artifacts/privaxy-*/; do diff --git a/.gitignore b/.gitignore index 8ca8437..1da5405 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,7 @@ filters/__pycache__ node_modules .vscode **/target/* -testing/ \ No newline at end of file +testing/ + +# Local opt-in request history can contain browsing metadata. +statistics.redb diff --git a/CHANGELOG.md b/CHANGELOG.md index c09ce61..f4bbf8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,86 @@ ## Unreleased +- Add a shared footer with the running server version linked to the project. + Check for stable releases automatically when opened and hourly thereafter, + showing an "Update available" release link only when a newer version is found. + Update checks are authenticated and time-limited, with successful checks cached + for one hour and failures for one minute. +- Added opt-in persistent statistics and individual request history using redb. + History includes time ranges, trends, top domains/clients, and pagination, with + privacy levels that remove identifiers before collection. URL paths, query + strings, headers, and bodies are never saved in statistics. +- Added Statistics settings for retention, a request-count limit, clearing saved + history, and excluding domain patterns from both statistics and the live + Requests log. Exclusions do not change filtering behavior and work even when + persistence is disabled. +- Bounded statistics buffering, batched transactional writes, visible storage + errors, and shutdown flushing keep storage failures off the proxy request path. + Existing installations retain memory-only statistics until explicitly enabled. +- Show saved statistics directly on the Dashboard when persistence is enabled. + With persistence off or Anonymous mode selected, keep the original live, + per-process dashboard. Remove the separate History tab and redirect old + `/history` bookmarks to `/`. Both layouts retain the blocking toggle and CA + certificate download. +- Live dashboard top blocked entries now use domains and select the top 50 after + sorting; client tracking is bounded. +- Count each modified HTML response once when both head scripts and cosmetic + styles are injected, instead of counting the two injection stages separately. +- Use TLS ClientHello SNI for IP-address CONNECT requests when choosing + interception certificates, applying hostname inclusions and exclusions, and + verifying upstream TLS. HTTP requests and WebSocket upgrades still connect to + the original CONNECT IP and port. Named CONNECT requests retain their existing + identity; DoH redirects and userscript fetches keep their own routing. +- Tunnel CONNECT traffic to port 853 without interception, preserving end-to-end + DNS-over-TLS. The DoH policy continues to apply only to HTTP traffic. Explicit + CONNECT-host/IP exclusions bypass protocol probing. +- Bound CONNECT protocol detection, ClientHello reading, and intercepted TLS + handshakes to 10 seconds each, with a 64 KiB ClientHello limit. Reject malformed, + truncated, oversized, or stalled handshakes while preserving fragmented TLS + records and buffered bytes, including when SNI selects a bypass tunnel. +- Increase the upstream HTTP/2 response header limit from 16 KiB to 64 KiB and + allow 64 KiB of headers on intercepted HTTP/2 requests. This fixes 502 stream + protocol errors from responses with large CSP or Set-Cookie headers. +- Advertise HTTP/2 and HTTP/1.1 through ALPN on intercepted TLS connections so + browsers can multiplex page resources. Enable adaptive HTTP/2 flow control on + those connections and TCP_NODELAY on accepted proxy sockets, tunnel sockets, + and upstream upgrade connections to reduce delays for small writes. +- Preserve known-empty request bodies so the upstream client can retry HTTP/2 + REFUSED_STREAM and graceful GOAWAY failures. Nonempty or unknown-length bodies + continue to stream without buffering or replay. +- Include underlying DNS, TLS, socket, and HTTP/2 failure causes in proxy 502 + pages and logs instead of only reporting "error sending request". +- Improve live statistics, Requests, and log streams with visible connection + status, automatic reconnection, and handling for malformed or unexpected + frames. Leaving a live view cancels its connection and reconnect attempts. +- Show retryable loading errors in settings, filter catalogs, API-key loading, + and userscript management. A failed sign-in status check now offers a retry + instead of incorrectly showing the login form. +- Fix settings save feedback: acknowledge only successfully submitted values, + preserve text drafts after failures and edits made during a save, and disable + unchanged Save buttons and duplicate submissions. Ignore stale responses from + previously visited text-settings pages, and preserve pending exclusion edits + without restoring previously removed hosts. +- Keep blocking controls at their confirmed state after failed updates, and + restore the confirmed Debug settings after failed automatic saves. Lock + Network and PAC fields while saving so later edits cannot be marked saved by + an earlier response. +- Refresh installed filter lists after additions and removals while preserving + pending selections. Reset catalog pagination when searching, bound Next to + matching results, and keep failed removals visible. Built-in lists cannot be + removed through the catalog. +- Make navigation, settings forms, and dialogs fit phone screens. Improve form + labels and error/status announcements, and wrap notification text instead of + truncating it. +- Include [RELEASE.md](RELEASE.md) in generated GitHub release notes and discard cached + Debian/RPM packages before packaging to avoid publishing stale versions. +- Keep Docker builds on pull requests while restricting image and manifest + publishing to pushes. Validate release tags against the server package version + before building, and use that version in release artifact names. +- Correct the frontend cache path and key Rust caches by toolchain, dependencies, + and target. Use pinned prebuilt packaging tools and reuse compiled binaries + when creating Debian packages. + ## v0.8.0 - 2026-10-02 Release highlights and upgrade notes are in [RELEASE.md](RELEASE.md). diff --git a/Cargo.lock b/Cargo.lock index 0ffb544..97948af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2176,6 +2176,7 @@ dependencies = [ "futures", "futures-util", "getrandom 0.3.4", + "h2", "hex", "hmac", "http 1.4.2", @@ -2192,9 +2193,11 @@ dependencies = [ "once_cell", "openssl", "portable-atomic", + "redb", "regex", "reqwest", "rustls", + "semver", "serde", "serde-tuple-vec-map", "serde_json", @@ -2208,6 +2211,7 @@ dependencies = [ "tokio-stream", "tokio-util", "toml", + "tower-service", "uluru", "url", "warp", @@ -2393,6 +2397,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "redb" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de6c3b63e007e90ce536ec2ae4690826136a20ec8dbbbb400daef1bb999d2e36" +dependencies = [ + "libc", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3657,6 +3670,7 @@ dependencies = [ "gloo-net 0.6.0", "gloo-timers 0.4.0", "gloo-utils 0.3.0", + "js-sys", "log", "num-format", "readonly", diff --git a/Cross.toml b/Cross.toml index e01eb01..cbfed87 100644 --- a/Cross.toml +++ b/Cross.toml @@ -6,10 +6,6 @@ build-std = true [target.mipsel-unknown-linux-musl] build-std = true -[target.mipsel-unknown-linux-musl.env] -RUSTFLAGS = "-C target-feature=+crt-static" [target.mips-unknown-linux-musl] build-std = true -[target.mips-unknown-linux-musl.env] -RUSTFLAGS = "-C target-feature=+crt-static" diff --git a/README.md b/README.md index 8b1315b..8910da1 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ features, dependency updates, an improved UI, and server-friendly configuration - [Cert-pinned hosts (exclusions)](#4-cert-pinned-hosts-exclusions) - [Filter only selected hosts (inclusions)](#5-filter-only-selected-hosts-inclusions) - [HTTP CONNECT and proxy chaining](#6-http-connect-and-proxy-chaining) + - [Persistent statistics and privacy](#7-persistent-statistics-and-privacy) - [Screenshots](#screenshots) - [Acknowledgements](#acknowledgements) @@ -73,6 +74,7 @@ blocker more than it strictly replaces one. - Suppport for [Adblock Plus filters](https://adblockplus.org/filter-cheatsheet), such as [easylist](https://easylist.to/). - Web graphical user interface with a statistics display as well as a live request explorer. +- Opt-in persistent statistics and request history, with privacy levels, retention limits, and domain exclusions. - Support for uBlock origin's `js` syntax. - Support for uBlock origin's `redirect` syntax. - Support for uBlock origin's scriptlets. @@ -285,18 +287,43 @@ exclusions used by ordinary proxy clients. No additional Privaxy setting is needed. This feature supports chains that feed into Privaxy; configuring an upstream proxy for Privaxy's own outbound traffic is outside its scope. -The preceding proxy should preserve the destination hostname in CONNECT, which -Privaxy uses for certificates and domain matching. An IP-only CONNECT for a -named HTTPS site does not recover the hostname from TLS SNI and can fail -certificate validation. Clients must trust Privaxy's CA; certificate-pinned -services still need exclusions. Excluded or unselected hosts are tunneled -without protocol probing. +For an IP CONNECT (including resolved destinations from sing-box/TUN clients), +Privaxy reads TLS ClientHello SNI before choosing an interception certificate +or applying hostname inclusions and exclusions. A DNS SNI hostname supplies +the certificate's DNS SAN, certificate cache identity, HTTP filtering identity, +and upstream TLS SNI/verification name. The upstream connection still goes to +the original CONNECT IP and port. No sing-box DNS or LAN routing changes are +needed. Named CONNECT requests retain their existing behavior. + +Without usable DNS SNI, Privaxy retains the CONNECT identity: an intercepted +literal IP receives an IP SAN, which cannot authenticate a hidden DNS hostname. +Encrypted ClientHello (ECH) does not reveal its inner hostname to Privaxy. +Malformed, truncated, oversized (over 64 KiB), or stalled ClientHellos fail +closed; protocol detection, ClientHello reading, and the intercepted TLS +handshake each have a 10-second timeout. Fragmented ClientHellos and buffered +bytes are preserved, including when SNI selects a raw tunnel. + +Clients must still trust the existing Privaxy CA; certificate-pinned services +need exclusions. Explicit CONNECT-host/IP exclusions bypass protocol probing. +For other IP CONNECTs, hostname rules (including include-only mode) are decided +after reading SNI. Exclusions take precedence over inclusions. + +CONNECT to port **853** is an opaque TCP tunnel for DNS-over-TLS, including +`9.9.9.9:853` with SNI `dns.quad9.net`. TLS stays end-to-end and decrypted DNS +never enters the HTTP parser. Privaxy's DoH policy applies only to HTTP traffic; +it does not filter DoT. Non-HTTP TLS on other ports requires an exclusion. Filtering covers TCP HTTP/HTTPS. Handle UDP/QUIC and non-HTTP protocols in your routing policy; HTTP/3 traffic allowed directly will bypass filtering. Privaxy's client statistics identify the connecting proxy's address, so devices sharing that address appear together. +Local TLS regression tests can be run with +`cargo test -p privaxy --lib proxy::mitm::connect_tests`. They create temporary +trusted test CAs in memory and use loopback fixtures, with no production config +or external services. The DoT exchange uses an unprivileged ephemeral port; +port 853's automatic tunnel selection is tested separately. + #### Example: tun2proxy [tun2proxy](https://github.com/tun2proxy/tun2proxy) is one adapter that can send @@ -319,6 +346,87 @@ for those steps. > `password_hash` value from the config file and restart. The web UI will > force the setup flow again. +### 7. Persistent statistics and privacy + +Enable collection under **Settings → Statistics** to show saved totals, +hourly/daily trends, top domains/clients, and a paginated request log directly +on the **Dashboard**. With persistence off or Anonymous mode selected, the +Dashboard keeps its original layout with the current process's live counters. +Old `/history` bookmarks redirect to the Dashboard. +Persistence is **off by default**; enabling it starts collecting new activity, +without importing the live Requests feed. + +History records timestamps, destination hostnames, client IP addresses, and +whether a request was blocked or successfully forwarded. It never stores URL +paths, query strings, headers, cookies, or request/response bodies. Raw CONNECT +tunnels and requests that fail before being forwarded are not counted. Clients +behind another proxy share that proxy's client address. + +The privacy levels follow [Pi-hole's approach](https://docs.pi-hole.net/ftldns/privacylevels/): + +| Level | Saved identifiers | +| --- | --- | +| Show domains and clients | Hostname and client IP | +| Hide domains | Client IP only | +| Hide domains and clients (default) | Neither; request times and outcomes remain | +| Anonymous | No new history; session counters only | + +Hidden identifiers are removed **before queueing or storage**, without retaining +a reversible mapping or a hash. While collection is enabled, raising privacy +also redacts identifiers in existing history, including when entering anonymous +mode. Lowering privacy cannot restore them. These controls apply to statistics; +the separate live Requests feed and diagnostic logs can still contain full URLs. + +**Exclude domains from statistics and request logs** skips matching new traffic +in session counters, modified-response counts, saved history, and the live +Requests feed, even with persistence disabled. Filtering and interception still +apply. Use one case-insensitive hostname or wildcard pattern per line: +`example.com` matches exactly; `*.example.com` matches its subdomains. Do not +include schemes, ports, or paths. Existing history and diagnostic logs are +unaffected. + +Defaults are **7 days** and **100,000 saved requests**. The oldest records expire +when either limit is reached; valid limits are 1–365 days and 100–1,000,000 +requests. History totals and top lists describe retained requests, so they can +decrease as older entries expire. Modified-response totals are separate hourly +aggregates retained by age. Time windows include the current hour and preceding +hours. **Clear saved history** removes the database and starts fresh if enabled. +Turning collection off stops recording and closes the database; it retains the +existing file until cleared or collection is re-enabled and retention runs. + +The equivalent configuration section is: + +```toml +[statistics] +enabled = false +privacy = "hide_domains_and_clients" # full, hide_domains, hide_domains_and_clients, anonymous +retention_days = 7 +max_entries = 100000 +excluded_domains = ["example.com", "*.example.com"] +``` + +UI changes apply immediately; file edits also apply on SIGHUP. Data stays local +in `statistics.redb` under `PRIVAXY_BASE_PATH` (`/etc/privaxy` by default, `/conf` +in Docker). Keep `/conf` on a persistent Docker volume for history to survive +container replacement. No new ports or external database service are required. +On Unix, the database is restricted to its owner (`0600`). Storage is not +encrypted, and redaction/deletion is not secure erasure of filesystem snapshots, +backups, or recoverable disk pages. + +[redb](https://github.com/cberner/redb) transactions run on a dedicated worker, +with a bounded queue and batches flushed every five seconds or 512 events. +SIGTERM/SIGINT also flush accepted events; an abrupt kill can lose the pending +batch. Expired rows are pruned every minute and free space is compacted hourly; +the row limit bounds retained data, not an exact file size. Disk/write errors +pause recording and appear in Statistics settings without stopping the proxy. +Free space and save settings again to retry. Queue overflow drops statistics +events and reports their count instead of delaying proxy requests. + +The authenticated APIs are `GET/PUT /api/settings/statistics` and +`GET/DELETE /api/history`. History accepts `hours`, `limit` (1–200), `before` +(the previous response's `next_before` cursor), and `blocked=true|false`. +Pagination and the blocked filter affect request rows, not the window's totals. + ## Screenshots
diff --git a/privaxy/Cargo.toml b/privaxy/Cargo.toml index 4fae88e..56a9b79 100644 --- a/privaxy/Cargo.toml +++ b/privaxy/Cargo.toml @@ -3,6 +3,7 @@ name = "privaxy" description = "Next generation tracker and advertisement blocker" version = "0.8.0" edition = "2021" +repository = "https://github.com/joshrmcdaniel/privaxy" authors = [ "Pierre Barre ", "Josh McDaniel <80354972+joshrmcdaniel@users.noreply.github.com>" @@ -64,10 +65,12 @@ hyper-util = { version = "0.1", features = [ http-body-util = "0.1" tokio = { version = "1.22.0", features = ["full"] } serde_json = "1.0.89" +semver = "1" toml = "1" serde = { version = "1.0.148", features = ["derive"] } tokio-util = { version = "0.7.4", features = ["full"] } tokio-stream = "0.1" +tower-service = "0.3" bytes = "1" socket2 = "0.5" # default-features = false only to drop `single-thread`: the multi-thread @@ -137,6 +140,13 @@ tera = { version = "1", default-features = false } argon2 = { version = "0.5", features = ["std"] } hmac = "0.12" filterlists-api = { path = "../filterlists-api", features = ["reqwest"] } +# redb 4.3+ unconditionally imports libc::flock64 on GNU/Linux, but libc +# does not expose that type on MIPS. Keep the last compatible release until +# upstream fixes the range-lock backend; 4.2 retains the v3 database format. +redb = "=4.2.0" + +[dev-dependencies] +h2 = "0.4" # getrandom 0.3 (pulled in via warp -> tokio-tungstenite -> rand) selects a # backend that calls the libc `getrandom()` wrapper on mips{,el}-unknown-linux-gnu, diff --git a/privaxy/src/server/cert.rs b/privaxy/src/server/cert.rs index 0acc140..afb857b 100644 --- a/privaxy/src/server/cert.rs +++ b/privaxy/src/server/cert.rs @@ -49,10 +49,13 @@ impl SignedWithCaCert { // (ring) is installed once as the process default at startup. let private_key = PrivateKeyDer::try_from(private_key.private_key_to_der().unwrap()).unwrap(); - let server_configuration = ServerConfig::builder() + let mut server_configuration = ServerConfig::builder() .with_no_client_auth() .with_single_cert(certs, private_key) .unwrap(); + // The CONNECT service supports both versions. Advertise HTTP/2 so + // browsers can multiplex page resources on one intercepted session. + server_configuration.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; Self { authority, @@ -218,6 +221,9 @@ impl CertCache { cache.insert(certificate); } + /// `authority` is the TLS identity (DNS SNI for an IP CONNECT), not + /// necessarily the socket destination. Cache and SAN generation must use + /// that same identity so virtual hosts sharing an IP cannot share a leaf. pub async fn get(&self, authority: Authority) -> SignedWithCaCert { let mut cache = self.cache.lock().await; diff --git a/privaxy/src/server/configuration/mod.rs b/privaxy/src/server/configuration/mod.rs index 1cac41a..faeb422 100644 --- a/privaxy/src/server/configuration/mod.rs +++ b/privaxy/src/server/configuration/mod.rs @@ -7,6 +7,7 @@ mod ca; mod filter; mod filter_failures; mod network; +mod statistics; mod updater; mod userscript; pub use auth::*; @@ -15,6 +16,7 @@ pub use filter::*; pub use filter_failures::*; use futures::future::join_all; pub use network::*; +pub use statistics::*; use std::env; use std::path::{Path, PathBuf}; pub use updater::*; @@ -85,6 +87,8 @@ pub struct Configuration { pub auth: Auth, #[serde(default)] pub debug: DebugConfig, + #[serde(default)] + pub statistics: StatisticsConfig, /// Userscripts and the master switch governing them. Absent from /// configuration files written before the feature existed, hence the serde /// default. @@ -588,6 +592,7 @@ impl Configuration { custom_filters: Vec::new(), auth: Auth::new_initialized(), debug: DebugConfig::default(), + statistics: StatisticsConfig::default(), userscripts: UserScriptsConfig::default(), }) } diff --git a/privaxy/src/server/configuration/statistics.rs b/privaxy/src/server/configuration/statistics.rs new file mode 100644 index 0000000..9f93c39 --- /dev/null +++ b/privaxy/src/server/configuration/statistics.rs @@ -0,0 +1,62 @@ +use serde::{Deserialize, Serialize}; + +/// These levels describe collected data, not just what the UI displays. +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum StatisticsPrivacy { + Full, + HideDomains, + #[default] + HideDomainsAndClients, + Anonymous, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct StatisticsConfig { + pub enabled: bool, + pub privacy: StatisticsPrivacy, + pub retention_days: u16, + pub max_entries: u32, + pub excluded_domains: Vec, +} + +impl Default for StatisticsConfig { + fn default() -> Self { + Self { + enabled: false, + privacy: StatisticsPrivacy::default(), + retention_days: 7, + max_entries: 100_000, + excluded_domains: Vec::new(), + } + } +} + +impl StatisticsConfig { + pub fn validate(&self) -> Result<(), String> { + if self.excluded_domains.len() > 1000 + || self.excluded_domains.iter().any(|pattern| { + let pattern = pattern.trim(); + pattern.is_empty() + || pattern.len() > 253 + || !pattern + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b".-*?".contains(&b)) + }) + { + return Err("Use at most 1,000 domain patterns, without schemes, ports, or paths (for example example.com or *.example.com)".into()); + } + if !(1..=365).contains(&self.retention_days) { + return Err("Statistics retention must be between 1 and 365 days".into()); + } + if !(100..=1_000_000).contains(&self.max_entries) { + return Err("Request history limit must be between 100 and 1,000,000 entries".into()); + } + Ok(()) + } + + pub fn persists(&self) -> bool { + self.enabled && self.privacy != StatisticsPrivacy::Anonymous + } +} diff --git a/privaxy/src/server/lib.rs b/privaxy/src/server/lib.rs index 7e32955..747df5b 100644 --- a/privaxy/src/server/lib.rs +++ b/privaxy/src/server/lib.rs @@ -3,17 +3,15 @@ use crate::configuration::{FilterFailureStore, NetworkConfig}; use crate::proxy::exclusions::LocalExclusionStore; use crate::web_gui::events::Event; use hyper::service::service_fn; -use hyper_util::client::legacy::Client; use hyper_util::rt::{TokioExecutor, TokioIo}; use hyper_util::server::conn::auto; use hyper_util::server::graceful::GracefulShutdown; use include_dir::{include_dir, Dir}; use proxy::exclusions; use proxy::gm::storage::GmStorageStore; -use proxy::serve::UpgradeClient; use proxy::tls_failures::TlsFailureStore; +use proxy::upstream::{http_client_builder, UpstreamClients}; use proxy::userscripts::{PrivateNetworkAccess, UserScriptContext, UserScriptStore}; -use reqwest::redirect::Policy; use std::env; use std::net::IpAddr; use std::net::SocketAddr; @@ -104,7 +102,7 @@ pub(crate) fn parse_ip_address(ip_str: &str) -> IpAddr { IpAddr::from_str(ip_str).unwrap() } -async fn handle_signals() -> (Arc, Arc) { +async fn handle_signals(statistics: statistics::Statistics) -> (Arc, Arc) { let notify_shutdown = Arc::new(Notify::new()); let notify_reload = Arc::new(Notify::new()); let notify_shutdown_clone = notify_shutdown.clone(); @@ -115,6 +113,8 @@ async fn handle_signals() -> (Arc, Arc) { signal(SignalKind::hangup()).expect("failed to set up SIGHUP signal handler"); let mut term_signal = signal(SignalKind::terminate()).expect("failed to set up SIGTERM signal handler"); + let mut int_signal = + signal(SignalKind::interrupt()).expect("failed to set up SIGINT signal handler"); loop { tokio::select! { @@ -122,9 +122,16 @@ async fn handle_signals() -> (Arc, Arc) { log::info!("Received SIGHUP signal, restarting child processes..."); notify_reload_clone.notify_waiters(); } - _ = term_signal.recv() => { - log::info!("Received SIGTERM signal, shutting down gracefully..."); + _ = async { tokio::select! { _ = term_signal.recv() => {}, _ = int_signal.recv() => {} } } => { + log::info!("Shutting down; flushing persistent statistics..."); notify_shutdown_clone.notify_waiters(); + if let Some(history) = &statistics.history { + match tokio::time::timeout(Duration::from_secs(8), history.shutdown()).await { + Ok(Ok(())) => {}, + Ok(Err(error)) => log::error!("Could not flush statistics: {error}"), + Err(_) => log::error!("Timed out flushing statistics"), + } + } std::process::exit(0); } } @@ -149,38 +156,7 @@ pub async fn start_privaxy() -> PrivaxyServer { // We use reqwest instead of hyper's client to perform most of the proxying as it's more convenient // to handle compression as well as offers a more convenient interface. - let client = reqwest::Client::builder() - .use_rustls_tls() - .redirect(Policy::none()) - .no_proxy() - .gzip(true) - .brotli(true) - .deflate(true) - // Without these, a proxied request can hang indefinitely on a pooled - // keep-alive connection the remote has silently dropped: reqwest reuses - // the dead connection and waits on a peer that will never answer. - // Retiring idle connections quickly (well under typical server keep-alive - // windows) plus OS-level keepalive probes bounds that. `connect_timeout` - // additionally fails fast on unreachable hosts. - .connect_timeout(Duration::from_secs(10)) - .pool_idle_timeout(Duration::from_secs(30)) - // Bounds a stalled read mid-response: a peer that stops sending without - // closing would otherwise hang the proxied request forever. Generous - // enough (5 minutes between reads) not to kill long-polls or quiet SSE - // streams, which routinely idle for a minute or two between events. - .read_timeout(Duration::from_secs(300)) - // h2-heavy origins multiplex every subresource over a - // single connection whose flow-control window defaults to a small, - // shared 64 KB. When we drain one stream's body slowly (the browser - // reads slowly, or the HTML rewriter backpressures), that window fills - // and stalls *every other stream* on the connection — head-of-line - // stutter across the whole site. Adaptive flow control grows the - // stream/connection windows based on the bandwidth-delay product, - // relieving the stall while keeping multiplexing. (This overrides any - // manual http2_initial_*_window_size, which is why none are set.) - .http2_adaptive_window(true) - .build() - .unwrap(); + let client = http_client_builder().build().unwrap(); let configuration = match configuration::Configuration::read_from_home().await { Ok(configuration) => configuration, @@ -220,7 +196,7 @@ pub async fn start_privaxy() -> PrivaxyServer { } }; - let statistics = statistics::Statistics::new(); + let statistics = statistics::Statistics::with_history(configuration.statistics.clone()).await; let statistics_clone = statistics.clone(); let tls_failure_store = TlsFailureStore::new(configuration.ignored_tls_failures.clone()); @@ -272,7 +248,7 @@ pub async fn start_privaxy() -> PrivaxyServer { let configuration_save_lock = Arc::new(tokio::sync::Mutex::new(())); - let (_notify_shutdown, notify_reload) = handle_signals().await; + let (_notify_shutdown, notify_reload) = handle_signals(statistics.clone()).await; let block_disable_ref = blocking_disabled_store.clone(); let local_exclusion_store_ref = local_exclusion_store.clone(); @@ -361,6 +337,19 @@ pub async fn start_privaxy() -> PrivaxyServer { // The UI updates this store live; SIGHUP must also pick up file // edits to the mode and both inclusion/exclusion lists. local_exclusion_store.replace_configuration(&cfg); + { + // Serialize this re-read/apply with the settings API. A reload + // must not restore a stale privacy policy over a concurrent save. + let _guard = cfg_lock_backend.lock().await; + match configuration::Configuration::read_from_home().await { + Ok(current) => { + if let Err(error) = statistics.configure_history(current.statistics).await { + log::error!("Could not apply statistics settings: {error}"); + } + } + Err(error) => log::error!("Could not reload statistics settings: {error}"), + } + } // Same for the TLS-failure ignore set: pick up hand-edited // `ignored_tls_failures` entries on SIGHUP reload. tls_failure_store.set_ignored(cfg.ignored_tls_failures.clone()); @@ -742,21 +731,7 @@ async fn privaxy_backend( configuration_save_lock: Arc>, notify_reload: Arc, ) { - // Mirror the reqwest client's connection hardening (see above): without a - // connect timeout and OS-level keepalive, an upgrade can hang on a pooled - // keep-alive connection the remote has silently dropped, surfacing as - // "upgrade expected but not completed". - let mut http_connector = hyper_util::client::legacy::connect::HttpConnector::new(); - http_connector.enforce_http(false); - http_connector.set_connect_timeout(Some(Duration::from_secs(10))); - http_connector.set_keepalive(Some(Duration::from_secs(60))); - - let https_connector = hyper_rustls::HttpsConnectorBuilder::new() - .with_native_roots() - .expect("failed to load native root certificates") - .https_or_http() - .enable_http1() - .wrap_connector(http_connector); + let clients = UpstreamClients::new(client).expect("failed to load native root certificates"); let config = match read_configuration(&configuration_save_lock).await { Ok(config) => config, Err(err) => { @@ -785,17 +760,6 @@ async fn privaxy_backend( allow_private_network_requests: private_network_access, }; - // The hyper client is only used to perform upgrades. We don't need to - // handle compression. - // Hyper's client don't follow redirects, which is what we want, nothing to - // disable here. - // An upgraded connection is consumed by the tunnel anyway, so idle pooling - // buys nothing and only risks reusing a stale connection under a long-lived - // WebSocket — disable it. - let hyper_client: UpgradeClient = Client::builder(TokioExecutor::new()) - .pool_max_idle_per_host(0) - .build(https_connector); - let ip = env_or_config_ip(network_config).await; let proxy_server_addr = SocketAddr::from((ip, network_config.proxy_port)); @@ -835,6 +799,9 @@ async fn privaxy_backend( } }; + // Flush small TLS records and streamed response chunks promptly; + // Nagle plus delayed ACKs otherwise introduces avoidable stalls. + let _ = stream.set_nodelay(true); // Reproduce the old `Server::tcp_keepalive(600s)`: enable // OS-level keepalive on each accepted socket. let _ = socket2::SockRef::from(&stream).set_tcp_keepalive( @@ -842,8 +809,7 @@ async fn privaxy_backend( ); let client_ip_address = peer_addr.ip(); - let client = client.clone(); - let hyper_client = hyper_client.clone(); + let clients = clients.clone(); let cert_cache = cert_cache.clone(); let blocker_requester = blocker_requester.clone(); let broadcast_tx = broadcast_tx.clone(); @@ -861,8 +827,7 @@ async fn privaxy_backend( let service = service_fn(move |req| { proxy::serve_mitm_session( blocker_requester.clone(), - hyper_client.clone(), - client.clone(), + clients.clone(), req, cert_cache.clone(), broadcast_tx.clone(), diff --git a/privaxy/src/server/proxy/client_hello.rs b/privaxy/src/server/proxy/client_hello.rs new file mode 100644 index 0000000..d8fc582 --- /dev/null +++ b/privaxy/src/server/proxy/client_hello.rs @@ -0,0 +1,47 @@ +//! Read the ClientHello before choosing an interception policy or certificate. +use std::io; +use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite, BufReader}; +use tokio_rustls::StartHandshake; + +// Bound both the TLS parser's input and the copy retained for a possible tunnel. +const MAX_CLIENT_HELLO_BYTES: usize = 64 * 1024; +pub(super) const HANDSHAKE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +pub(super) async fn read_client_hello( + mut stream: BufReader, +) -> io::Result<(StartHandshake>, Vec)> +where + IO: AsyncRead + AsyncWrite + Unpin, +{ + let mut acceptor = rustls::server::Acceptor::default(); + let mut prefix = Vec::new(); + loop { + let remaining = MAX_CLIENT_HELLO_BYTES - prefix.len(); + if remaining == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "ClientHello too large", + )); + } + let buffered = stream.fill_buf().await?; + let available = &buffered[..buffered.len().min(remaining)]; + let consumed = acceptor.read_tls(&mut &available[..])?; + if consumed == 0 { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + prefix.extend_from_slice(&available[..consumed]); + stream.consume(consumed); + match acceptor.accept() { + Ok(Some(accepted)) => { + // rustls retains the parsed handshake for interception. For a + // bypass, replay exactly the consumed TLS bytes, then drain + // BufReader (including hyper's CONNECT read-ahead) normally. + return Ok((StartHandshake::from_parts(accepted, stream), prefix)); + } + Ok(None) => {} // TCP and TLS-record fragmentation are both valid. + Err((error, _alert)) => { + return Err(io::Error::new(io::ErrorKind::InvalidData, error)); + } + } + } +} diff --git a/privaxy/src/server/proxy/connect_tests.rs b/privaxy/src/server/proxy/connect_tests.rs new file mode 100644 index 0000000..cda8fdf --- /dev/null +++ b/privaxy/src/server/proxy/connect_tests.rs @@ -0,0 +1,1164 @@ +//! Local CONNECT/TLS regressions. All keys and trusted roots are ephemeral; +//! these fixtures never load configuration, production CAs or public services. +use super::*; +use crate::blocker::BlockingDisabledStore; +use crate::proxy::{gm::storage::GmStorageStore, userscripts::UserScriptStore}; +use http_body_util::BodyExt; +use openssl::{ + asn1::Asn1Time, + bn::{BigNum, MsbOption}, + hash::MessageDigest, + pkey::{PKey, Private}, + rsa::Rsa, + x509::{ + extension::{BasicConstraints, ExtendedKeyUsage, SubjectAlternativeName}, + X509NameBuilder, X509, + }, +}; +use rustls::{ + pki_types::{CertificateDer, PrivateKeyDer, ServerName}, + ClientConfig, ServerConfig, +}; +use std::{ + collections::BTreeMap, + io, + sync::{OnceLock, RwLock}, + time::Duration, +}; +use tokio::{ + net::TcpListener, + sync::mpsc, + task::{JoinHandle, JoinSet}, + time::timeout, +}; +use tokio_rustls::{client::TlsStream, LazyConfigAcceptor, TlsConnector}; + +const TEST_TIMEOUT: Duration = Duration::from_secs(15); + +struct Pki { + proxy: (X509, PKey), + upstream: (X509, PKey), + untrusted: (X509, PKey), + leaf_key: PKey, +} + +fn pki() -> &'static Pki { + static PKI: OnceLock = OnceLock::new(); + PKI.get_or_init(|| { + let _ = rustls::crypto::ring::default_provider().install_default(); + Pki { + proxy: crate::ca::make_ca_certificate(), + upstream: crate::ca::make_ca_certificate(), + untrusted: crate::ca::make_ca_certificate(), + leaf_key: PKey::from_rsa(Rsa::generate(2048).unwrap()).unwrap(), + } + }) +} + +fn der(cert: &X509) -> CertificateDer<'static> { + cert.to_der().unwrap().into() +} + +// Independent origin certificates: don't use Privaxy's leaf generator for +// both sides of a hostname-verification regression. +fn origin_config(host: &str, trusted: bool, expired: bool) -> Arc { + let pki = pki(); + let (ca, key) = if trusted { + &pki.upstream + } else { + &pki.untrusted + }; + let mut cert = X509::builder().unwrap(); + cert.set_version(2).unwrap(); + let mut serial = BigNum::new().unwrap(); + serial.rand(128, MsbOption::MAYBE_ZERO, false).unwrap(); + cert.set_serial_number(&serial.to_asn1_integer().unwrap()) + .unwrap(); + let mut subject = X509NameBuilder::new().unwrap(); + subject.append_entry_by_text("CN", host).unwrap(); + cert.set_subject_name(&subject.build()).unwrap(); + cert.set_issuer_name(ca.subject_name()).unwrap(); + cert.set_pubkey(&pki.leaf_key).unwrap(); + let (start, end) = if expired { + ( + Asn1Time::from_unix(1).unwrap(), + Asn1Time::from_unix(2).unwrap(), + ) + } else { + ( + Asn1Time::days_from_now(0).unwrap(), + Asn1Time::days_from_now(1).unwrap(), + ) + }; + cert.set_not_before(&start).unwrap(); + cert.set_not_after(&end).unwrap(); + cert.append_extension(BasicConstraints::new().build().unwrap()) + .unwrap(); + cert.append_extension(ExtendedKeyUsage::new().server_auth().build().unwrap()) + .unwrap(); + let mut san = SubjectAlternativeName::new(); + if host.parse::().is_ok() { + san.ip(host); + } else { + san.dns(host); + } + cert.append_extension(san.build(&cert.x509v3_context(Some(ca), None)).unwrap()) + .unwrap(); + cert.sign(key, MessageDigest::sha256()).unwrap(); + let config = ServerConfig::builder() + .with_no_client_auth() + .with_single_cert( + vec![der(&cert.build()), der(ca)], + PrivateKeyDer::try_from(pki.leaf_key.private_key_to_der().unwrap()).unwrap(), + ) + .unwrap(); + Arc::new(config) +} + +fn client_config(proxy: bool, sni: bool) -> Arc { + let mut roots = rustls::RootCertStore::empty(); + roots + .add(der(if proxy { + &pki().proxy.0 + } else { + &pki().upstream.0 + })) + .unwrap(); + let mut config = ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + // Disabling SNI does not disable certificate hostname/IP verification. + config.enable_sni = sni; + Arc::new(config) +} + +struct Task(JoinHandle<()>); +impl Drop for Task { + fn drop(&mut self) { + self.0.abort(); + } +} + +struct Proxy { + addr: SocketAddr, + _task: Task, +} + +impl Proxy { + async fn start(policy: LocalExclusionStore) -> Self { + Self::with_doh(policy, DohConfig::default()).await + } + + async fn with_doh(policy: LocalExclusionStore, doh_config: DohConfig) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let clients = UpstreamClients::with_test_roots(&[der(&pki().upstream.0)]); + let cache = CertCache::new(pki().proxy.0.clone(), pki().proxy.1.clone()); + let blocker = AdblockRequester::new(BlockingDisabledStore(Arc::new(RwLock::new(false)))); + let (events, _) = broadcast::channel(4); + let task = tokio::spawn(async move { + let mut sessions = JoinSet::new(); + while let Ok((stream, peer)) = listener.accept().await { + let clients = clients.clone(); + let cache = cache.clone(); + let blocker = blocker.clone(); + let events = events.clone(); + let policy = policy.clone(); + let doh_config = doh_config.clone(); + sessions.spawn(async move { + let service = service_fn(move |request| { + serve_mitm_session( + blocker.clone(), + clients.clone(), + request, + cache.clone(), + events.clone(), + Statistics::new(), + peer.ip(), + policy.clone(), + doh_config.clone(), + false, + TlsFailureStore::new(Default::default()), + None, + UserScriptContext { + store: UserScriptStore::default(), + gm_storage: GmStorageStore::in_memory(), + endpoint_signing_key: "fixture-only".into(), + allow_private_network_requests: Default::default(), + }, + ) + }); + let _ = auto::Builder::new(TokioExecutor::new()) + .serve_connection_with_upgrades(TokioIo::new(stream), service) + .await; + }); + } + }); + Self { + addr, + _task: Task(task), + } + } + + async fn connect( + &self, + destination: &str, + config: Arc, + name: &str, + ) -> TlsStream { + let mut tcp = TcpStream::connect(self.addr).await.unwrap(); + tcp.write_all( + format!("CONNECT {destination} HTTP/1.1\r\nHost: {destination}\r\n\r\n").as_bytes(), + ) + .await + .unwrap(); + assert!(read_headers(&mut tcp) + .await + .unwrap() + .starts_with(b"HTTP/1.1 200")); + timeout( + TEST_TIMEOUT, + TlsConnector::from(config).connect(ServerName::try_from(name.to_owned()).unwrap(), tcp), + ) + .await + .unwrap() + .unwrap() + } +} + +// Read exactly through the header terminator, retaining TLS/DNS bytes that +// might arrive in the same TCP packet. +async fn read_headers(stream: &mut (impl AsyncRead + Unpin)) -> io::Result> { + let mut bytes = Vec::new(); + while !bytes.ends_with(b"\r\n\r\n") { + bytes.push(stream.read_u8().await?); + assert!(bytes.len() < 32 * 1024); + } + Ok(bytes) +} + +#[derive(Clone, Copy)] +enum Protocol { + Http, + HttpKeepAlive, + HttpWaitForPeer, + Http2RefuseOnce, + Http2ResponseHeaders(usize), + Dns, + Upgrade, +} + +struct Origin { + addr: SocketAddr, + requests: mpsc::UnboundedReceiver<(Option, Vec)>, + _task: Task, +} + +impl Origin { + async fn start(configs: Vec<(&str, Arc)>, protocol: Protocol) -> Self { + Self::bind("127.0.0.1:0", configs, protocol).await + } + + async fn bind( + address: &str, + configs: Vec<(&str, Arc)>, + protocol: Protocol, + ) -> Self { + let listener = TcpListener::bind(address).await.unwrap(); + let addr = listener.local_addr().unwrap(); + let fallback = configs[0].1.clone(); + let configs: BTreeMap<_, _> = configs + .into_iter() + .map(|(name, config)| (name.to_owned(), config)) + .collect(); + let (tx, requests) = mpsc::unbounded_channel(); + let response_gate = Arc::new(tokio::sync::Notify::new()); + let task = tokio::spawn(async move { + let mut sessions = JoinSet::new(); + while let Ok((tcp, _)) = listener.accept().await { + let configs = configs.clone(); + let fallback = fallback.clone(); + let tx = tx.clone(); + let response_gate = response_gate.clone(); + sessions.spawn(async move { + let Ok(hello) = LazyConfigAcceptor::new(Default::default(), tcp).await else { return; }; + let name = hello.client_hello().server_name().map(str::to_owned); + let config = name.as_ref().and_then(|name| configs.get(name)).unwrap_or(&fallback).clone(); + let Ok(mut tls) = hello.into_stream(config).await else { return; }; + match protocol { + Protocol::Http | Protocol::HttpKeepAlive | Protocol::HttpWaitForPeer => { + let keep_alive = matches!(protocol, Protocol::HttpKeepAlive); + loop { + let Ok(request) = read_headers(&mut tls).await else { return; }; + let slow = request.starts_with(b"GET /slow "); + let fast = request.starts_with(b"GET /fast "); + tx.send((name.clone(), request)).unwrap(); + if matches!(protocol, Protocol::HttpWaitForPeer) { + if slow { + response_gate.notified().await; + } else if fast { + response_gate.notify_one(); + } + } + let body = format!("upstream:{}", name.as_deref().unwrap_or("no-sni")); + let connection = if keep_alive { "keep-alive" } else { "close" }; + tls.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: {connection}\r\n\r\n{body}", body.len()).as_bytes()).await.unwrap(); + if !keep_alive { break; } + } + } + Protocol::Dns => { + // Two actual DNS-over-TCP frames, including their + // length prefixes and binary DNS response headers. + for _ in 0..2 { + let length = tls.read_u16().await.unwrap(); + let mut query = vec![0; length as usize]; + tls.read_exact(&mut query).await.unwrap(); + tx.send((name.clone(), query.clone())).unwrap(); + query[2..4].copy_from_slice(&[0x81, 0x83]); // NXDOMAIN + tls.write_u16(length).await.unwrap(); + tls.write_all(&query).await.unwrap(); + } + } + Protocol::Http2RefuseOnce => { + let mut connection = h2::server::handshake(tls).await.unwrap(); + let mut refused = false; + while let Some(request) = connection.accept().await { + let (request, mut respond) = request.unwrap(); + tx.send((name.clone(), request.method().as_str().as_bytes().to_vec())).unwrap(); + if !refused { + // The origin explicitly guarantees it did + // not process this request. A bodyless GET + // should be replayed by reqwest's policy. + respond.send_reset(h2::Reason::REFUSED_STREAM); + refused = true; + } else { + let response = http::Response::builder() + .header("content-type", "text/plain") + .body(()) + .unwrap(); + respond.send_response(response, false).unwrap() + .send_data(bytes::Bytes::from_static(b"retried"), true).unwrap(); + } + } + return; + } + Protocol::Http2ResponseHeaders(size) => { + let mut connection = h2::server::handshake(tls).await.unwrap(); + while let Some(request) = connection.accept().await { + let (request, mut respond) = request.unwrap(); + tx.send((name.clone(), request.method().as_str().as_bytes().to_vec())).unwrap(); + let response = http::Response::builder() + .header("content-type", "text/plain") + .header("content-security-policy", format!("script-src https://{}.test", "a".repeat(size))) + .body(()) + .unwrap(); + respond.send_response(response, false).unwrap() + .send_data(bytes::Bytes::from_static(b"large headers"), true).unwrap(); + } + return; + } + Protocol::Upgrade => { + let request = read_headers(&mut tls).await.unwrap(); + tx.send((name, request)).unwrap(); + tls.write_all(b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n").await.unwrap(); + let mut payload = [0; 4]; + tls.read_exact(&mut payload).await.unwrap(); + tls.write_all(&payload).await.unwrap(); + } + } + let _ = tls.shutdown().await; + }); + } + }); + Self { + addr, + requests, + _task: Task(task), + } + } + + async fn request(&mut self) -> (Option, Vec) { + timeout(TEST_TIMEOUT, self.requests.recv()) + .await + .unwrap() + .unwrap() + } +} + +async fn http_get( + stream: impl AsyncRead + AsyncWrite + Unpin + Send + 'static, +) -> (http::StatusCode, String) { + http_request(stream, "https://unrelated.invalid:1234/test", "text/plain").await +} + +async fn http_request( + stream: impl AsyncRead + AsyncWrite + Unpin + Send + 'static, + uri: &str, + accept: &str, +) -> (http::StatusCode, String) { + // Neither an inner Host nor an absolute URL may change the selected peer. + let request = Request::builder() + .uri(uri) + .header("Host", "unrelated.invalid:1234") + .header("Accept", accept) + .body(empty_body()) + .unwrap(); + send_http_request(stream, request).await +} + +async fn send_http_request( + stream: impl AsyncRead + AsyncWrite + Unpin + Send + 'static, + request: Request, +) -> (http::StatusCode, String) { + let (mut sender, connection) = hyper::client::conn::http1::handshake(TokioIo::new(stream)) + .await + .unwrap(); + let _task = Task(tokio::spawn(async { + let _ = connection.await; + })); + let response = timeout(TEST_TIMEOUT, sender.send_request(request)) + .await + .unwrap() + .unwrap(); + let status = response.status(); + let bytes = timeout(TEST_TIMEOUT, response.into_body().collect()) + .await + .unwrap() + .unwrap() + .to_bytes(); + (status, String::from_utf8(bytes.to_vec()).unwrap()) +} + +fn peer_certificate(tls: &TlsStream) -> X509 { + X509::from_der(tls.get_ref().1.peer_certificates().unwrap()[0].as_ref()).unwrap() +} + +fn assert_dns_identity(cert: &X509, host: &str) { + let names = cert.subject_alt_names().unwrap(); + assert_eq!(names.len(), 1); + assert_eq!(names[0].dnsname(), Some(host)); + assert!(cert.verify(&pki().proxy.0.public_key().unwrap()).unwrap()); +} + +#[tokio::test] +async fn ip_connect_uses_sni_for_certificates_upstream_tls_and_cache() { + let mut origin = Origin::start( + vec![ + ("one.test", origin_config("one.test", true, false)), + ("two.test", origin_config("two.test", true, false)), + ], + Protocol::Http, + ) + .await; + let proxy = Proxy::start(LocalExclusionStore::new(vec![])).await; + let mut first_cert = None; + for host in ["one.test", "two.test", "one.test"] { + let tls = proxy + .connect(&origin.addr.to_string(), client_config(true, true), host) + .await; + let cert = peer_certificate(&tls); + assert_dns_identity(&cert, host); + if host == "one.test" { + if let Some(first) = &first_cert { + assert_eq!(&cert.to_der().unwrap(), first); + } else { + first_cert = Some(cert.to_der().unwrap()); + } + } + let (status, body) = http_get(tls).await; + assert_eq!(status, http::StatusCode::OK, "{body}"); + assert_eq!(body, format!("upstream:{host}")); + let (sni, request) = origin.request().await; + assert_eq!(sni.as_deref(), Some(host)); + let request = String::from_utf8(request).unwrap().to_ascii_lowercase(); + assert!(request.starts_with("get /test http/1.1")); + assert!(request.contains(&format!("host: {host}:{}\r\n", origin.addr.port()))); + } +} + +#[tokio::test] +async fn same_sni_on_different_connect_destinations_does_not_reuse_the_wrong_pool() { + let mut first = Origin::start( + vec![("shared.test", origin_config("shared.test", true, false))], + Protocol::HttpKeepAlive, + ) + .await; + let address = format!("127.0.0.2:{}", first.addr.port()); + let mut second = Origin::bind( + &address, + vec![("shared.test", origin_config("shared.test", true, false))], + Protocol::HttpKeepAlive, + ) + .await; + let proxy = Proxy::start(LocalExclusionStore::new(vec![])).await; + for index in 0..3 { + let origin = if index == 1 { &mut second } else { &mut first }; + let tls = proxy + .connect( + &origin.addr.to_string(), + client_config(true, true), + "shared.test", + ) + .await; + assert_eq!(http_get(tls).await.0, http::StatusCode::OK); + assert_eq!(origin.request().await.0.as_deref(), Some("shared.test")); + } +} + +#[tokio::test] +async fn hostname_and_literal_ip_connect_keep_verification_and_missing_sni_behavior() { + for (connect_host, identity, send_sni) in [ + ("localhost", "localhost", true), + ("localhost", "localhost", false), + ("127.0.0.1", "127.0.0.1", false), + ("[::1]", "::1", false), + ("[::1]", "one.test", true), + ] { + let bind = if connect_host == "[::1]" { + "[::1]:0" + } else { + "127.0.0.1:0" + }; + let mut origin = Origin::bind( + bind, + vec![(identity, origin_config(identity, true, false))], + Protocol::Http, + ) + .await; + let proxy = Proxy::start(LocalExclusionStore::new(vec![])).await; + let authority = format!("{connect_host}:{}", origin.addr.port()); + let tls = proxy + .connect(&authority, client_config(true, send_sni), identity) + .await; + let cert = peer_certificate(&tls); + if identity.parse::().is_err() { + assert_dns_identity(&cert, identity); + } else { + assert!(cert.subject_alt_names().unwrap()[0].ipaddress().is_some()); + } + let (status, body) = http_get(tls).await; + assert_eq!(status, http::StatusCode::OK, "{body}"); + let upstream_sni = origin.request().await.0; + assert_eq!( + upstream_sni.as_deref(), + if identity.parse::().is_err() { + Some(identity) + } else { + None + } + ); + } +} + +#[tokio::test] +async fn ip_connect_without_sni_does_not_guess_a_dns_identity() { + let origin = Origin::start( + vec![("one.test", origin_config("one.test", true, false))], + Protocol::Http, + ) + .await; + let proxy = Proxy::start(LocalExclusionStore::new(vec![])).await; + let mut tcp = TcpStream::connect(proxy.addr).await.unwrap(); + tcp.write_all(format!("CONNECT {} HTTP/1.1\r\n\r\n", origin.addr).as_bytes()) + .await + .unwrap(); + read_headers(&mut tcp).await.unwrap(); + let result = timeout( + TEST_TIMEOUT, + TlsConnector::from(client_config(true, false)) + .connect(ServerName::try_from("one.test").unwrap(), tcp), + ) + .await + .unwrap(); + assert!( + result.is_err(), + "a DNS verifier must not accept an IP-only certificate" + ); +} + +#[tokio::test] +async fn sni_exclusions_and_ip_exclusions_bypass_interception() { + for (host, exclusions) in [ + ("excluded.test", vec!["*.test".into()]), + ("www.apple.com", vec![]), // built-in exclusion from the reported failure + ("one.test", vec!["127.0.0.1".into()]), + ] { + let mut origin = Origin::start( + vec![(host, origin_config(host, true, false))], + Protocol::Http, + ) + .await; + let proxy = Proxy::start(LocalExclusionStore::new(exclusions)).await; + // Trust only the origin CA: a minted interception cert must fail here. + let tls = proxy + .connect(&origin.addr.to_string(), client_config(false, true), host) + .await; + assert!(peer_certificate(&tls) + .verify(&pki().upstream.0.public_key().unwrap()) + .unwrap()); + assert_eq!(http_get(tls).await.0, http::StatusCode::OK); + assert_eq!(origin.request().await.0.as_deref(), Some(host)); + } +} + +#[tokio::test] +async fn include_only_rules_are_evaluated_after_ip_sni_and_exclusions_win() { + let origin = Origin::start( + vec![("included.test", origin_config("included.test", true, false))], + Protocol::Http, + ) + .await; + for (exclusions, intercept) in [ + (vec![], true), + (vec!["included.test".into()], false), + (vec!["127.0.0.1".into()], false), + ] { + let policy = + LocalExclusionStore::with_rules(true, vec!["included.test".into()], exclusions); + let proxy = Proxy::start(policy).await; + let tls = proxy + .connect( + &origin.addr.to_string(), + client_config(intercept, true), + "included.test", + ) + .await; + if intercept { + assert_dns_identity(&peer_certificate(&tls), "included.test"); + } + assert_eq!(http_get(tls).await.0, http::StatusCode::OK); + } +} + +#[tokio::test] +async fn browser_alpn_enables_parallel_http2_requests_and_keeps_http1() { + let host = "multiplex.test"; + let mut origin = Origin::start( + vec![(host, origin_config(host, true, false))], + Protocol::HttpWaitForPeer, + ) + .await; + let proxy = Proxy::start(LocalExclusionStore::new(vec![])).await; + let mut config = (*client_config(true, true)).clone(); + config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; + let tls = proxy + .connect(&origin.addr.to_string(), Arc::new(config), host) + .await; + assert_eq!(tls.get_ref().1.alpn_protocol(), Some(b"h2".as_slice())); + let (mut sender, connection) = + hyper::client::conn::http2::handshake(TokioExecutor::new(), TokioIo::new(tls)) + .await + .unwrap(); + let _task = Task(tokio::spawn(async { + let _ = connection.await; + })); + + // The first origin response is held until the second request arrives. + // Both requests must share this one browser-side TLS connection. + let slow = tokio::spawn( + sender.send_request( + Request::builder() + .uri("https://unrelated.invalid/slow") + .body(empty_body()) + .unwrap(), + ), + ); + let (sni, request) = origin.request().await; + assert_eq!(sni.as_deref(), Some(host)); + assert!(request.starts_with(b"GET /slow ")); + let response = timeout( + TEST_TIMEOUT, + sender.send_request( + Request::builder() + .uri("https://unrelated.invalid/fast") + // Cookie-heavy browsers must not hit hyper's 16 KiB default + // when they start negotiating HTTP/2 with the proxy. + .header("cookie", format!("padding={}", "a".repeat(24 * 1024))) + .body(empty_body()) + .unwrap(), + ), + ) + .await + .unwrap() + .unwrap(); + let slow_response = timeout(TEST_TIMEOUT, slow).await.unwrap().unwrap().unwrap(); + for response in [response, slow_response] { + assert_eq!(response.status(), http::StatusCode::OK); + let body = timeout(TEST_TIMEOUT, response.into_body().collect()) + .await + .unwrap() + .unwrap() + .to_bytes(); + assert_eq!(body, format!("upstream:{host}")); + } + let (sni, request) = origin.request().await; + assert_eq!(sni.as_deref(), Some(host)); + assert!(request.starts_with(b"GET /fast ")); + assert!(String::from_utf8(request) + .unwrap() + .contains(&format!("host: {host}:{}", origin.addr.port()))); + + let mut config = (*client_config(true, true)).clone(); + config.alpn_protocols = vec![b"http/1.1".to_vec()]; + let tls = proxy + .connect(&origin.addr.to_string(), Arc::new(config), host) + .await; + assert_eq!( + tls.get_ref().1.alpn_protocol(), + Some(b"http/1.1".as_slice()) + ); + assert_eq!(http_get(tls).await.0, http::StatusCode::OK); +} + +#[tokio::test] +async fn empty_get_retries_a_refused_http2_stream() { + let mut config = (*origin_config("retry.test", true, false)).clone(); + config.alpn_protocols = vec![b"h2".to_vec()]; + let mut origin = Origin::start( + vec![("retry.test", Arc::new(config))], + Protocol::Http2RefuseOnce, + ) + .await; + let proxy = Proxy::start(LocalExclusionStore::new(vec![])).await; + let tls = proxy + .connect( + &origin.addr.to_string(), + client_config(true, true), + "retry.test", + ) + .await; + + let (status, body) = http_get(tls).await; + assert_eq!(status, http::StatusCode::OK, "{body}"); + assert_eq!(body, "retried"); + for _ in 0..2 { + let (sni, method) = origin.request().await; + assert_eq!(sni.as_deref(), Some("retry.test")); + assert_eq!(method, b"GET"); + } + assert!(origin.requests.try_recv().is_err()); +} + +#[tokio::test] +async fn nonempty_requests_are_not_replayed_after_http2_refusal() { + for method in [Method::GET, Method::POST] { + let mut config = (*origin_config("retry.test", true, false)).clone(); + config.alpn_protocols = vec![b"h2".to_vec()]; + let mut origin = Origin::start( + vec![("retry.test", Arc::new(config))], + Protocol::Http2RefuseOnce, + ) + .await; + let proxy = Proxy::start(LocalExclusionStore::new(vec![])).await; + let tls = proxy + .connect( + &origin.addr.to_string(), + client_config(true, true), + "retry.test", + ) + .await; + let request = Request::builder() + .method(method.clone()) + .uri("/upload") + .body(crate::proxy::full_body("must not be replayed")) + .unwrap(); + let (status, _) = send_http_request(tls, request).await; + assert_eq!(status, http::StatusCode::BAD_GATEWAY); + assert_eq!(origin.request().await.1, method.as_str().as_bytes()); + assert!(origin.requests.try_recv().is_err()); + } +} + +#[tokio::test] +async fn large_http2_response_headers_are_accepted_with_a_bounded_limit() { + // Exercise the shared client (literal IP) and an SNI-pinned client. The + // decoded size, including HPACK's per-field overhead, matters here, not + // the compressed bytes on the wire. + for host in ["127.0.0.1", "headers.test"] { + for (size, expected) in [ + (24 * 1024, http::StatusCode::OK), + (128 * 1024, http::StatusCode::BAD_GATEWAY), + ] { + let mut config = (*origin_config(host, true, false)).clone(); + config.alpn_protocols = vec![b"h2".to_vec()]; + let origin = Origin::start( + vec![(host, Arc::new(config))], + Protocol::Http2ResponseHeaders(size), + ) + .await; + let proxy = Proxy::start(LocalExclusionStore::new(vec![])).await; + let tls = proxy + .connect(&origin.addr.to_string(), client_config(true, true), host) + .await; + let (status, body) = http_get(tls).await; + let reason = body + .lines() + .find(|line| line.contains("error sending request")) + .unwrap_or(&body); + assert_eq!(status, expected, "host={host}, size={size}: {reason}"); + if expected == http::StatusCode::OK { + assert_eq!(body, "large headers"); + } + } + } +} + +#[tokio::test] +async fn invalid_upstream_certificates_are_rejected() { + for (cert_host, trusted, expired) in [ + ("one.test", false, false), + ("wrong.test", true, false), + ("one.test", true, true), + ] { + let mut origin = Origin::start( + vec![("one.test", origin_config(cert_host, trusted, expired))], + Protocol::Http, + ) + .await; + let proxy = Proxy::start(LocalExclusionStore::new(vec![])).await; + let tls = proxy + .connect( + &origin.addr.to_string(), + client_config(true, true), + "one.test", + ) + .await; + let (status, body) = http_get(tls).await; + assert_eq!( + status, + http::StatusCode::BAD_GATEWAY, + "cert={cert_host}, trusted={trusted}, expired={expired}" + ); + assert!(body.contains("invalid peer certificate"), "{body}"); + assert!( + origin.requests.try_recv().is_err(), + "HTTP must never reach an unverified peer" + ); + // The separate hyper client used for upgrades must reject it too. + let mut tls = proxy + .connect( + &origin.addr.to_string(), + client_config(true, true), + "one.test", + ) + .await; + tls.write_all(b"GET /socket HTTP/1.1\r\nHost: one.test\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n").await.unwrap(); + let response = timeout(TEST_TIMEOUT, read_headers(&mut tls)) + .await + .unwrap() + .unwrap(); + assert!(response.starts_with(b"HTTP/1.1 502")); + assert!(origin.requests.try_recv().is_err()); + } +} + +#[tokio::test] +async fn ip_connect_protocol_upgrades_keep_the_destination_and_verify_sni() { + let mut origin = Origin::start( + vec![("upgrade.test", origin_config("upgrade.test", true, false))], + Protocol::Upgrade, + ) + .await; + let proxy = Proxy::start(LocalExclusionStore::new(vec![])).await; + let mut tls = proxy + .connect( + &origin.addr.to_string(), + client_config(true, true), + "upgrade.test", + ) + .await; + tls.write_all(b"GET /socket HTTP/1.1\r\nHost: unrelated.invalid\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n").await.unwrap(); + let response = timeout(TEST_TIMEOUT, read_headers(&mut tls)) + .await + .unwrap() + .unwrap(); + assert!(response.starts_with(b"HTTP/1.1 101")); + tls.write_all(b"ping").await.unwrap(); + let mut echo = [0; 4]; + timeout(TEST_TIMEOUT, tls.read_exact(&mut echo)) + .await + .unwrap() + .unwrap(); + assert_eq!(&echo, b"ping"); + let (sni, request) = origin.request().await; + assert_eq!(sni.as_deref(), Some("upgrade.test")); + assert!(String::from_utf8(request) + .unwrap() + .contains(&format!("host: upgrade.test:{}", origin.addr.port()))); +} + +#[tokio::test] +async fn dns_over_tls_passes_binary_queries_and_responses_unchanged() { + let mut origin = Origin::start( + vec![("dns.quad9.net", origin_config("dns.quad9.net", true, false))], + Protocol::Dns, + ) + .await; + // An ephemeral unprivileged port keeps this test runnable without root. + // Standard port 853's automatic bypass is covered separately below. + let proxy = Proxy::start(LocalExclusionStore::new(vec!["dns.quad9.net".into()])).await; + let mut tls = proxy + .connect( + &origin.addr.to_string(), + client_config(false, true), + "dns.quad9.net", + ) + .await; + for id in [0x12, 0x34] { + let query = [ + id, 0x56, 0x01, 0x00, 0, 1, 0, 0, 0, 0, 0, 0, 7, b'e', b'x', b'a', b'm', b'p', b'l', + b'e', 3, b'c', b'o', b'm', 0, 0, 1, 0, 1, + ]; + tls.write_u16(query.len() as u16).await.unwrap(); + tls.write_all(&query).await.unwrap(); + let length = timeout(TEST_TIMEOUT, tls.read_u16()) + .await + .unwrap() + .unwrap(); + let mut response = vec![0; length as usize]; + timeout(TEST_TIMEOUT, tls.read_exact(&mut response)) + .await + .unwrap() + .unwrap(); + let mut expected = query; + expected[2..4].copy_from_slice(&[0x81, 0x83]); + assert_eq!(response, expected); + let (sni, received) = origin.request().await; + assert_eq!(sni.as_deref(), Some("dns.quad9.net")); + assert_eq!(received, query); + } +} + +#[test] +fn standard_dot_port_bypasses_without_consuming_tls_or_entering_http() { + let policy = LocalExclusionStore::new(vec![]); + for destination in ["9.9.9.9:853", "[2620:fe::fe]:853", "dns.quad9.net:853"] { + let authority: Authority = destination.parse().unwrap(); + assert!(bypass_before_client_hello(&policy, &authority, &authority)); + } + let https: Authority = "9.9.9.9:443".parse().unwrap(); + assert!(!bypass_before_client_hello(&policy, &https, &https)); +} + +fn initial_client_hello( + config: Arc, + host: &str, +) -> (rustls::ClientConnection, Vec) { + let mut client = + rustls::ClientConnection::new(config, ServerName::try_from(host.to_owned()).unwrap()) + .unwrap(); + let mut bytes = Vec::new(); + client.write_tls(&mut bytes).unwrap(); + (client, bytes) +} + +/// Split the handshake itself across TLS records as well as TCP writes. Just +/// fragmenting TCP would miss implementations that only parse the first record. +fn fragment_records(wire: &[u8]) -> Vec { + let mut fragmented = Vec::new(); + let mut offset = 0; + while offset < wire.len() { + let header = &wire[offset..offset + 5]; + let length = u16::from_be_bytes([header[3], header[4]]) as usize; + for chunk in wire[offset + 5..offset + 5 + length].chunks(17) { + fragmented.extend_from_slice(&header[..3]); + fragmented.extend_from_slice(&(chunk.len() as u16).to_be_bytes()); + fragmented.extend_from_slice(chunk); + } + offset += 5 + length; + } + fragmented +} + +async fn flush_tls(client: &mut rustls::ClientConnection, tcp: &mut TcpStream) { + while client.wants_write() { + let mut bytes = Vec::new(); + client.write_tls(&mut bytes).unwrap(); + tcp.write_all(&bytes).await.unwrap(); + } +} + +async fn receive_tls(client: &mut rustls::ClientConnection, tcp: &mut TcpStream) { + let mut bytes = [0; 16 * 1024]; + let length = tcp.read(&mut bytes).await.unwrap(); + assert_ne!(length, 0, "TLS peer closed unexpectedly"); + let mut reader = &bytes[..length]; + while !reader.is_empty() { + assert_ne!(client.read_tls(&mut reader).unwrap(), 0); + client.process_new_packets().unwrap(); + } +} + +#[tokio::test] +async fn fragmented_client_hello_and_connect_read_ahead_work_for_mitm_and_bypass() { + for intercept in [true, false] { + let mut origin = Origin::start( + vec![("fragment.test", origin_config("fragment.test", true, false))], + Protocol::Http, + ) + .await; + let policy = LocalExclusionStore::new(if intercept { + vec![] + } else { + vec!["fragment.test".into()] + }); + let proxy = Proxy::start(policy).await; + timeout(TEST_TIMEOUT, async { + let (mut client, hello) = + initial_client_hello(client_config(intercept, true), "fragment.test"); + let hello = fragment_records(&hello); + let mut tcp = TcpStream::connect(proxy.addr).await.unwrap(); + tcp.set_nodelay(true).unwrap(); + let mut request = format!( + "CONNECT {} HTTP/1.1\r\nHost: {}\r\n\r\n", + origin.addr, origin.addr + ) + .into_bytes(); + // Exercise hyper's upgrade read-ahead with a partial TLS header. + request.extend_from_slice(&hello[..3]); + tcp.write_all(&request).await.unwrap(); + assert!(read_headers(&mut tcp) + .await + .unwrap() + .starts_with(b"HTTP/1.1 200")); + for chunk in hello[3..].chunks(3) { + tcp.write_all(chunk).await.unwrap(); + tokio::task::yield_now().await; + } + while client.is_handshaking() { + flush_tls(&mut client, &mut tcp).await; + receive_tls(&mut client, &mut tcp).await; + } + if intercept { + let cert = X509::from_der(client.peer_certificates().unwrap()[0].as_ref()).unwrap(); + assert_dns_identity(&cert, "fragment.test"); + } + std::io::Write::write_all( + &mut client.writer(), + b"GET /fragment HTTP/1.1\r\nHost: fragment.test\r\nConnection: close\r\n\r\n", + ) + .unwrap(); + flush_tls(&mut client, &mut tcp).await; + let mut response = Vec::new(); + loop { + receive_tls(&mut client, &mut tcp).await; + match std::io::Read::read_to_end(&mut client.reader(), &mut response) { + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::WouldBlock => {} + Err(error) => panic!("TLS read: {error}"), + } + if response.ends_with(b"upstream:fragment.test") { + break; + } + } + assert!(response.starts_with(b"HTTP/1.1 200")); + assert_eq!(origin.request().await.0.as_deref(), Some("fragment.test")); + }) + .await + .unwrap(); + } +} + +#[tokio::test] +async fn client_hello_parser_retains_every_byte_at_all_record_and_buffer_boundaries() { + let (_, hello) = initial_client_hello(client_config(true, true), "fragment.test"); + let mut wire = fragment_records(&hello); + // Bytes after the ClientHello may already be buffered, including TLS 1.3 + // compatibility CCS or early data. A tunnel must preserve them verbatim. + wire.extend_from_slice(b"\x14\x03\x03\x00\x01\x01buffered-after-hello"); + for capacity in [1, 2, 5, 19, 8192] { + let (mut writer, reader) = tokio::io::duplex(wire.len()); + writer.write_all(&wire).await.unwrap(); + writer.shutdown().await.unwrap(); + let (mut hello, mut replay) = read_client_hello(BufReader::with_capacity(capacity, reader)) + .await + .unwrap(); + assert_eq!(hello.client_hello().server_name(), Some("fragment.test")); + hello.io.read_to_end(&mut replay).await.unwrap(); + assert_eq!(replay, wire, "buffer capacity {capacity}"); + } +} + +#[tokio::test] +async fn malformed_truncated_and_oversized_client_hellos_fail_closed() { + let mut oversized = vec![0x16, 3, 1, 0, 4, 1, 1, 0, 1]; + for _ in 0..5 { + oversized.extend_from_slice(&[0x16, 3, 1, 0x40, 0]); + oversized.extend_from_slice(&[0; 16 * 1024]); + } + for bytes in [ + b"\x16\x03\x01\x00\x20\x01".to_vec(), // truncated handshake + b"\x16\x03\x01\x00\x05\x02\x00\x00\x01\x00".to_vec(), // not ClientHello + oversized, + ] { + let (mut writer, reader) = tokio::io::duplex(bytes.len()); + writer.write_all(&bytes).await.unwrap(); + writer.shutdown().await.unwrap(); + assert!(read_client_hello(BufReader::new(reader)).await.is_err()); + } +} + +#[test] +fn tls_identity_only_uses_dns_sni_for_ip_connect_and_keeps_the_port() { + for (connect, sni, expected) in [ + ( + "192.0.2.1:4443", + Some("WWW.Example.test"), + "www.example.test:4443", + ), + ("[2001:db8::1]:443", Some("one.test"), "one.test:443"), + ("192.0.2.1:443", None, "192.0.2.1:443"), + ("[2001:db8::1]:443", Some("192.0.2.2"), "[2001:db8::1]:443"), + ("192.0.2.1:443", Some("invalid/host"), "192.0.2.1:443"), + ( + "original.test:443", + Some("different.test"), + "original.test:443", + ), + ] { + assert_eq!( + tls_identity(&connect.parse().unwrap(), sni).as_str(), + expected + ); + } +} + +#[tokio::test] +async fn connect_ip_pin_does_not_override_configured_doh_redirect_routing() { + let mut resolver = Origin::start( + vec![("localhost", origin_config("localhost", true, false))], + Protocol::Http, + ) + .await; + let mut destination = Origin::bind( + &format!("127.0.0.2:{}", resolver.addr.port()), + vec![("localhost", origin_config("localhost", true, false))], + Protocol::Http, + ) + .await; + let doh = DohConfig { + mode: crate::configuration::DohMode::Redirect, + upstream: Some(format!( + "https://localhost:{}/dns-query", + resolver.addr.port() + )), + extra_hosts: vec![], + }; + let proxy = Proxy::with_doh(LocalExclusionStore::new(vec![]), doh).await; + let tls = proxy + .connect( + &destination.addr.to_string(), + client_config(true, true), + "localhost", + ) + .await; + let (status, body) = http_request(tls, "/dns-query?dns=AA", "application/dns-message").await; + assert_eq!(status, http::StatusCode::OK, "{body}"); + let (sni, request) = resolver.request().await; + assert_eq!(sni.as_deref(), Some("localhost")); + assert!(request.starts_with(b"GET /dns-query?dns=AA HTTP/1.1")); + assert!( + destination.requests.try_recv().is_err(), + "redirect must resolve its own destination" + ); +} diff --git a/privaxy/src/server/proxy/exclusions.rs b/privaxy/src/server/proxy/exclusions.rs index c6aeb10..3d2056f 100644 --- a/privaxy/src/server/proxy/exclusions.rs +++ b/privaxy/src/server/proxy/exclusions.rs @@ -298,12 +298,30 @@ impl From<&crate::configuration::Configuration> for InterceptionRules { pub struct LocalExclusionStore(Arc>); impl LocalExclusionStore { + /// Unlike an include-only miss, an explicit exclusion can be honored + /// before looking for SNI on an IP CONNECT (including server-first flows). + pub fn is_excluded(&self, host: &str, raw_host: &str) -> bool { + let rules = self.0.read().unwrap(); + [host, raw_host] + .iter() + .any(|host| DEFAULT_EXCLUSIONS.is_match(host) || rules.exclusions.is_match(host)) + } + #[cfg(test)] pub fn new(exclusions: Vec) -> Self { + Self::with_rules(false, Vec::new(), exclusions) + } + + #[cfg(test)] + pub(super) fn with_rules( + include_only: bool, + inclusions: Vec, + exclusions: Vec, + ) -> Self { Self(Arc::new(RwLock::new(InterceptionRules { exclusions: WildMatchCollection::new(exclusions), - inclusions: WildMatchCollection::new(Vec::new()), - include_only: false, + inclusions: WildMatchCollection::new(inclusions), + include_only, }))) } diff --git a/privaxy/src/server/proxy/html_rewriter.rs b/privaxy/src/server/proxy/html_rewriter.rs index d3cfd65..7d3ae4c 100644 --- a/privaxy/src/server/proxy/html_rewriter.rs +++ b/privaxy/src/server/proxy/html_rewriter.rs @@ -427,6 +427,10 @@ impl Rewriter { endpoint_tokens, } = self; + // Head scriptlets and the trailing cosmetic stylesheet belong to one + // response. Their separate tasks share a single counting guard. + let statistics = statistics.for_response(); + let CosmeticBlockerResult { hidden_selectors, style_selectors, @@ -676,6 +680,56 @@ impl Rewriter { #[cfg(test)] mod tests { use super::*; + + #[tokio::test] + async fn head_and_body_injections_count_each_response_only_once() { + use crate::blocker::BlockingDisabledStore; + use http_body_util::BodyExt; + use std::sync::RwLock; + + let statistics = Statistics::new(); + for expected in 1..=2 { + let requester = + AdblockRequester::new(BlockingDisabledStore(Arc::new(RwLock::new(false)))); + let mut cosmetics = requester + .get_cosmetic_response("https://example.test/".into()) + .await; + cosmetics.hidden_selectors.push(".ad-test".into()); + let (input, receiver) = mpsc::channel(2); + let (sender, mut body) = body_channel(); + let rewriter = Rewriter::new( + requester, + receiver, + sender, + statistics.clone(), + "fixture".into(), + cosmetics, + false, + Vec::new(), + GmStorageStore::in_memory(), + HashMap::new(), + ); + input + .send(Ok(Bytes::from_static( + b"
ad
", + ))) + .await + .unwrap(); + drop(input); + let task = tokio::task::spawn_blocking(move || rewriter.rewrite()); + let mut output = Vec::new(); + while let Some(frame) = body.frame().await { + if let Ok(bytes) = frame.unwrap().into_data() { + output.extend_from_slice(&bytes); + } + } + task.await.unwrap(); + let html = String::from_utf8(output).unwrap(); + assert!(html.contains(", cert_cache: CertCache, broadcast_tx: broadcast::Sender, @@ -65,8 +69,9 @@ pub(crate) async fn serve_mitm_session( if Method::CONNECT == req.method() { // tun2proxy uses CONNECT for every TCP flow, including plain HTTP. // Reply before reading its payload; clients wait for the 200 first. - let intercept = - should_intercept_authority(&local_exclusion_store, &authority, &raw_authority); + let destination = authority.clone(); + let destination_ip = authority_ip(&destination); + let bypass = bypass_before_client_hello(&local_exclusion_store, &authority, &raw_authority); tokio::task::spawn(async move { let upgraded = match hyper::upgrade::on(req).await { @@ -77,34 +82,99 @@ pub(crate) async fn serve_mitm_session( } }; let mut upgraded = TokioIo::new(upgraded); - if !intercept { + if bypass { // Bypass before looking at any payload. This also preserves - // server-first protocols and avoids unnecessary cert minting. - let _ = tunnel(&mut upgraded, &authority, &raw_authority).await; + // server-first protocols. Port 853 is DNS-over-TLS, never HTTP: + // keep its end-to-end TLS and DNS framing completely opaque. + let _ = tunnel(&mut upgraded, &destination, &raw_authority, &[]).await; return; } // Keep both hyper's upgrade read-ahead and our protocol peek. The // first HTTP bytes or TLS ClientHello must reach the parser intact. let mut upgraded = BufReader::new(upgraded); - let protocol = match peek_connect_protocol(&mut upgraded).await { - Ok(Some(protocol)) => protocol, - Ok(None) => return, - Err(error) => { - log::debug!("Unable to read CONNECT payload for {raw_authority}: {error}"); - return; - } - }; + let protocol = + match tokio::time::timeout(HANDSHAKE_TIMEOUT, peek_connect_protocol(&mut upgraded)) + .await + .unwrap_or_else(|_| Err(std::io::ErrorKind::TimedOut.into())) + { + Ok(Some(protocol)) => protocol, + Ok(None) => return, + Err(error) => { + log::debug!("Unable to read CONNECT payload for {raw_authority}: {error}"); + return; + } + }; + let mut authority = authority; + let mut clients = clients; let (stream, scheme) = match protocol { - ConnectProtocol::Http => (Either::Left(upgraded), Scheme::HTTP), + ConnectProtocol::Http => { + if !should_intercept_authority( + &local_exclusion_store, + &authority, + &raw_authority, + ) { + let _ = tunnel(&mut upgraded, &destination, &raw_authority, &[]).await; + return; + } + (Either::Left(upgraded), Scheme::HTTP) + } ConnectProtocol::Tls => { - // Hostnames come from CONNECT (tun2proxy's virtual DNS - // supplies them). Mint only once we know this is TLS. + let (mut hello, prefix) = + match tokio::time::timeout(HANDSHAKE_TIMEOUT, read_client_hello(upgraded)) + .await + .unwrap_or_else(|_| Err(std::io::ErrorKind::TimedOut.into())) + { + Ok(hello) => hello, + Err(error) => { + tls_failure_store.record( + authority.host(), + &error.to_string(), + false, + ); + log::debug!( + "Unable to read ClientHello for {raw_authority}: {error}" + ); + return; + } + }; + authority = tls_identity(&destination, hello.client_hello().server_name()); + log::debug!("CONNECT {raw_authority}: TLS identity {authority}"); + if !should_intercept_authority( + &local_exclusion_store, + &authority, + &raw_authority, + ) { + let _ = tunnel(&mut hello.io, &destination, &raw_authority, &prefix).await; + return; + } + if authority != destination { + // Only IP CONNECTs can acquire a different TLS identity. + // Pin both HTTP clients to that IP and original port; + // resolving SNI could select a different server entirely. + let address = SocketAddr::new( + destination_ip.unwrap(), + destination.port_u16().unwrap_or(443), + ); + clients = match clients.for_destination(&authority, address) { + Ok(clients) => clients, + Err(error) => { + log::warn!( + "Unable to build upstream client for {raw_authority}: {error}" + ); + return; + } + }; + } + // The cache key is the effective TLS identity, not the IP. let server_configuration = Arc::new(cert_cache.get(authority.clone()).await.server_configuration); - match TlsAcceptor::from(server_configuration) - .accept(upgraded) - .await + match tokio::time::timeout( + HANDSHAKE_TIMEOUT, + hello.into_stream(server_configuration), + ) + .await + .unwrap_or_else(|_| Err(std::io::ErrorKind::TimedOut.into())) { Ok(tls_stream) => (Either::Right(tls_stream), Scheme::HTTPS), Err(error) => { @@ -131,18 +201,22 @@ pub(crate) async fn serve_mitm_session( }; // Both transports use the same filtering, rewriting and upgrade - // path. Bind requests to the CONNECT destination, not an inner - // Host header or absolute URI that could name a different host. + // path. Bind requests to the selected identity and the CONNECT + // destination; an inner Host header/URI cannot redirect the socket. let session_authority = raw_authority.clone(); - let result = auto::Builder::new(TokioExecutor::new()) + let mut builder = auto::Builder::new(TokioExecutor::new()); + builder + .http2() + .adaptive_window(true) + .max_header_list_size(MAX_HTTP2_HEADER_LIST_SIZE); + let result = builder .serve_connection_with_upgrades( TokioIo::new(stream), service_fn(move |req| { serve( adblock_requester.clone(), req, - hyper_client.clone(), - client.clone(), + clients.clone(), authority.clone(), scheme.clone(), broadcast_tx.clone(), @@ -202,8 +276,7 @@ pub(crate) async fn serve_mitm_session( serve( adblock_requester, req, - hyper_client.clone(), - client.clone(), + clients, authority, Scheme::HTTP, broadcast_tx, @@ -331,6 +404,7 @@ async fn bridge_http_upgrade( // `TokioIo` bridges hyper 1.0's `Upgraded` to tokio's IO traits. let mut client = TokioIo::new(upgraded); let mut upstream = TcpStream::connect((host, port)).await?; + let _ = upstream.set_nodelay(true); upstream.write_all(head.as_bytes()).await?; let leftover = read_past_response_headers(&mut upstream).await?; @@ -373,11 +447,17 @@ async fn read_past_response_headers(stream: &mut TcpStream) -> std::io::Result, + upgraded: &mut (impl AsyncRead + AsyncWrite + Unpin), authority: &Authority, raw_authority: &Authority, + prefix: &[u8], ) -> std::io::Result<()> { - let mut server = match TcpStream::connect(authority.to_string()).await { + let mut server = match TcpStream::connect(( + authority.host().trim_matches(['[', ']']), + authority.port_u16().unwrap_or(443), + )) + .await + { Ok(server) => server, Err(error) => { if authority == raw_authority { @@ -394,7 +474,9 @@ async fn tunnel( } }; + let _ = server.set_nodelay(true); log::debug!("Started tunneling host: {}", authority); + server.write_all(prefix).await?; // Byte counts and lifetime make tunnel health diagnosable from logs: a // tunnel that closes quickly having received 0 bytes from upstream means @@ -424,6 +506,41 @@ async fn tunnel( } } +fn authority_ip(authority: &Authority) -> Option { + authority.host().trim_matches(['[', ']']).parse().ok() +} + +fn bypass_before_client_hello( + exclusions: &LocalExclusionStore, + destination: &Authority, + raw_authority: &Authority, +) -> bool { + destination.port_u16() == Some(853) + || exclusions.is_excluded(destination.host(), raw_authority.host()) + || (authority_ip(destination).is_none() + && !should_intercept_authority(exclusions, destination, raw_authority)) +} + +/// SNI is an identity hint only for IP CONNECTs. Keep named CONNECT behavior, +/// port selection and literal-IP/no-SNI certificate verification unchanged. +fn tls_identity(destination: &Authority, sni: Option<&str>) -> Authority { + if authority_ip(destination).is_some() { + if let Some(name) = sni { + if matches!( + rustls::pki_types::ServerName::try_from(name), + Ok(rustls::pki_types::ServerName::DnsName(_)) + ) { + let name = name.to_ascii_lowercase(); + let candidate = format!("{}:{}", name, destination.port_u16().unwrap_or(443)); + if let Ok(identity) = candidate.parse() { + return identity; + } + } + } + } + destination.clone() +} + /// Strip a trailing parenthesized service selector from an authority's host. /// /// Apple's RCS client tunnels to Google's Jibe backend using CONNECT @@ -472,6 +589,10 @@ where Ok(()) } +#[cfg(test)] +#[path = "connect_tests.rs"] +mod connect_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/privaxy/src/server/proxy/mod.rs b/privaxy/src/server/proxy/mod.rs index 0c162b0..3e90f7f 100644 --- a/privaxy/src/server/proxy/mod.rs +++ b/privaxy/src/server/proxy/mod.rs @@ -1,3 +1,4 @@ +mod client_hello; pub(crate) mod doh; pub(crate) mod mitm; pub(crate) mod serve; @@ -6,6 +7,7 @@ pub(crate) mod exclusions; pub(crate) mod gm; pub(crate) mod html_rewriter; pub(crate) mod tls_failures; +pub(crate) mod upstream; pub(crate) mod userscripts; use bytes::Bytes; @@ -13,6 +15,10 @@ use http_body_util::{combinators::BoxBody, BodyExt, Empty, Full, StreamBody}; use hyper::body::Frame; use tokio_stream::wrappers::ReceiverStream; +/// Bound decoded HTTP/2 headers on both sides of intercepted connections. +/// Allow cookie-heavy requests and large CSP/Set-Cookie responses. +pub(crate) const MAX_HTTP2_HEADER_LIST_SIZE: u32 = 64 * 1024; + /// Response body type used throughout the proxy. hyper 1.0 removed the built-in /// `hyper::Body`, so we standardize on a boxed body of `Bytes` with an /// `io::Error` error type (the channel/stream bodies below can surface IO diff --git a/privaxy/src/server/proxy/serve.rs b/privaxy/src/server/proxy/serve.rs index 17e481c..697018a 100644 --- a/privaxy/src/server/proxy/serve.rs +++ b/privaxy/src/server/proxy/serve.rs @@ -1,6 +1,7 @@ use super::doh::{self, DohAction}; use super::gm::endpoint as gm_endpoint; use super::html_rewriter::Rewriter; +use super::upstream::{UpgradeClient, UpstreamClients}; use super::userscripts::UserScriptContext; use super::{body_channel, boxed_incoming, empty_body, full_body, BodySender, ProxyBody}; use crate::blocker::AdblockRequester; @@ -13,23 +14,23 @@ use futures::TryStreamExt; use http::uri::{Authority, Scheme}; use http::{HeaderMap, HeaderValue, Request, Response, StatusCode, Uri}; use http_body_util::{BodyExt, BodyStream, Limited}; -use hyper::body::{Frame, Incoming}; -use hyper_rustls::HttpsConnector; -use hyper_util::client::legacy::connect::HttpConnector; -use hyper_util::client::legacy::Client as HyperClient; +use hyper::body::{Body, Frame, Incoming}; use hyper_util::rt::TokioIo; use std::net::IpAddr; use tokio::sync::broadcast; use url::Url; -/// Type of the hyper client used for upgrade tunneling (websockets, etc.). -pub(crate) type UpgradeClient = HyperClient, ProxyBody>; - /// Adapt an incoming request body into a `reqwest::Body`, preserving streaming /// (hyper 1.0 + reqwest 0.13 no longer share a body type, so we bridge via a /// `Bytes` stream). Trailer frames are dropped — proxied request bodies don't /// carry meaningful trailers. fn incoming_to_reqwest_body(incoming: Incoming) -> reqwest::Body { + // Even an empty wrap_stream body cannot be cloned. Preserve a known-empty + // body so reqwest can retry an HTTP/2 REFUSED_STREAM or graceful GOAWAY. + // Nonempty/unknown bodies still stream without buffering or replaying them. + if incoming.is_end_stream() { + return reqwest::Body::default(); + } let stream = BodyStream::new(incoming).try_filter_map(|frame| async move { Ok(frame.into_data().ok()) }); reqwest::Body::wrap_stream(stream) @@ -289,8 +290,7 @@ fn url_for_matching(uri: &Uri) -> String { pub(crate) async fn serve( adblock_requester: AdblockRequester, request: Request, - hyper_client: UpgradeClient, - client: reqwest::Client, + clients: UpstreamClients, authority: Authority, scheme: Scheme, broadcast_sender: broadcast::Sender, @@ -302,8 +302,7 @@ pub(crate) async fn serve( user_scripts: UserScriptContext, intercept: bool, ) -> Result, hyper::Error> { - let scheme_string = scheme.to_string(); - + let (client, hyper_client) = clients.proxy_clients(); let uri = match http::uri::Builder::new() .scheme(scheme) .authority(authority) @@ -338,9 +337,14 @@ pub(crate) async fn serve( .map(|collected| collected.to_bytes()) .unwrap_or_default(); - return Ok( - gm_endpoint::handle(&uri, &parts.method, &collected, &user_scripts, &client).await, - ); + return Ok(gm_endpoint::handle( + &uri, + &parts.method, + &collected, + &user_scripts, + &clients.http, + ) + .await); } let (mut parts, body) = request.into_parts(); @@ -349,14 +353,13 @@ pub(crate) async fn serve( let req = Request::from_parts(parts, body); log::debug!("{} {}", req.method(), req.uri()); - - statistics.increment_top_clients(client_ip_address); + let statistics = statistics.for_domain(uri.host()); if !intercept { let mut req = req; req.headers_mut().remove(http::header::PROXY_AUTHORIZATION); req.headers_mut().remove("proxy-connection"); - if broadcast_sender.receiver_count() > 0 { + if statistics.should_collect() && broadcast_sender.receiver_count() > 0 { let _ = broadcast_sender.send(Event { now: chrono::Utc::now(), method: req.method().to_string(), @@ -367,7 +370,7 @@ pub(crate) async fn serve( // Forward with hyper so bodies, compression and CSP remain untouched. // This path skips adblock, DoH policy and all HTML/userscript injection. let response = if req.headers().contains_key(http::header::UPGRADE) { - perform_two_ends_upgrade(req, uri, hyper_client).await + perform_two_ends_upgrade(req, uri.clone(), hyper_client).await } else { let (mut parts, body) = req.into_parts(); parts.headers.remove(http::header::HOST); @@ -385,7 +388,7 @@ pub(crate) async fn serve( } } }; - statistics.increment_proxied_requests(); + statistics.record_request(uri.host(), client_ip_address, false); return Ok(response); } @@ -403,7 +406,7 @@ pub(crate) async fn serve( log::debug!("Refusing DoH request: {}", uri); // Events only feed the live requests view; when nobody is subscribed, // building one (two Strings and a timestamp) is wasted work. - if broadcast_sender.receiver_count() > 0 { + if statistics.should_collect() && broadcast_sender.receiver_count() > 0 { let _result = broadcast_sender.send(Event { now: chrono::Utc::now(), method: req.method().to_string(), @@ -411,7 +414,7 @@ pub(crate) async fn serve( is_request_blocked: true, }); } - statistics.increment_blocked_requests(); + statistics.record_request(uri.host(), client_ip_address, true); return Ok(get_empty_response(StatusCode::BAD_GATEWAY)); } @@ -426,7 +429,7 @@ pub(crate) async fn serve( request_type, ); - if broadcast_sender.receiver_count() > 0 { + if statistics.should_collect() && broadcast_sender.receiver_count() > 0 { let _result = broadcast_sender.send(Event { now: chrono::Utc::now(), method: req.method().to_string(), @@ -436,13 +439,7 @@ pub(crate) async fn serve( } if is_request_blocked { - statistics.increment_blocked_requests(); - statistics.increment_top_blocked_paths(format!( - "{}://{}{}", - scheme_string, - uri.host().unwrap(), - uri.path() - )); + statistics.record_request(uri.host(), client_ip_address, true); // adblock-rust fuses many same-option patterns into one filter and // reports the union as the matched filter, which can be enormous; cap @@ -471,12 +468,19 @@ pub(crate) async fn serve( } _ => req.uri().to_string(), }; + // A configured resolver is a separate destination and must use its usual + // DNS/TLS checks rather than inheriting this CONNECT session's IP pin. + let (client, hyper_client) = if matches!(doh_action, DohAction::Redirect(_)) { + (clients.http, clients.upgrade) + } else { + (client, hyper_client) + }; if req.headers().contains_key(http::header::UPGRADE) { let Ok(outbound_uri) = outbound_url.parse() else { return Ok(get_empty_response(StatusCode::BAD_GATEWAY)); }; let response = perform_two_ends_upgrade(req, outbound_uri, hyper_client).await; - statistics.increment_proxied_requests(); + statistics.record_request(uri.host(), client_ip_address, false); return Ok(response); } @@ -519,20 +523,21 @@ pub(crate) async fn serve( { Ok(response) => response, Err(err) => { - log::error!("Failed to send request: {}", err); + let reason = format_error_chain(&err); + log::error!("Failed to send request: {reason}"); // `authority` was moved into the uri builder above; the host of // the rebuilt `uri` is the same portless hostname. let failing_host = uri.host(); let exclude_url = build_exclude_url(failing_host, gui_base_url.as_deref()); return Ok(get_informative_error_response( - &err.to_string(), + &reason, failing_host, exclude_url.as_deref(), )); } }; - statistics.increment_proxied_requests(); + statistics.record_request(uri.host(), client_ip_address, false); *new_response.headers_mut() = response.headers().clone(); @@ -673,6 +678,19 @@ pub(crate) async fn serve( Ok(new_response) } +/// reqwest's Display only says "error sending request"; its sources contain +/// the actual DNS, TLS, socket or HTTP/2 failure needed to diagnose a 502. +fn format_error_chain(error: &(dyn std::error::Error + 'static)) -> String { + let mut reason = error.to_string(); + let mut source = error.source(); + while let Some(error) = source { + reason.push_str(": "); + reason.push_str(&error.to_string()); + source = error.source(); + } + reason +} + /// Minimal HTML escaping for text interpolated into the proxy's error page. /// The failure reason comes from reqwest and can embed attacker-influenced /// URLs; the failing host is client-controlled. diff --git a/privaxy/src/server/proxy/upstream.rs b/privaxy/src/server/proxy/upstream.rs new file mode 100644 index 0000000..64c52c6 --- /dev/null +++ b/privaxy/src/server/proxy/upstream.rs @@ -0,0 +1,177 @@ +//! Keep the HTTP/TLS identity separate from an IP CONNECT's TCP destination. +use super::{ProxyBody, MAX_HTTP2_HEADER_LIST_SIZE}; +use http::{uri::Authority, Uri}; +use hyper_rustls::{ConfigBuilderExt, HttpsConnector}; +use hyper_util::{ + client::legacy::{connect::HttpConnector, Client}, + rt::TokioExecutor, +}; +use std::{ + net::SocketAddr, + task::{Context, Poll}, + time::Duration, +}; +use tower_service::Service; + +/// HTTP/1 client for protocol upgrades and unchanged, unfiltered responses. +pub(crate) type UpgradeClient = Client, ProxyBody>; + +/// Shared defaults for ordinary requests and clients scoped to an IP CONNECT. +/// Keep redirects disabled: each request must go through proxy policy checks. +pub(crate) fn http_client_builder() -> reqwest::ClientBuilder { + reqwest::Client::builder() + .use_rustls_tls() + .redirect(reqwest::redirect::Policy::none()) + .no_proxy() + .gzip(true) + .brotli(true) + .deflate(true) + // Bound unreachable peers and retire stale idle connections quickly. + .connect_timeout(Duration::from_secs(10)) + .pool_idle_timeout(Duration::from_secs(30)) + // Permit long polls/SSE while bounding a stalled response read. + .read_timeout(Duration::from_secs(300)) + // Avoid a slow response stalling other streams on an HTTP/2 connection. + .http2_adaptive_window(true) + // Large CSP and Set-Cookie responses can exceed hyper's 16 KiB + // default, which h2 reports as a generic stream PROTOCOL_ERROR. Allow + // 64 KiB of decoded headers (including per-field overhead) while + // retaining a finite limit for both shared and IP-pinned clients. + .http2_max_header_list_size(MAX_HTTP2_HEADER_LIST_SIZE) +} + +#[derive(Clone)] +pub(crate) struct UpstreamClients { + // Global clients remain available for policy-directed DoH redirects and + // GM requests, whose destination checks must not inherit a CONNECT pin. + pub http: reqwest::Client, + pub upgrade: UpgradeClient, + pinned: Option<(reqwest::Client, UpgradeClient)>, + upgrade_tls: rustls::ClientConfig, + #[cfg(test)] + test_roots: Option>, +} + +impl UpstreamClients { + pub fn new(http: reqwest::Client) -> std::io::Result { + let tls = rustls::ClientConfig::builder() + .with_native_roots()? + .with_no_client_auth(); + Ok(Self::from_config(http, tls)) + } + + fn from_config(http: reqwest::Client, upgrade_tls: rustls::ClientConfig) -> Self { + Self { + http, + upgrade: upgrade_client(&upgrade_tls, None), + pinned: None, + upgrade_tls, + #[cfg(test)] + test_roots: None, + } + } + + /// A separate pool per CONNECT prevents one IP (or hostname) from reusing + /// another tunnel's connection. The URL remains the TLS identity, so both + /// clients send its SNI and verify its certificate using their usual roots. + pub fn for_destination( + &self, + identity: &Authority, + destination: SocketAddr, + ) -> Result { + // The URI carries the original port. A zero resolver port lets the + // HTTP client use that URI port, including explicit default ports. + let builder = + http_client_builder().resolve(identity.host(), SocketAddr::new(destination.ip(), 0)); + #[cfg(test)] + let builder = match &self.test_roots { + Some(roots) => builder.tls_certs_only(roots.clone()), + None => builder, + }; + Ok(Self { + pinned: Some(( + builder.build()?, + upgrade_client(&self.upgrade_tls, Some((identity.clone(), destination))), + )), + ..self.clone() + }) + } + + pub fn proxy_clients(&self) -> (reqwest::Client, UpgradeClient) { + self.pinned + .clone() + .unwrap_or_else(|| (self.http.clone(), self.upgrade.clone())) + } + + #[cfg(test)] + pub(super) fn with_test_roots(roots: &[rustls::pki_types::CertificateDer<'_>]) -> Self { + let mut store = rustls::RootCertStore::empty(); + for root in roots { + store.add(root.clone().into_owned()).unwrap(); + } + let tls = rustls::ClientConfig::builder() + .with_root_certificates(store) + .with_no_client_auth(); + let roots: Vec<_> = roots + .iter() + .map(|root| reqwest::Certificate::from_der(root).unwrap()) + .collect(); + let http = http_client_builder() + .tls_certs_only(roots.clone()) + .build() + .unwrap(); + let mut clients = Self::from_config(http, tls); + clients.test_roots = Some(roots); + clients + } +} + +fn upgrade_client( + tls: &rustls::ClientConfig, + destination: Option<(Authority, SocketAddr)>, +) -> UpgradeClient { + let mut http = HttpConnector::new(); + http.enforce_http(false); + http.set_nodelay(true); + http.set_connect_timeout(Some(Duration::from_secs(10))); + http.set_keepalive(Some(Duration::from_secs(60))); + let https = hyper_rustls::HttpsConnectorBuilder::new() + .with_tls_config(tls.clone()) + .https_or_http() + .enable_http1() + .wrap_connector(DestinationConnector { http, destination }); + // Upgraded connections are consumed by their tunnels; do not pool them. + Client::builder(TokioExecutor::new()) + .pool_max_idle_per_host(0) + .build(https) +} + +/// hyper-rustls chooses SNI/verification from the original URI before calling +/// this connector. Only TCP dialing sees the CONNECT IP. Other destinations +/// (e.g. a configured DoH redirect) keep their normal routing and verification. +#[derive(Clone)] +pub(crate) struct DestinationConnector { + http: HttpConnector, + destination: Option<(Authority, SocketAddr)>, +} + +impl Service for DestinationConnector { + type Response = >::Response; + type Error = >::Error; + type Future = >::Future; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.http.poll_ready(cx) + } + + fn call(&mut self, mut uri: Uri) -> Self::Future { + if let Some((identity, destination)) = &self.destination { + if uri.authority() == Some(identity) { + let mut parts = uri.into_parts(); + parts.authority = Some(destination.to_string().parse().expect("socket authority")); + uri = Uri::from_parts(parts).expect("valid destination URI"); + } + } + self.http.call(uri) + } +} diff --git a/privaxy/src/server/statistics.rs b/privaxy/src/server/statistics.rs index 133984b..ec0c7c0 100644 --- a/privaxy/src/server/statistics.rs +++ b/privaxy/src/server/statistics.rs @@ -1,13 +1,16 @@ +use crate::configuration::{StatisticsConfig, StatisticsPrivacy}; use portable_atomic::{AtomicU64, Ordering}; use serde::Serialize; use std::{ collections::HashMap, net::IpAddr, - sync::{Arc, Mutex}, + sync::{atomic::AtomicBool, Arc, Mutex}, }; use uluru::LRUCache; -const ENTRIES_PER_STATISTICS_TABLE: u8 = 50; +pub mod history; + +const ENTRIES_PER_STATISTICS_TABLE: usize = 50; #[derive(Debug, Serialize)] pub struct SerializableStatistics { @@ -27,6 +30,9 @@ pub struct Statistics { pub modified_responses: Arc, pub top_blocked_paths: Arc>>, pub top_clients: Arc>>, + pub history: Option, + collect: bool, + response_modified: Option>, } impl Default for Statistics { @@ -43,6 +49,79 @@ impl Statistics { modified_responses: Arc::new(AtomicU64::new(0)), top_blocked_paths: Arc::new(Mutex::new(LRUCache::default())), top_clients: Arc::new(Mutex::new(HashMap::new())), + history: None, + collect: true, + response_modified: None, + } + } + + pub async fn with_history(config: StatisticsConfig) -> Self { + let path = crate::configuration::get_config_file().with_file_name("statistics.redb"); + Self { + history: Some(history::HistoryStore::start(path, config).await), + ..Self::new() + } + } + + pub async fn configure_history(&self, config: StatisticsConfig) -> Result<(), String> { + let Some(history) = &self.history else { + return Ok(()); + }; + if config == history.config() && history.status().error.is_none() { + return Ok(()); + } + let result = history.configure(config).await; + // Discard session identifiers too. Lowering privacy later must not + // resurrect identifiers from before the change. + self.top_blocked_paths.lock().unwrap().clear(); + self.top_clients.lock().unwrap().clear(); + result + } + + /// Carry one domain's collection decision through response rewriting too. + /// Statistics exclusions never change interception or blocking decisions. + pub fn for_domain(&self, domain: Option<&str>) -> Self { + Self { + collect: self.collect && !self.history.as_ref().is_some_and(|h| h.is_excluded(domain)), + ..self.clone() + } + } + + pub fn should_collect(&self) -> bool { + self.collect + } + + pub(crate) fn for_response(&self) -> Self { + Self { + response_modified: Some(Arc::new(AtomicBool::new(false))), + ..self.clone() + } + } + + pub fn record_request(&self, domain: Option<&str>, client: IpAddr, blocked: bool) { + if !self.collect || self.history.as_ref().is_some_and(|h| h.is_excluded(domain)) { + return; + } + let privacy = self + .history + .as_ref() + .map(|h| h.session_privacy()) + .unwrap_or(StatisticsPrivacy::Full); + if privacy < StatisticsPrivacy::HideDomainsAndClients { + self.increment_top_clients(client); + } + if blocked { + self.increment_blocked_requests(); + if privacy == StatisticsPrivacy::Full { + if let Some(domain) = domain { + self.increment_top_blocked_paths(domain.to_ascii_lowercase()); + } + } + } else { + self.increment_proxied_requests(); + } + if let Some(history) = &self.history { + history.request(domain, client, blocked); } } @@ -60,7 +139,10 @@ impl Statistics { } pub fn increment_top_clients(&self, client: IpAddr) { - *self.top_clients.lock().unwrap().entry(client).or_insert(0) += 1; + let mut clients = self.top_clients.lock().unwrap(); + if clients.len() < 1000 || clients.contains_key(&client) { + *clients.entry(client).or_insert(0) += 1; + } } pub fn increment_proxied_requests(&self) -> u64 { @@ -72,46 +154,55 @@ impl Statistics { } pub fn increment_modified_responses(&self) -> u64 { + if !self.collect + || self + .response_modified + .as_ref() + .is_some_and(|modified| modified.swap(true, Ordering::Relaxed)) + { + return self.modified_responses.load(Ordering::Relaxed); + } + if let Some(history) = &self.history { + history.modified(); + } self.modified_responses.fetch_add(1, Ordering::Relaxed) + 1 } pub fn get_serialized(&self) -> SerializableStatistics { - SerializableStatistics { + let mut serialized = SerializableStatistics { proxied_requests: self.proxied_requests.load(Ordering::Relaxed), blocked_requests: self.blocked_requests.load(Ordering::Relaxed), modified_responses: self.modified_responses.load(Ordering::Relaxed), top_blocked_paths: { let top_blocked_paths = self.top_blocked_paths.lock().unwrap(); - let mut top_blocked_paths_iterator = top_blocked_paths.iter(); - - let mut top_blocked_paths = (0..=ENTRIES_PER_STATISTICS_TABLE) - .filter_map(|_| { - let (path, count) = top_blocked_paths_iterator.next()?; - - Some((path.clone(), *count)) - }) - .collect::>(); - - top_blocked_paths.sort_by_key(|entry| std::cmp::Reverse(entry.1)); + let mut top_blocked_paths = top_blocked_paths.iter().cloned().collect::>(); + top_blocked_paths + .sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + top_blocked_paths.truncate(ENTRIES_PER_STATISTICS_TABLE); top_blocked_paths }, top_clients: { let top_clients = self.top_clients.lock().unwrap(); - let mut top_clients_iter = top_clients.iter(); - - let mut top_clients = (0..=ENTRIES_PER_STATISTICS_TABLE) - .filter_map(|_| { - let (ipv4, count) = top_clients_iter.next()?; - - Some((ipv4.to_string(), *count)) - }) + let mut top_clients = top_clients + .iter() + .map(|(ip, count)| (ip.to_string(), *count)) .collect::>(); - - top_clients.sort_by_key(|entry| std::cmp::Reverse(entry.1)); + top_clients.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + top_clients.truncate(ENTRIES_PER_STATISTICS_TABLE); top_clients }, + }; + if let Some(history) = &self.history { + let privacy = history.session_privacy(); + if privacy >= StatisticsPrivacy::HideDomains { + serialized.top_blocked_paths.clear(); + } + if privacy >= StatisticsPrivacy::HideDomainsAndClients { + serialized.top_clients.clear(); + } } + serialized } } diff --git a/privaxy/src/server/statistics/history.rs b/privaxy/src/server/statistics/history.rs new file mode 100644 index 0000000..8e29b41 --- /dev/null +++ b/privaxy/src/server/statistics/history.rs @@ -0,0 +1,722 @@ +//! Opt-in, bounded request history. Only the worker thread touches redb. +//! Request tasks redact identifiers before trying a non-blocking channel send. +use crate::configuration::{StatisticsConfig, StatisticsPrivacy}; +use portable_atomic::{AtomicU64, Ordering}; +use redb::{Database, ReadableDatabase, ReadableTable, ReadableTableMetadata, TableDefinition}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, HashMap}, + fs::OpenOptions, + net::IpAddr, + path::PathBuf, + sync::{mpsc, Arc, Mutex, RwLock}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; +use tokio::sync::{oneshot, watch}; + +const REQUESTS: TableDefinition = TableDefinition::new("requests_v1"); +const MODIFIED: TableDefinition = TableDefinition::new("modified_hours_v1"); +const META: TableDefinition<&str, u64> = TableDefinition::new("metadata"); +const FLUSH_INTERVAL: Duration = Duration::from_secs(5); +const QUEUE_CAPACITY: usize = 4096; +const BATCH_SIZE: usize = 512; +const HOUR_MS: u64 = 3_600_000; +const DAY_MS: u64 = 24 * HOUR_MS; +type StoreResult = Result>; + +#[derive(Debug, Clone, Default, Serialize)] +pub struct HistoryStatus { + pub active: bool, + pub error: Option, + pub dropped_events: u64, + pub stored_requests: u64, + pub database_bytes: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RequestRecord { + pub id: u64, + pub timestamp: u64, + pub domain: Option, + pub client: Option, + pub blocked: bool, +} + +impl RequestRecord { + fn redact(&mut self, privacy: StatisticsPrivacy) { + if privacy >= StatisticsPrivacy::HideDomains { + self.domain = None; + } + if privacy >= StatisticsPrivacy::HideDomainsAndClients { + self.client = None; + } + } +} + +#[derive(Debug, Clone, Copy, Default, Serialize)] +pub struct Counts { + pub proxied_requests: u64, + pub blocked_requests: u64, + pub modified_responses: u64, +} + +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct HistoryQuery { + pub hours: Option, + pub before: Option, + pub blocked: Option, + pub limit: Option, +} + +impl HistoryQuery { + pub fn validate(&self) -> Result<(), String> { + if self.hours.is_some_and(|v| v == 0 || v > 365 * 24) { + return Err("History window must be between 1 and 8760 hours".into()); + } + if self.limit.is_some_and(|v| v == 0 || v > 200) { + return Err("History page size must be between 1 and 200".into()); + } + Ok(()) + } +} + +#[derive(Debug, Default, Serialize)] +pub struct HistorySnapshot { + pub status: HistoryStatus, + pub totals: Counts, + pub hourly: Vec<(u64, Counts)>, + pub top_domains: Vec<(String, u64)>, + pub top_blocked_domains: Vec<(String, u64)>, + pub top_clients: Vec<(String, u64)>, + pub requests: Vec, + pub next_before: Option, +} + +#[derive(Debug, Default)] +struct Policy { + config: StatisticsConfig, + generation: u64, + stopped: bool, + exclusions: Vec, +} + +enum Event { + Request(RequestRecord), + Modified(u64), +} + +enum Command { + Event(u64, Event), + Configure(StatisticsConfig, u64, oneshot::Sender>), + Query { + query: HistoryQuery, + flush: bool, + reply: oneshot::Sender>, + }, + Clear(u64, oneshot::Sender>), + Shutdown(oneshot::Sender>), +} + +#[derive(Debug, Clone)] +pub struct HistoryStore { + sender: mpsc::SyncSender, + policy: Arc>, + status: Arc>, + dropped: Arc, + operations: Arc>, + updates: watch::Receiver<()>, +} + +impl HistoryStore { + pub async fn start(path: PathBuf, config: StatisticsConfig) -> Self { + let (sender, receiver) = mpsc::sync_channel(QUEUE_CAPACITY); + let (updates, subscriber) = watch::channel(()); + let status = Arc::new(Mutex::new(HistoryStatus::default())); + let dropped = Arc::new(AtomicU64::new(0)); + let store = Self { + sender, + policy: Arc::new(RwLock::new(Policy::default())), + status: status.clone(), + dropped: dropped.clone(), + operations: Arc::new(tokio::sync::Mutex::new(())), + updates: subscriber, + }; + let worker = Worker { + path, + database: None, + config: StatisticsConfig::default(), + generation: 0, + pending: Vec::new(), + status, + dropped, + last_id: 0, + last_flush: Instant::now(), + last_compaction: Instant::now(), + last_prune: Instant::now() - Duration::from_secs(60), + updates, + }; + match std::thread::Builder::new() + .name("statistics".into()) + .spawn(move || worker.run(receiver)) + { + Ok(_) => { + if let Err(error) = store.configure(config).await { + log::error!("Persistent statistics unavailable: {error}"); + store.status.lock().unwrap().error = Some(error); + } + } + Err(error) => store.status.lock().unwrap().error = Some(error.to_string()), + } + store + } + + pub fn config(&self) -> StatisticsConfig { + self.policy.read().unwrap().config.clone() + } + + pub fn session_privacy(&self) -> StatisticsPrivacy { + let policy = self.policy.read().unwrap(); + if policy.config.enabled { + policy.config.privacy + } else { + StatisticsPrivacy::Full + } + } + + pub fn status(&self) -> HistoryStatus { + let mut status = self.status.lock().unwrap().clone(); + status.dropped_events = self.dropped.load(Ordering::Relaxed); + status + } + + pub fn is_excluded(&self, domain: Option<&str>) -> bool { + let policy = self.policy.read().unwrap(); + domain.is_some_and(|domain| matches_exclusion(&policy, domain)) + } + + pub fn request(&self, domain: Option<&str>, client: IpAddr, blocked: bool) { + let policy = self.policy.read().unwrap(); + if policy.stopped + || !policy.config.persists() + || domain.is_some_and(|domain| matches_exclusion(&policy, domain)) + { + return; + } + // No URL, path, query string, headers, request body, or client/domain hash + // is ever queued. Hidden values cannot be restored by changing settings. + let record = RequestRecord { + id: 0, + timestamp: now_ms(), + domain: if policy.config.privacy == StatisticsPrivacy::Full { + domain + .filter(|v| v.len() <= 253) + .map(|v| v.trim_end_matches('.').to_ascii_lowercase()) + } else { + None + }, + client: (policy.config.privacy < StatisticsPrivacy::HideDomainsAndClients) + .then(|| client.to_string()), + blocked, + }; + self.enqueue(&policy, Event::Request(record)); + } + + pub fn modified(&self) { + let policy = self.policy.read().unwrap(); + if !policy.stopped && policy.config.persists() { + self.enqueue(&policy, Event::Modified(now_ms())); + } + } + + fn enqueue(&self, policy: &Policy, event: Event) { + if self + .sender + .try_send(Command::Event(policy.generation, event)) + .is_err() + { + self.dropped.fetch_add(1, Ordering::Relaxed); + } + } + + async fn send( + &self, + command: Command, + reply: oneshot::Receiver>, + ) -> Result { + let sender = self.sender.clone(); + tokio::task::spawn_blocking(move || sender.send(command)) + .await + .map_err(|e| e.to_string())? + .map_err(|_| "Statistics worker has stopped".to_string())?; + reply + .await + .map_err(|_| "Statistics worker has stopped".to_string())? + } + + pub async fn configure(&self, config: StatisticsConfig) -> Result<(), String> { + config.validate()?; + let _operation = self.operations.lock().await; + // Pause collection across the command boundary. An old queued event + // cannot reintroduce identifiers after a privacy change or clear. + let generation = { + let mut policy = self.policy.write().unwrap(); + policy.stopped = true; + policy.generation += 1; + policy.generation + }; + let (tx, rx) = oneshot::channel(); + let result = self + .send(Command::Configure(config.clone(), generation, tx), rx) + .await; + let mut policy = self.policy.write().unwrap(); + policy.exclusions = config + .excluded_domains + .iter() + .map(|pattern| { + wildmatch::WildMatch::new( + &pattern.trim().trim_end_matches('.').to_ascii_lowercase(), + ) + }) + .collect(); + policy.config = config; + policy.stopped = result.is_err(); + result + } + + pub async fn query(&self, query: HistoryQuery) -> Result { + self.read(query, true).await + } + + /// Live subscribers read committed batches without forcing extra disk writes. + pub(crate) async fn query_committed( + &self, + query: HistoryQuery, + ) -> Result { + self.read(query, false).await + } + + pub(crate) fn subscribe(&self) -> watch::Receiver<()> { + self.updates.clone() + } + + async fn read(&self, query: HistoryQuery, flush: bool) -> Result { + query.validate()?; + let (tx, rx) = oneshot::channel(); + self.send( + Command::Query { + query, + flush, + reply: tx, + }, + rx, + ) + .await + } + + pub async fn clear(&self) -> Result<(), String> { + let _operation = self.operations.lock().await; + let generation = { + let mut policy = self.policy.write().unwrap(); + policy.stopped = true; + policy.generation += 1; + policy.generation + }; + let (tx, rx) = oneshot::channel(); + let result = self.send(Command::Clear(generation, tx), rx).await; + self.policy.write().unwrap().stopped = result.is_err(); + result + } + + pub async fn shutdown(&self) -> Result<(), String> { + let _operation = self.operations.lock().await; + self.policy.write().unwrap().stopped = true; + let (tx, rx) = oneshot::channel(); + self.send(Command::Shutdown(tx), rx).await + } +} + +struct Worker { + path: PathBuf, + database: Option, + config: StatisticsConfig, + generation: u64, + pending: Vec, + status: Arc>, + dropped: Arc, + last_id: u64, + last_flush: Instant, + last_compaction: Instant, + last_prune: Instant, + updates: watch::Sender<()>, +} + +impl Worker { + fn run(mut self, receiver: mpsc::Receiver) { + loop { + match receiver.recv_timeout(Duration::from_secs(1)) { + Ok(Command::Event(generation, event)) => { + if generation == self.generation + && self.database.is_some() + && self.status.lock().unwrap().error.is_none() + { + self.pending.push(event); + } else { + self.dropped.fetch_add(1, Ordering::Relaxed); + } + } + Ok(Command::Configure(config, generation, reply)) => { + let result = self + .configure(config, generation) + .map_err(|e| e.to_string()); + self.report(&result); + self.updates.send_replace(()); + let _ = reply.send(result); + } + Ok(Command::Query { + query, + flush, + reply, + }) => { + let result = (if flush { self.flush() } else { Ok(()) }) + .and_then(|()| self.query(query)) + .map_err(|e| e.to_string()); + if let Err(ref error) = result { + self.fail(error.clone()); + } + let _ = reply.send(result); + } + Ok(Command::Clear(generation, reply)) => { + self.generation = generation; + self.pending.clear(); + self.database = None; + let result = self.clear().map_err(|e| e.to_string()); + self.report(&result); + self.updates.send_replace(()); + let _ = reply.send(result); + } + Ok(Command::Shutdown(reply)) => { + let result = self.flush().map_err(|e| e.to_string()); + self.database = None; + let _ = reply.send(result); + break; + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + if let Err(error) = self.flush() { + self.fail(error.to_string()); + } + break; + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + if self.pending.len() >= BATCH_SIZE || self.last_flush.elapsed() >= FLUSH_INTERVAL { + if let Err(error) = self.flush() { + self.fail(error.to_string()); + } + self.last_flush = Instant::now(); + } + } + } + + fn fail(&mut self, error: String) { + let mut status = self.status.lock().unwrap(); + let changed = status.active || status.error.as_ref() != Some(&error); + if status.error.as_ref() != Some(&error) { + log::error!("Persistent statistics paused: {error}"); + } + status.active = false; + status.error = Some(error); + self.dropped + .fetch_add(self.pending.len() as u64, Ordering::Relaxed); + self.pending.clear(); + if changed { + self.updates.send_replace(()); + } + } + + fn report(&mut self, result: &Result<(), String>) { + if let Err(error) = result { + self.fail(error.clone()); + } + } + + fn configure(&mut self, config: StatisticsConfig, generation: u64) -> StoreResult<()> { + // Previously accepted events still have the previous privacy policy. + // Flush them before redacting the database with the new policy. + let flush_result = self.flush(); + if let Err(ref error) = flush_result { + self.fail(error.to_string()); + } + self.pending.clear(); + self.database = None; + self.config = config; + self.generation = generation; + *self.status.lock().unwrap() = HistoryStatus::default(); + if !self.config.persists() { + // Disabled stores are never opened. Anonymous mode may redact a + // database from an earlier opt-in, but never creates one or records + // new events. Switching back cannot recover old identifiers. + if self.config.enabled && self.path.exists() { + self.open()?; + self.database = None; + self.status.lock().unwrap().active = false; + } + return flush_result; + } + self.open()?; + self.last_prune = Instant::now() - Duration::from_secs(60); + self.flush()?; + Ok(()) + } + + fn open(&mut self) -> StoreResult<()> { + let mut options = OpenOptions::new(); + options.read(true).write(true).create(true).truncate(false); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let file = options.open(&self.path)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(std::fs::Permissions::from_mode(0o600))?; + } + let mut database = Database::builder() + .set_cache_size(8 * 1024 * 1024) + .create_file(file)?; + let tx = database.begin_write()?; + let redact; + { + let mut meta = tx.open_table(META)?; + if meta.get("version")?.is_some_and(|v| v.value() != 1) { + return Err("Unsupported statistics database version".into()); + } + meta.insert("version", 1)?; + let previous = meta.get("privacy")?.map(|v| v.value()).unwrap_or(0); + redact = (self.config.privacy as u64) > previous; + meta.insert("privacy", self.config.privacy as u64)?; + } + { + let mut table = tx.open_table(REQUESTS)?; + self.last_id = table.last()?.map(|(id, _)| id.value()).unwrap_or(0); + if redact { + let ids = table + .iter()? + .map(|r| r.map(|(k, _)| k.value())) + .collect::, _>>()?; + for id in ids { + let mut record: RequestRecord = + serde_json::from_slice(table.get(id)?.unwrap().value())?; + record.redact(self.config.privacy); + table.insert(id, serde_json::to_vec(&record)?.as_slice())?; + } + } + } + tx.open_table(MODIFIED)?; + tx.commit()?; + if redact { + database.compact()?; + } + self.database = Some(database); + self.status.lock().unwrap().active = true; + Ok(()) + } + + fn clear(&mut self) -> StoreResult<()> { + match std::fs::remove_file(&self.path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + self.last_id = 0; + self.dropped.store(0, Ordering::Relaxed); + *self.status.lock().unwrap() = HistoryStatus::default(); + if self.config.persists() { + self.open()?; + } + Ok(()) + } + + fn flush(&mut self) -> StoreResult<()> { + if self.status.lock().unwrap().error.is_some() { + return Ok(()); + } + let Some(database) = self.database.as_ref() else { + return Ok(()); + }; + let prune = self.last_prune.elapsed() >= Duration::from_secs(60); + if self.pending.is_empty() && !prune { + return Ok(()); + } + let now = now_ms(); + let cutoff = now.saturating_sub(u64::from(self.config.retention_days) * DAY_MS); + let tx = database.begin_write()?; + let stored; + { + let mut requests = tx.open_table(REQUESTS)?; + let mut modified = tx.open_table(MODIFIED)?; + for event in &self.pending { + match event { + Event::Request(record) => { + let mut record = record.clone(); + // Microsecond-sized IDs remain ordered and fit exactly in + // JavaScript numbers for centuries. The clock may step back. + self.last_id = (record.timestamp * 1000).max(self.last_id + 1); + record.id = self.last_id; + requests.insert(record.id, serde_json::to_vec(&record)?.as_slice())?; + } + Event::Modified(timestamp) => { + let hour = timestamp / HOUR_MS * HOUR_MS; + let count = modified.get(hour)?.map(|v| v.value()).unwrap_or(0) + 1; + modified.insert(hour, count)?; + } + } + } + // Also inspect stored timestamps: an NTP clock adjustment must not + // make old rows immortal merely because their IDs are monotonic. + if prune { + let mut invalid_row = false; + requests.retain(|_, bytes| { + match serde_json::from_slice::(bytes) { + Ok(record) => record.timestamp >= cutoff, + Err(_) => { + invalid_row = true; + true + } + } + })?; + if invalid_row { + return Err("Invalid record in statistics database; recording paused".into()); + } + } + while requests.len()? > u64::from(self.config.max_entries) { + let first = requests.first()?.unwrap().0.value(); + requests.remove(first)?; + } + modified.retain(|hour, _| hour + HOUR_MS > cutoff)?; + stored = requests.len()?; + } + tx.commit()?; + self.pending.clear(); + if prune { + self.last_prune = Instant::now(); + } + if self.last_compaction.elapsed() >= Duration::from_secs(3600) { + self.database.as_mut().unwrap().compact()?; + self.last_compaction = Instant::now(); + } + let mut status = self.status.lock().unwrap(); + status.stored_requests = stored; + status.database_bytes = std::fs::metadata(&self.path)?.len(); + self.updates.send_replace(()); + Ok(()) + } + + fn query(&self, query: HistoryQuery) -> StoreResult { + let mut snapshot = HistorySnapshot { + status: self.status.lock().unwrap().clone(), + ..Default::default() + }; + snapshot.status.dropped_events = self.dropped.load(Ordering::Relaxed); + let Some(database) = &self.database else { + return Ok(snapshot); + }; + // A failed privacy transition must never reveal previously collected data. + if snapshot.status.error.is_some() { + return Ok(snapshot); + } + let hours = query + .hours + .unwrap_or(24) + .min(self.config.retention_days * 24); + let cutoff = (now_ms() / HOUR_MS * HOUR_MS).saturating_sub(u64::from(hours - 1) * HOUR_MS); + let limit = query.limit.unwrap_or(50); + let tx = database.begin_read()?; + let table = tx.open_table(REQUESTS)?; + let mut hourly: BTreeMap = BTreeMap::new(); + let mut domains = HashMap::new(); + let mut blocked_domains = HashMap::new(); + let mut clients = HashMap::new(); + // IDs are at least timestamp * 1000, even after a clock correction, + // so this range skips older pages without losing in-window records. + for entry in table.range(cutoff.saturating_mul(1000)..)?.rev() { + let (_, bytes) = entry?; + let record: RequestRecord = serde_json::from_slice(bytes.value())?; + if record.timestamp < cutoff { + continue; + } + let counts = hourly + .entry(record.timestamp / HOUR_MS * HOUR_MS) + .or_default(); + if record.blocked { + snapshot.totals.blocked_requests += 1; + counts.blocked_requests += 1; + } else { + snapshot.totals.proxied_requests += 1; + counts.proxied_requests += 1; + } + if let Some(ref domain) = record.domain { + *domains.entry(domain.clone()).or_insert(0) += 1; + if record.blocked { + *blocked_domains.entry(domain.clone()).or_insert(0) += 1; + } + } + if let Some(ref client) = record.client { + *clients.entry(client.clone()).or_insert(0) += 1; + } + if query.before.is_none_or(|id| record.id < id) + && query.blocked.is_none_or(|v| record.blocked == v) + { + if snapshot.requests.len() < limit { + snapshot.requests.push(record); + } else if snapshot.next_before.is_none() { + snapshot.next_before = snapshot.requests.last().map(|r| r.id); + } + } + } + for entry in tx + .open_table(MODIFIED)? + .range(cutoff / HOUR_MS * HOUR_MS..)? + { + let (hour, value) = entry?; + let count = value.value(); + snapshot.totals.modified_responses += count; + hourly.entry(hour.value()).or_default().modified_responses = count; + } + snapshot.hourly = hourly.into_iter().collect(); + snapshot.top_domains = top_entries(domains); + snapshot.top_blocked_domains = top_entries(blocked_domains); + snapshot.top_clients = top_entries(clients); + Ok(snapshot) + } +} + +fn top_entries(entries: HashMap) -> Vec<(String, u64)> { + let mut entries: Vec<_> = entries.into_iter().collect(); + entries.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + entries.truncate(50); + entries +} + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + +fn matches_exclusion(policy: &Policy, domain: &str) -> bool { + if policy.exclusions.is_empty() { + return false; + } + let domain = domain.trim_end_matches('.').to_ascii_lowercase(); + policy + .exclusions + .iter() + .any(|pattern| pattern.matches(&domain)) +} + +#[cfg(test)] +mod tests; diff --git a/privaxy/src/server/statistics/history/tests.rs b/privaxy/src/server/statistics/history/tests.rs new file mode 100644 index 0000000..797ddfe --- /dev/null +++ b/privaxy/src/server/statistics/history/tests.rs @@ -0,0 +1,528 @@ +use super::*; +use crate::statistics::Statistics; + +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "privaxy-stats-test-{}", + crate::configuration::generate_random_hex(12) + )); + std::fs::create_dir(&path).unwrap(); + Self(path) + } + fn database(&self) -> PathBuf { + self.0.join("statistics.redb") + } +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn enabled(privacy: StatisticsPrivacy) -> StatisticsConfig { + StatisticsConfig { + enabled: true, + privacy, + ..Default::default() + } +} +fn client() -> IpAddr { + "192.0.2.10".parse().unwrap() +} +async fn snapshot(store: &HistoryStore) -> HistorySnapshot { + store.query(HistoryQuery::default()).await.unwrap() +} + +#[tokio::test] +async fn live_subscribers_receive_committed_batches_without_polling() { + let fixture = Fixture::new(); + let store = HistoryStore::start(fixture.database(), enabled(StatisticsPrivacy::Full)).await; + let mut updates = store.subscribe(); + updates.borrow_and_update(); + store.request(Some("allowed.example.test"), client(), false); + store.request(Some("blocked.example.test"), client(), true); + store.modified(); + + // No query is used to force a flush: the normal batching timer wakes viewers. + tokio::time::timeout(Duration::from_secs(10), updates.changed()) + .await + .unwrap() + .unwrap(); + let data = store + .query_committed(HistoryQuery::default()) + .await + .unwrap(); + assert_eq!(data.totals.proxied_requests, 1); + assert_eq!(data.totals.blocked_requests, 1); + assert_eq!(data.totals.modified_responses, 1); + assert_eq!(data.requests.len(), 2); + // Reading a snapshot must not trigger another update loop. + assert!(!updates.has_changed().unwrap()); + store.shutdown().await.unwrap(); + assert!(updates.changed().await.is_err()); +} + +#[tokio::test] +async fn live_subscribers_see_privacy_changes_clears_and_disabled_history() { + let fixture = Fixture::new(); + let store = HistoryStore::start(fixture.database(), enabled(StatisticsPrivacy::Full)).await; + store.request(Some("private.example.test"), client(), true); + snapshot(&store).await; + let mut updates = store.subscribe(); + updates.borrow_and_update(); + + store + .configure(enabled(StatisticsPrivacy::HideDomainsAndClients)) + .await + .unwrap(); + assert!(updates.has_changed().unwrap()); + updates.borrow_and_update(); + let data = store + .query_committed(HistoryQuery::default()) + .await + .unwrap(); + assert!(data.requests[0].domain.is_none()); + assert!(data.requests[0].client.is_none()); + assert!(data.top_domains.is_empty()); + assert!(data.top_clients.is_empty()); + + store.clear().await.unwrap(); + assert!(updates.has_changed().unwrap()); + updates.borrow_and_update(); + let data = store + .query_committed(HistoryQuery::default()) + .await + .unwrap(); + assert_eq!(data.totals.blocked_requests, 0); + assert!(data.requests.is_empty()); + + store.configure(StatisticsConfig::default()).await.unwrap(); + assert!(updates.has_changed().unwrap()); + assert!( + !store + .query_committed(HistoryQuery::default()) + .await + .unwrap() + .status + .active + ); + store.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn disabled_and_anonymous_do_not_create_databases_or_retroactively_collect() { + for config in [ + StatisticsConfig::default(), + enabled(StatisticsPrivacy::Anonymous), + ] { + let fixture = Fixture::new(); + let store = HistoryStore::start(fixture.database(), config).await; + store.request(Some("private.example.test"), client(), true); + store.modified(); + assert!(snapshot(&store).await.requests.is_empty()); + assert!(!fixture.database().exists()); + store + .configure(enabled(StatisticsPrivacy::Full)) + .await + .unwrap(); + assert!(snapshot(&store).await.requests.is_empty()); + store.shutdown().await.unwrap(); + } +} + +#[tokio::test] +async fn requests_and_modified_counts_survive_restart_without_url_details() { + let fixture = Fixture::new(); + let config = enabled(StatisticsPrivacy::Full); + let store = HistoryStore::start(fixture.database(), config.clone()).await; + let uri: http::Uri = "https://EXAMPLE.test:443/private-account?secret=do-not-store" + .parse() + .unwrap(); + store.request(uri.host(), client(), true); + store.request(Some("allowed.example.test"), client(), false); + store.modified(); + store.shutdown().await.unwrap(); + let store = HistoryStore::start(fixture.database(), config).await; + let data = snapshot(&store).await; + assert_eq!(data.totals.blocked_requests, 1); + assert_eq!(data.totals.proxied_requests, 1); + assert_eq!(data.totals.modified_responses, 1); + assert_eq!(data.requests[1].domain.as_deref(), Some("example.test")); + assert_eq!(data.requests[1].client.as_deref(), Some("192.0.2.10")); + assert_eq!(data.top_blocked_domains, vec![("example.test".into(), 1)]); + store.shutdown().await.unwrap(); + let bytes = std::fs::read(fixture.database()).unwrap(); + let text = String::from_utf8_lossy(&bytes); + assert!(!text.contains("private-account")); + assert!(!text.contains("do-not-store")); + assert!(!text.contains(":443")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(fixture.database()) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + } +} + +#[tokio::test] +async fn identifiers_are_redacted_before_queueing_and_cannot_be_recovered() { + for privacy in [ + StatisticsPrivacy::HideDomains, + StatisticsPrivacy::HideDomainsAndClients, + ] { + let fixture = Fixture::new(); + let store = HistoryStore::start(fixture.database(), enabled(privacy)).await; + store.request(Some("private.example.test"), client(), true); + store + .configure(enabled(StatisticsPrivacy::Full)) + .await + .unwrap(); + let data = snapshot(&store).await; + assert_eq!(data.requests.len(), 1); + assert!(data.requests[0].domain.is_none()); + assert_eq!( + data.requests[0].client.is_none(), + privacy == StatisticsPrivacy::HideDomainsAndClients + ); + store.shutdown().await.unwrap(); + let bytes = std::fs::read(fixture.database()).unwrap(); + let text = String::from_utf8_lossy(&bytes); + assert!(!text.contains("private.example.test")); + if privacy == StatisticsPrivacy::HideDomainsAndClients { + assert!(!text.contains("192.0.2.10")); + } + } +} + +#[tokio::test] +async fn increasing_privacy_redacts_old_rows_including_before_anonymous_mode() { + for privacy in [ + StatisticsPrivacy::HideDomains, + StatisticsPrivacy::HideDomainsAndClients, + StatisticsPrivacy::Anonymous, + ] { + let fixture = Fixture::new(); + let store = HistoryStore::start(fixture.database(), enabled(StatisticsPrivacy::Full)).await; + store.request(Some("private.example.test"), client(), true); + // The pending record must also be covered by this transition. + store.configure(enabled(privacy)).await.unwrap(); + store + .configure(enabled(StatisticsPrivacy::Full)) + .await + .unwrap(); + let data = snapshot(&store).await; + assert_eq!(data.requests.len(), 1); + assert!(data.requests[0].domain.is_none()); + assert_eq!( + data.requests[0].client.is_none(), + privacy >= StatisticsPrivacy::HideDomainsAndClients + ); + store.shutdown().await.unwrap(); + } +} + +#[tokio::test] +async fn retention_entry_cap_and_cursor_pagination_are_enforced() { + let fixture = Fixture::new(); + let mut config = enabled(StatisticsPrivacy::Full); + config.max_entries = 100; + let store = HistoryStore::start(fixture.database(), config.clone()).await; + for i in 0..150 { + store.request( + Some(&format!("host-{i}.example.test")), + client(), + i % 2 == 0, + ); + } + let query = || HistoryQuery { + limit: Some(10), + blocked: Some(true), + ..Default::default() + }; + let first = store.query(query()).await.unwrap(); + assert_eq!(first.status.stored_requests, 100); + assert_eq!(first.totals.blocked_requests, 50); + assert_eq!(first.totals.proxied_requests, 50); + assert!(first.requests.iter().all(|r| r.blocked)); + assert_eq!(first.requests.len(), 10); + let second = store + .query(HistoryQuery { + before: first.next_before, + ..query() + }) + .await + .unwrap(); + assert_eq!(second.requests.len(), 10); + assert!(second + .requests + .iter() + .all(|r| r.id < first.requests.last().unwrap().id)); + assert_eq!(second.totals.blocked_requests, 50); + let generation = store.policy.read().unwrap().generation; + store + .sender + .send(Command::Event( + generation, + Event::Request(RequestRecord { + id: 0, + timestamp: now_ms() - 2 * DAY_MS, + domain: Some("old.example.test".into()), + client: None, + blocked: true, + }), + )) + .unwrap(); + config.retention_days = 1; + store.configure(config).await.unwrap(); + let data = snapshot(&store).await; + // Old timestamp with a newer ID is pruned too (clock corrections). + assert_eq!(data.status.stored_requests, 99); + store.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn clear_and_disable_do_not_replay_queued_records() { + let fixture = Fixture::new(); + let config = enabled(StatisticsPrivacy::Full); + let store = HistoryStore::start(fixture.database(), config.clone()).await; + for _ in 0..100 { + store.request(Some("old.example.test"), client(), true); + } + store.clear().await.unwrap(); + assert!(snapshot(&store).await.requests.is_empty()); + store.configure(StatisticsConfig::default()).await.unwrap(); + store.request(Some("disabled.example.test"), client(), false); + store.clear().await.unwrap(); + assert!(!fixture.database().exists()); + store.configure(config).await.unwrap(); + assert!(snapshot(&store).await.requests.is_empty()); + store.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn corrupt_storage_is_reported_without_truncation_or_breaking_session_counts() { + let fixture = Fixture::new(); + std::fs::write(fixture.database(), b"corrupt-fixture-do-not-truncate").unwrap(); + let history = HistoryStore::start(fixture.database(), enabled(StatisticsPrivacy::Full)).await; + assert!(history.status().error.is_some()); + assert_eq!( + std::fs::read(fixture.database()).unwrap(), + b"corrupt-fixture-do-not-truncate" + ); + let stats = Statistics { + history: Some(history.clone()), + ..Statistics::new() + }; + stats.record_request(Some("example.test"), client(), true); + assert_eq!(stats.get_serialized().blocked_requests, 1); + history.clear().await.unwrap(); + assert!(history.status().error.is_none()); + stats.record_request(Some("example.test"), client(), true); + assert_eq!(snapshot(&history).await.totals.blocked_requests, 1); + history.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn excluded_domains_skip_counters_history_and_the_live_log_even_when_disabled() { + for persistent in [false, true] { + let fixture = Fixture::new(); + let config = StatisticsConfig { + enabled: persistent, + privacy: StatisticsPrivacy::Full, + excluded_domains: vec!["EXAMPLE.test".into(), "*.private.test".into()], + ..Default::default() + }; + let history = HistoryStore::start(fixture.database(), config).await; + let stats = Statistics { + history: Some(history.clone()), + ..Statistics::new() + }; + for authority in ["example.test:443", "Sub.Private.Test.:8443"] { + let uri: http::Uri = format!("https://{authority}/secret").parse().unwrap(); + let request_stats = stats.for_domain(uri.host()); + assert!(!request_stats.should_collect()); + request_stats.record_request(uri.host(), client(), true); + request_stats.increment_modified_responses(); + // The history collector independently enforces the same policy. + history.request(uri.host(), client(), true); + } + let data = stats.get_serialized(); + assert_eq!(data.blocked_requests, 0); + assert_eq!(data.modified_responses, 0); + assert!(data.top_clients.is_empty()); + assert!(snapshot(&history).await.requests.is_empty()); + for domain in ["private.test", "sub.example.test", "example.test.evil.test"] { + assert!(stats.for_domain(Some(domain)).should_collect()); + } + history.shutdown().await.unwrap(); + } +} + +#[test] +fn rejects_unbounded_settings_and_urls_in_domain_exclusions() { + for pattern in [ + "https://example.test", + "example.test:443", + "example.test/path", + " ", + ] { + let config = StatisticsConfig { + excluded_domains: vec![pattern.into()], + ..Default::default() + }; + assert!(config.validate().is_err()); + } + for days in [0, 366] { + assert!(StatisticsConfig { + retention_days: days, + ..Default::default() + } + .validate() + .is_err()); + } + for limit in [0, 99, 1_000_001] { + assert!(StatisticsConfig { + max_entries: limit, + ..Default::default() + } + .validate() + .is_err()); + } + assert!(HistoryQuery { + limit: Some(201), + ..Default::default() + } + .validate() + .is_err()); +} + +#[test] +fn a_full_queue_drops_events_without_blocking_requests() { + let (sender, receiver) = mpsc::sync_channel(1); + let store = HistoryStore { + sender, + policy: Arc::new(RwLock::new(Policy { + config: enabled(StatisticsPrivacy::HideDomainsAndClients), + ..Default::default() + })), + status: Arc::new(Mutex::new(HistoryStatus::default())), + dropped: Arc::new(AtomicU64::new(0)), + operations: Arc::new(tokio::sync::Mutex::new(())), + updates: watch::channel(()).1, + }; + store.request(Some("private.example.test"), client(), true); + store.request(Some("private.example.test"), client(), true); + assert_eq!(store.status().dropped_events, 1); + let Command::Event(_, Event::Request(record)) = receiver.try_recv().unwrap() else { + panic!("expected a request"); + }; + assert!(record.domain.is_none()); + assert!(record.client.is_none()); +} + +#[derive(Debug)] +struct FailingDisk { + inner: redb::backends::FileBackend, + full: Arc, +} +impl redb::StorageBackend for FailingDisk { + fn len(&self) -> std::io::Result { + self.inner.len() + } + fn read(&self, offset: u64, out: &mut [u8]) -> std::io::Result<()> { + self.inner.read(offset, out) + } + fn set_len(&self, len: u64) -> std::io::Result<()> { + if self.full.load(Ordering::Relaxed) { + return Err(std::io::ErrorKind::StorageFull.into()); + } + self.inner.set_len(len) + } + fn sync_data(&self) -> std::io::Result<()> { + if self.full.load(Ordering::Relaxed) { + return Err(std::io::ErrorKind::StorageFull.into()); + } + self.inner.sync_data() + } + fn write(&self, offset: u64, bytes: &[u8]) -> std::io::Result<()> { + if self.full.load(Ordering::Relaxed) { + return Err(std::io::ErrorKind::StorageFull.into()); + } + self.inner.write(offset, bytes) + } + fn close(&self) -> std::io::Result<()> { + self.inner.close() + } +} + +#[test] +fn disk_full_pauses_recording_and_preserves_committed_history() { + let fixture = Fixture::new(); + let mut worker = Worker { + path: fixture.database(), + database: None, + config: enabled(StatisticsPrivacy::Full), + generation: 0, + pending: Vec::new(), + status: Arc::new(Mutex::new(HistoryStatus::default())), + dropped: Arc::new(AtomicU64::new(0)), + last_id: 0, + last_flush: Instant::now(), + last_compaction: Instant::now(), + last_prune: Instant::now(), + updates: watch::channel(()).0, + }; + worker.open().unwrap(); + let event = || { + Event::Request(RequestRecord { + id: 0, + timestamp: now_ms(), + domain: Some("example.test".into()), + client: None, + blocked: true, + }) + }; + worker.pending.push(event()); + worker.flush().unwrap(); + worker.database = None; + let full = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let file = OpenOptions::new() + .read(true) + .write(true) + .open(fixture.database()) + .unwrap(); + let backend = FailingDisk { + inner: redb::backends::FileBackend::new(file).unwrap(), + full: full.clone(), + }; + worker.database = Some(Database::builder().create_with_backend(backend).unwrap()); + full.store(true, Ordering::Relaxed); + worker.pending.push(event()); + let error = worker.flush().unwrap_err(); + worker.fail(error.to_string()); + assert!(!worker.status.lock().unwrap().active); + assert!(worker.status.lock().unwrap().error.is_some()); + assert!(worker.pending.is_empty()); + assert_eq!(worker.dropped.load(Ordering::Relaxed), 1); + worker.database = None; + full.store(false, Ordering::Relaxed); + worker + .configure(enabled(StatisticsPrivacy::Full), 1) + .unwrap(); + assert_eq!( + worker + .query(HistoryQuery::default()) + .unwrap() + .totals + .blocked_requests, + 1 + ); +} diff --git a/privaxy/src/server/web_gui/history.rs b/privaxy/src/server/web_gui/history.rs new file mode 100644 index 0000000..2b92b47 --- /dev/null +++ b/privaxy/src/server/web_gui/history.rs @@ -0,0 +1,215 @@ +use super::{json_status, with_arc, with_configuration_save_lock}; +use crate::{ + configuration::{Configuration, StatisticsConfig}, + statistics::{ + history::{HistoryQuery, HistorySnapshot, HistoryStore}, + Statistics, + }, +}; +use futures::{SinkExt, StreamExt}; +use http::StatusCode; +use serde::Serialize; +use std::{convert::Infallible, sync::Arc, time::Duration}; +use warp::{ + filters::BoxedFilter, + ws::{Message, WebSocket, Ws}, + Filter, Reply, +}; + +#[derive(Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum HistoryUpdate { + Snapshot { snapshot: HistorySnapshot }, + Error { error: String }, +} + +#[derive(Serialize)] +struct SettingsResponse { + config: StatisticsConfig, + status: crate::statistics::history::HistoryStatus, +} + +fn error(message: &str) -> Box { + json_status(StatusCode::INTERNAL_SERVER_ERROR, message) +} + +async fn get_settings(statistics: Statistics) -> Result, Infallible> { + let Some(history) = statistics.history else { + return Ok(error("Statistics worker unavailable")); + }; + Ok(match Configuration::read_from_home().await { + Ok(configuration) => Box::new(warp::reply::json(&SettingsResponse { + config: configuration.statistics, + status: history.status(), + })), + Err(err) => error(&err.to_string()), + }) +} + +async fn put_settings( + config: StatisticsConfig, + statistics: Statistics, + lock: Arc>, +) -> Result, Infallible> { + if let Err(message) = config.validate() { + return Ok(json_status(StatusCode::UNPROCESSABLE_ENTITY, &message)); + } + let _guard = lock.lock().await; + let mut configuration = match Configuration::read_from_home().await { + Ok(configuration) => configuration, + Err(err) => return Ok(error(&err.to_string())), + }; + configuration.statistics = config; + if let Err(err) = configuration.save().await { + return Ok(error(&err.to_string())); + } + // A saved opt-in and a working database are distinct; report disk errors + // alongside the saved config. Saving again retries without proxy reloads. + if let Err(err) = statistics.configure_history(configuration.statistics).await { + log::error!("Could not apply statistics settings: {err}"); + } + get_settings(statistics).await +} + +async fn get_history( + query: HistoryQuery, + statistics: Statistics, +) -> Result, Infallible> { + if let Err(message) = query.validate() { + return Ok(json_status(StatusCode::UNPROCESSABLE_ENTITY, &message)); + } + let Some(history) = statistics.history else { + return Ok(error("Statistics worker unavailable")); + }; + Ok(match history.query(query).await { + Ok(snapshot) => Box::new(warp::reply::json(&snapshot)), + Err(err) => error(&err), + }) +} + +async fn clear_history( + statistics: Statistics, + lock: Arc>, +) -> Result, Infallible> { + let _guard = lock.lock().await; + let Some(history) = statistics.history else { + return Ok(error("Statistics worker unavailable")); + }; + Ok(match history.clear().await { + Ok(()) => Box::new(warp::reply::with_status("", StatusCode::NO_CONTENT)), + Err(err) => error(&err), + }) +} + +async fn live_history( + ws: Ws, + query: HistoryQuery, + statistics: Statistics, +) -> Result, Infallible> { + if let Err(message) = query.validate() { + return Ok(json_status(StatusCode::UNPROCESSABLE_ENTITY, &message)); + } + let Some(history) = statistics.history else { + return Ok(error("Statistics worker unavailable")); + }; + Ok(Box::new(ws.on_upgrade(move |socket| { + stream_history(socket, history, query) + }))) +} + +async fn stream_history(socket: WebSocket, history: HistoryStore, query: HistoryQuery) { + let (mut tx, mut rx) = socket.split(); + let mut updates = history.subscribe(); + let send = async move { + let mut keepalive = tokio::time::interval(Duration::from_secs(30)); + keepalive.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + keepalive.tick().await; + let mut hour = chrono::Utc::now().timestamp() / 3600; + loop { + updates.borrow_and_update(); + let update = match history.query_committed(query.clone()).await { + Ok(snapshot) => HistoryUpdate::Snapshot { snapshot }, + Err(error) => HistoryUpdate::Error { error }, + }; + let Ok(json) = serde_json::to_string(&update) else { + break; + }; + if tx.send(Message::text(json)).await.is_err() { + break; + } + // Coalesce bursts of committed batches. A slow client has at most + // one query in flight; network backpressure never blocks the worker. + tokio::time::sleep(Duration::from_secs(1)).await; + loop { + tokio::select! { + changed = updates.changed() => { + if changed.is_err() { return; } + break; + } + _ = keepalive.tick() => { + let current_hour = chrono::Utc::now().timestamp() / 3600; + if current_hour != hour { + hour = current_hour; + break; + } + if tx.send(Message::ping(Vec::new())).await.is_err() { return; } + } + } + } + } + }; + let receive = async move { + // Poll the reader for Ping/Pong and stop even when an idle client leaves. + while let Some(Ok(message)) = rx.next().await { + if message.is_close() { + break; + } + } + }; + tokio::select! { + _ = send => {}, + _ = receive => {}, + } +} + +pub(super) fn settings_routes( + statistics: Statistics, + lock: Arc>, +) -> BoxedFilter<(impl Reply,)> { + let get = warp::get() + .and(warp::path::end()) + .and(with_arc(statistics.clone())) + .and_then(get_settings); + let put = warp::put() + .and(warp::path::end()) + .and(warp::body::content_length_limit(512 * 1024)) + .and(warp::body::json()) + .and(with_arc(statistics)) + .and(with_configuration_save_lock(lock)) + .and_then(put_settings); + get.or(put).boxed() +} + +pub(super) fn routes( + statistics: Statistics, + lock: Arc>, +) -> BoxedFilter<(impl Reply,)> { + // The parent /history route applies require_auth to all of these endpoints. + let live = warp::path("live") + .and(warp::path::end()) + .and(warp::ws()) + .and(warp::query::()) + .and(with_arc(statistics.clone())) + .and_then(live_history); + let get = warp::get() + .and(warp::path::end()) + .and(warp::query::()) + .and(with_arc(statistics.clone())) + .and_then(get_history); + let delete = warp::delete() + .and(warp::path::end()) + .and(with_arc(statistics)) + .and(with_configuration_save_lock(lock)) + .and_then(clear_history); + live.or(get).or(delete).boxed() +} diff --git a/privaxy/src/server/web_gui/mod.rs b/privaxy/src/server/web_gui/mod.rs index 54209ce..8331774 100644 --- a/privaxy/src/server/web_gui/mod.rs +++ b/privaxy/src/server/web_gui/mod.rs @@ -23,6 +23,7 @@ pub(crate) mod events; pub(crate) mod exclusions; mod filterlists; pub(crate) mod filters; +mod history; mod inclusions; pub(crate) mod logs; mod pac; @@ -30,6 +31,7 @@ pub(crate) mod settings; pub(crate) mod statistics; pub(crate) mod tls_failures; pub(crate) mod userscripts; +mod version; #[derive(Debug, Serialize)] pub(crate) struct ApiError { @@ -147,6 +149,15 @@ fn create_api_routes( let require_auth = auth::require_auth(configuration_save_lock.clone()); + let version_route = warp::path!("version") + .and(warp::get()) + .and(require_auth.clone()) + .map(version::version); + let updates_route = warp::path("updates") + .and(require_auth.clone()) + .and(version::updates(http_client.clone())); + let version_routes = version_route.or(updates_route).boxed(); + let events_route = warp::path("events") .and(require_auth.clone()) .and(warp::ws()) @@ -155,7 +166,21 @@ fn create_api_routes( ws.on_upgrade(move |websocket| events::events(websocket, events_sender)) }); + let history_route = warp::path("history") + .and(require_auth.clone()) + .and(history::routes( + statistics.clone(), + configuration_save_lock.clone(), + )); + let statistics_settings_route = warp::path!("settings" / "statistics" / ..) + .and(require_auth.clone()) + .and(history::settings_routes( + statistics.clone(), + configuration_save_lock.clone(), + )); + let statistics_route = warp::path("statistics") + .and(warp::path::end()) .and(require_auth.clone()) .and(warp::ws()) .map(move |ws: warp::ws::Ws| { @@ -255,8 +280,11 @@ fn create_api_routes( // a request to `/`), turning every non-API request into a JSON 404 // and preventing the static-files branch from serving the SPA. let api_inner = auth_routes + .or(version_routes) .or(events_route) .or(statistics_route) + .or(history_route) + .or(statistics_settings_route) .or(logs_route) .or(filters_route) .or(custom_filters_route) diff --git a/privaxy/src/server/web_gui/version.rs b/privaxy/src/server/web_gui/version.rs new file mode 100644 index 0000000..3362b88 --- /dev/null +++ b/privaxy/src/server/web_gui/version.rs @@ -0,0 +1,230 @@ +use super::{json_status, with_arc}; +use reqwest::Client; +use semver::Version; +use serde::{Deserialize, Serialize}; +use std::{ + convert::Infallible, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::sync::Mutex; +use warp::{filters::BoxedFilter, http::StatusCode, Filter, Reply}; + +const VERSION: &str = env!("CARGO_PKG_VERSION"); +const PROJECT_URL: &str = env!("CARGO_PKG_REPOSITORY"); +const RELEASE_API: &str = "https://api.github.com/repos/joshrmcdaniel/privaxy/releases/latest"; +const CACHE_DURATION: Duration = Duration::from_secs(3600); +const ERROR_CACHE_DURATION: Duration = Duration::from_secs(60); + +#[derive(Serialize)] +struct VersionInfo { + version: &'static str, + project_url: &'static str, +} + +#[derive(Clone, Serialize)] +struct ReleaseInfo { + version: String, + url: String, +} + +#[derive(Clone, Serialize)] +struct UpdateCheck { + current_version: &'static str, + latest_release: Option, + update_available: bool, + checked_at: i64, +} + +#[derive(Deserialize)] +struct GitHubRelease { + tag_name: String, + draft: bool, + prerelease: bool, +} + +struct CachedCheck { + result: Result, + fetched_at: Instant, +} + +#[derive(Clone)] +struct UpdateChecker { + client: Client, + cache: Arc>>, +} + +impl UpdateChecker { + async fn check(&self) -> Result { + // Share both successful and failed checks across tabs. Holding this + // lock also combines simultaneous requests into one bounded fetch. + let mut cache = self.cache.lock().await; + if let Some(cached) = cache.as_ref() { + let ttl = if cached.result.is_ok() { + CACHE_DURATION + } else { + ERROR_CACHE_DURATION + }; + if cached.fetched_at.elapsed() < ttl { + return cached.result.clone(); + } + } + let result = self.fetch().await; + *cache = Some(CachedCheck { + result: result.clone(), + fetched_at: Instant::now(), + }); + result + } + + async fn fetch(&self) -> Result { + let response = self + .client + .get(RELEASE_API) + .header( + reqwest::header::USER_AGENT, + concat!("Privaxy/", env!("CARGO_PKG_VERSION")), + ) + .header(reqwest::header::ACCEPT, "application/vnd.github+json") + .header("X-GitHub-Api-Version", "2026-03-10") + .timeout(Duration::from_secs(10)) + .send() + .await + .map_err(|error| { + if error.is_timeout() { + "Update check timed out. Try again in a minute." + } else { + "Could not reach GitHub. Try again in a minute." + } + .to_string() + })?; + let release = match response.status() { + StatusCode::NOT_FOUND => None, + StatusCode::OK => Some(response.json::().await.map_err(|_| { + "GitHub returned invalid release information. Try again in a minute.".to_string() + })?), + status => return Err(format!("GitHub returned HTTP {status}. Try again later.")), + }; + let (latest_release, update_available) = match release { + Some(release) => { + let (info, newer) = compare_release(VERSION, release)?; + (Some(info), newer) + } + None => (None, false), + }; + Ok(UpdateCheck { + current_version: VERSION, + latest_release, + update_available, + checked_at: chrono::Utc::now().timestamp(), + }) + } +} + +fn compare_release(current: &str, release: GitHubRelease) -> Result<(ReleaseInfo, bool), String> { + let version = Version::parse( + release + .tag_name + .strip_prefix('v') + .unwrap_or(&release.tag_name), + ) + .map_err(|_| "The latest release has an unrecognized version tag.".to_string())?; + if release.draft || release.prerelease || !version.pre.is_empty() { + return Err("GitHub did not return a stable release.".to_string()); + } + let current = Version::parse(current) + .map_err(|_| "The running version could not be compared.".to_string())?; + let newer = version.cmp_precedence(¤t).is_gt(); + // Construct the link from the fixed project URL and validated version tag; + // the remote response cannot supply an arbitrary link to the frontend. + Ok(( + ReleaseInfo { + version: version.to_string(), + url: format!("{PROJECT_URL}/releases/tag/{}", release.tag_name), + }, + newer, + )) +} + +async fn check_update(checker: UpdateChecker) -> Result, Infallible> { + Ok(match checker.check().await { + Ok(result) => Box::new(warp::reply::json(&result)), + Err(error) => json_status(StatusCode::BAD_GATEWAY, &error), + }) +} + +pub(super) fn version() -> impl Reply { + warp::reply::with_header( + warp::reply::json(&VersionInfo { + version: VERSION, + project_url: PROJECT_URL, + }), + "cache-control", + "no-store", + ) +} + +pub(super) fn updates(client: Client) -> BoxedFilter<(impl Reply,)> { + warp::path::end() + .and(warp::get()) + .and(with_arc(UpdateChecker { + client, + cache: Arc::new(Mutex::new(None)), + })) + .and_then(check_update) + .with(warp::reply::with::header("cache-control", "no-store")) + .boxed() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn release(tag: &str) -> GitHubRelease { + GitHubRelease { + tag_name: tag.into(), + draft: false, + prerelease: false, + } + } + + #[test] + fn updates_use_semantic_precedence_and_preserve_release_links() { + for (current, tag, expected) in [ + ("0.8.0", "v0.8.1", true), + ("0.9.0", "v0.10.0", true), + ("0.8.0", "0.8.0", false), + ("0.9.0", "v0.8.0", false), + ("0.8.0-rc.1", "v0.8.0", true), + ("0.8.0+local", "v0.8.0+release", false), + ] { + let (info, newer) = compare_release(current, release(tag)).unwrap(); + assert_eq!(newer, expected, "{current} -> {tag}"); + assert_eq!(info.url, format!("{PROJECT_URL}/releases/tag/{tag}")); + } + } + + #[test] + fn invalid_and_unstable_releases_cannot_be_reported_as_updates() { + for tag in [ + "latest", + "dev", + "v1.0.0-beta.1", + "https://example.test/", + "v1.0.0/extra", + ] { + assert!(compare_release("0.8.0", release(tag)).is_err()); + } + for (draft, prerelease) in [(true, false), (false, true)] { + assert!(compare_release( + "0.8.0", + GitHubRelease { + draft, + prerelease, + ..release("v1.0.0") + } + ) + .is_err()); + } + } +} diff --git a/web_frontend/Cargo.toml b/web_frontend/Cargo.toml index 38caf90..2ba4290 100644 --- a/web_frontend/Cargo.toml +++ b/web_frontend/Cargo.toml @@ -12,6 +12,7 @@ num-format = "0.4.0" serde = { version = "1.0.137", features = ["derive"] } gloo-utils = "0.3" wasm-bindgen = "0.2.125" +js-sys = "0.3" wasm-bindgen-futures = "0.4" futures = "0.3.21" log = "0.4.17" diff --git a/web_frontend/src/dashboard.rs b/web_frontend/src/dashboard.rs index 9972a02..f2939d3 100644 --- a/web_frontend/src/dashboard.rs +++ b/web_frontend/src/dashboard.rs @@ -1,9 +1,86 @@ use crate::blocking_enabled::BlockingEnabled; use crate::live_stream; -use futures::future::AbortHandle; +use crate::{api, history::HistoryPage, statistics_settings::StatisticsSettings}; +use futures::future::{AbortHandle, Abortable}; use num_format::{Locale, ToFormattedString}; use serde::Deserialize; -use yew::{html, Component, Context, Html}; +use wasm_bindgen_futures::spawn_local; +use yew::prelude::*; + +#[function_component(Dashboard)] +pub fn dashboard() -> Html { + let detailed = use_state(|| None::>); + let reload = use_state(|| 0_u32); + { + let detailed = detailed.clone(); + use_effect_with(*reload, move |_| { + let (handle, registration) = AbortHandle::new_pair(); + spawn_local(async move { + if let Ok(result) = Abortable::new( + api::get_json::("/api/settings/statistics"), + registration, + ) + .await + { + detailed.set(Some(result.map(|settings| { + settings.config.enabled && settings.config.privacy != "anonymous" + }))); + } + }); + move || handle.abort() + }); + } + + match &*detailed { + Some(Ok(true)) => html! { }, + Some(Ok(false)) => html! { }, + result => { + let retry = { + let detailed = detailed.clone(); + Callback::from(move |_| { + detailed.set(None); + reload.set(*reload + 1); + }) + }; + html! { <> + + if let Some(Err(error)) = result { + + + } else { +

{"Loading statistics…"}

+ } + } + } + } +} + +#[derive(Properties, PartialEq)] +pub struct DashboardHeaderProps { + #[prop_or_default] + pub connected: bool, +} + +#[function_component(DashboardHeader)] +pub fn dashboard_header(props: &DashboardHeaderProps) -> Html { + html! { +
+
+

{ "Dashboard" } if props.connected {
}

+
+
+ + + + + {"Download CA certificate"} + + +
+
+ } +} #[derive(Debug, Deserialize, PartialEq, Eq)] pub struct Statistics { @@ -21,13 +98,13 @@ pub enum Message { Connection(bool), } -pub struct Dashboard { +pub struct LiveDashboard { message: Statistics, connected: bool, ws_abort_handle: AbortHandle, } -impl Component for Dashboard { +impl Component for LiveDashboard { type Message = Message; type Properties = (); @@ -89,29 +166,12 @@ impl Component for Dashboard { html! { <> -
-
-

{ "Dashboard" } if self.connected {
} -

-
-
- - - - - {"Download CA certificate"} - - -
-
+ if !self.connected {

{"Connecting to statistics… Retrying automatically. Displayed counts may be out of date."}

} +

{"Live statistics since this process started. "}{"Statistics settings"}

@@ -150,7 +210,7 @@ impl Component for Dashboard {
-

{"Top blocked paths"}

+

{"Top blocked domains"}

    diff --git a/web_frontend/src/footer.rs b/web_frontend/src/footer.rs new file mode 100644 index 0000000..9edbd5f --- /dev/null +++ b/web_frontend/src/footer.rs @@ -0,0 +1,186 @@ +use crate::api; +use futures::future::{AbortHandle, Abortable}; +use gloo_timers::callback::Interval; +use serde::{de::DeserializeOwned, Deserialize}; +use wasm_bindgen_futures::spawn_local; +use yew::prelude::*; + +const UPDATE_CHECK_INTERVAL_MS: u32 = 60 * 60 * 1000; + +#[derive(Deserialize)] +pub struct VersionInfo { + version: String, + project_url: String, +} + +#[derive(Deserialize)] +struct ReleaseInfo { + version: String, + url: String, +} + +#[derive(Deserialize)] +pub struct UpdateCheck { + current_version: String, + latest_release: Option, + update_available: bool, + checked_at: i64, +} + +pub enum Message { + LoadVersion, + VersionLoaded(Result), + Check, + Checked(Result), +} + +pub struct Footer { + version: Option, + version_error: Option, + update: Option, + version_request: Option, + update_request: Option, + _update_interval: Interval, +} + +impl Component for Footer { + type Message = Message; + type Properties = (); + + fn create(ctx: &Context) -> Self { + ctx.link().send_message(Message::LoadVersion); + let link = ctx.link().clone(); + Self { + version: None, + version_error: None, + update: None, + version_request: None, + update_request: None, + _update_interval: Interval::new(UPDATE_CHECK_INTERVAL_MS, move || { + link.send_message(Message::Check); + }), + } + } + + fn update(&mut self, ctx: &Context, message: Message) -> bool { + match message { + Message::LoadVersion => { + if self.version_request.is_some() { + return false; + } + self.version_error = None; + self.version_request = Some(fetch( + "/api/version", + ctx.link().callback(Message::VersionLoaded), + )); + } + Message::VersionLoaded(result) => { + self.version_request = None; + match result { + Ok(version) => { + self.version = Some(version); + ctx.link().send_message(Message::Check); + } + Err(error) => self.version_error = Some(error), + } + } + Message::Check => { + if self.version.is_none() || self.update_request.is_some() { + return false; + } + self.update_request = + Some(fetch("/api/updates", ctx.link().callback(Message::Checked))); + return false; + } + Message::Checked(result) => { + self.update_request = None; + let update = match result { + Ok(update) => update, + Err(error) => { + log::warn!("Automatic update check failed: {error}"); + return false; + } + }; + if let Some(version) = &mut self.version { + version.version.clone_from(&update.current_version); + } + self.update = Some(update); + } + } + true + } + + fn destroy(&mut self, _ctx: &Context) { + for request in [&mut self.version_request, &mut self.update_request] { + if let Some(request) = request.take() { + request.abort(); + } + } + } + + fn view(&self, ctx: &Context) -> Html { + html! { +
    +
    +
    + if let Some(version) = &self.version { + {format!("Privaxy v{}", version.version)} + } else { + {"Privaxy"} + if let Some(error) = &self.version_error { + {"Version unavailable"} + + } else { + {"Loading version…"} + } + } +
    + {self.update_status()} +
    +
    + } + } +} + +impl Footer { + fn update_status(&self) -> Html { + let Some(update) = self + .update + .as_ref() + .filter(|update| update.update_available) + else { + return Html::default(); + }; + let Some(release) = &update.latest_release else { + return Html::default(); + }; + let checked = js_sys::Date::new(&wasm_bindgen::JsValue::from_f64( + update.checked_at as f64 * 1000.0, + )) + .to_iso_string() + .as_string() + .unwrap_or_default(); + let title = format!( + "Privaxy v{} is available. Last checked: {checked}", + release.version + ); + html! { + + } + } +} + +fn fetch( + url: &'static str, + callback: Callback>, +) -> AbortHandle { + let (handle, registration) = AbortHandle::new_pair(); + spawn_local(async move { + if let Ok(result) = Abortable::new(api::get_json(url), registration).await { + callback.emit(result); + } + }); + handle +} diff --git a/web_frontend/src/history.rs b/web_frontend/src/history.rs new file mode 100644 index 0000000..1805ef2 --- /dev/null +++ b/web_frontend/src/history.rs @@ -0,0 +1,317 @@ +use crate::button::{get_css, ButtonColor}; +use crate::dashboard::DashboardHeader; +use crate::{live_stream, statistics_settings::HistoryStatus}; +use futures::future::AbortHandle; +use num_format::{Locale, ToFormattedString}; +use serde::Deserialize; +use std::collections::BTreeMap; +use web_sys::HtmlSelectElement; +use yew::prelude::*; + +#[derive(Clone, Debug, Default, Deserialize)] +pub struct Counts { + proxied_requests: u64, + blocked_requests: u64, + modified_responses: u64, +} + +#[derive(Clone, Debug, Deserialize)] +struct Record { + id: u64, + timestamp: u64, + domain: Option, + client: Option, + blocked: bool, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct Snapshot { + status: HistoryStatus, + totals: Counts, + hourly: Vec<(u64, Counts)>, + top_domains: Vec<(String, u64)>, + top_blocked_domains: Vec<(String, u64)>, + top_clients: Vec<(String, u64)>, + requests: Vec, + next_before: Option, +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum HistoryUpdate { + Snapshot { snapshot: Snapshot }, + Error { error: String }, +} + +pub enum Message { + Load, + Loaded(u64, Result), + Connection(u64, bool), + Hours(String), + Filter(String), + Older, + Newest, +} + +pub struct HistoryPage { + snapshot: Option, + hours: u16, + filter: String, + before: Option, + loading: bool, + error: Option, + generation: u64, + stream: Option, + connected: bool, +} + +impl Component for HistoryPage { + type Message = Message; + type Properties = (); + fn create(ctx: &Context) -> Self { + ctx.link().send_message(Message::Load); + Self { + snapshot: None, + hours: 24, + filter: String::new(), + before: None, + loading: false, + error: None, + generation: 0, + stream: None, + connected: false, + } + } + + fn update(&mut self, ctx: &Context, message: Message) -> bool { + match message { + Message::Load => { + self.load(ctx); + } + Message::Connection(generation, connected) => { + if generation != self.generation || self.connected == connected { + return false; + } + self.connected = connected; + } + Message::Loaded(generation, result) => { + if generation != self.generation { + return false; + } + self.loading = false; + match result { + Ok(snapshot) => { + self.error = snapshot.status.error.clone(); + self.snapshot = Some(snapshot); + } + Err(error) => self.error = Some(error), + } + } + Message::Hours(hours) => { + self.hours = hours.parse().unwrap_or(24); + self.before = None; + self.snapshot = None; + self.load(ctx); + } + Message::Filter(filter) => { + self.filter = filter; + self.before = None; + self.load(ctx); + } + Message::Older => { + if self.loading { + return false; + } + self.before = self.snapshot.as_ref().and_then(|s| s.next_before); + self.load(ctx); + } + Message::Newest => { + self.before = None; + self.load(ctx); + } + } + true + } + + fn destroy(&mut self, _ctx: &Context) { + if let Some(stream) = self.stream.take() { + stream.abort(); + } + } + + fn view(&self, ctx: &Context) -> Html { + let on_hours = ctx.link().callback(|e: Event| { + Message::Hours(e.target_unchecked_into::().value()) + }); + let on_filter = ctx.link().callback(|e: Event| { + Message::Filter(e.target_unchecked_into::().value()) + }); + html! { <> + +
    + + + {"Statistics settings"} +
    +

    {"Saved statistics survive restarts. Totals and top lists cover retained requests in this window; the request filter and pagination only affect the log. Times are UTC. Updates appear automatically as requests are saved."}

    + if !self.connected { +

    {"Connecting to live statistics… Retrying automatically. Displayed data may be out of date."}

    + } + if let Some(error) = &self.error { } + if self.loading {

    {"Loading history…"}

    } + if let Some(snapshot) = &self.snapshot { + if !snapshot.status.active && snapshot.status.error.is_none() { +

    {"Persistent statistics are off or anonymous mode is selected. Enable history in Statistics settings to start collecting it."}

    + } else if snapshot.status.active { + if snapshot.status.dropped_events > 0 { } +
    + {for [("Proxied requests", snapshot.totals.proxied_requests), ("Blocked requests", snapshot.totals.blocked_requests), ("Modified responses", snapshot.totals.modified_responses)].into_iter().map(|(label, count)| html! { +
    {label}
    {count.to_formatted_string(&Locale::en)}
    + })} +
    +
    +

    {"Requests over time"}

    + {self.chart(snapshot)} +
    +
    + {top_list("Top domains", &snapshot.top_domains)} + {top_list("Top blocked domains", &snapshot.top_blocked_domains)} + {top_list("Top clients", &snapshot.top_clients)} +
    +
    +
    +

    {"Request history"}

    + +
    + if self.before.is_some() { +

    {"Viewing older requests. Totals and charts continue updating; select Newest requests to follow new activity."}

    + } + if snapshot.requests.is_empty() {

    {"No saved requests in this window."}

    } +
    + + + {for snapshot.requests.iter().map(|record| html! { + + + + + + + })} +
    {"Time (UTC)"}{"Domain"}{"Client"}{"Result"}
    {timestamp(record.timestamp)}{record.domain.as_deref().unwrap_or("Hidden")}{record.client.as_deref().unwrap_or("Hidden")}{if record.blocked {"Blocked"} else {"Proxied"}}
    +
    +
    + + +
    +
    + } + } + } + } +} + +impl HistoryPage { + fn load(&mut self, ctx: &Context) { + if let Some(stream) = self.stream.take() { + stream.abort(); + } + self.loading = true; + self.connected = false; + self.error = None; + self.generation += 1; + let generation = self.generation; + let mut url = format!("/api/history/live?hours={}&limit=50", self.hours); + if let Some(before) = self.before { + url.push_str(&format!("&before={before}")); + } + if !self.filter.is_empty() { + url.push_str(&format!("&blocked={}", self.filter)); + } + self.stream = Some(live_stream::subscribe( + url, + ctx.link().callback(move |update| { + Message::Loaded( + generation, + match update { + HistoryUpdate::Snapshot { snapshot } => Ok(snapshot), + HistoryUpdate::Error { error } => Err(error), + }, + ) + }), + ctx.link() + .callback(move |connected| Message::Connection(generation, connected)), + )); + } + + fn chart(&self, snapshot: &Snapshot) -> Html { + let bucket_ms = if self.hours <= 24 { + 3_600_000 + } else { + 86_400_000 + }; + let mut buckets = BTreeMap::::new(); + for (time, counts) in &snapshot.hourly { + let bucket = buckets.entry(time / bucket_ms * bucket_ms).or_default(); + bucket.0 += counts.proxied_requests; + bucket.1 += counts.blocked_requests; + } + if buckets.is_empty() { + return html! {

    {"No saved activity yet."}

    }; + } + let now = js_sys::Date::now() as u64; + let last = now / bucket_ms * bucket_ms; + let first = if self.hours == 8760 { + (*buckets.first_key_value().unwrap().0).min(last) + } else { + (now / 3_600_000 * 3_600_000).saturating_sub(u64::from(self.hours - 1) * 3_600_000) + / bucket_ms + * bucket_ms + }; + let max = buckets + .values() + .map(|(p, b)| p + b) + .max() + .unwrap_or(1) + .max(1) as f64; + html! { <> +

    {format!("Peak: {} requests per {}", max as u64, if self.hours <= 24 {"hour"} else {"day"})}

    + +
    {timestamp(first)}{timestamp(last)}
    +

    {if self.hours <= 24 {"Hourly totals · Blue: proxied · Red: blocked"} else {"Daily totals · Blue: proxied · Red: blocked"}}

    + } + } +} + +fn top_list(title: &str, entries: &[(String, u64)]) -> Html { + html! {
    +

    {title}

    + if entries.is_empty() {

    {"No visible data at this privacy level or in this window."}

    } +
      {for entries.iter().take(10).map(|(label, count)| html! { +
    1. {label}{count.to_formatted_string(&Locale::en)}
      2. + })}
    +
    } +} + +fn timestamp(value: u64) -> String { + let iso = js_sys::Date::new(&wasm_bindgen::JsValue::from_f64(value as f64)) + .to_iso_string() + .as_string() + .unwrap_or_default(); + iso.get(..19).unwrap_or(&iso).replace('T', " ") +} diff --git a/web_frontend/src/live_stream.rs b/web_frontend/src/live_stream.rs index d71a316..7766ce9 100644 --- a/web_frontend/src/live_stream.rs +++ b/web_frontend/src/live_stream.rs @@ -9,15 +9,16 @@ use yew::Callback; /// Reconnect live views after a server restart or network interruption. The /// caller aborts this task when its component is destroyed. pub fn subscribe( - url: &'static str, + url: impl Into, on_message: Callback, on_connection: Callback, ) -> AbortHandle { + let url = url.into(); let (handle, registration) = AbortHandle::new_pair(); let future = Abortable::new( async move { loop { - if let Ok(mut ws) = WebSocket::open(url) { + if let Ok(mut ws) = WebSocket::open(&url) { while matches!(ws.state(), State::Connecting) { TimeoutFuture::new(50).await; } diff --git a/web_frontend/src/main.rs b/web_frontend/src/main.rs index 72909a1..5f57fd7 100644 --- a/web_frontend/src/main.rs +++ b/web_frontend/src/main.rs @@ -18,7 +18,9 @@ mod filter_edit; mod filter_failures; mod filterlists; mod filters; +mod footer; mod general; +mod history; mod inclusions; mod live_stream; mod logs; @@ -27,6 +29,7 @@ mod requests; mod save_button; mod settings; mod settings_textarea; +mod statistics_settings; mod submit_banner; mod tls_failures; mod userscript_edit; @@ -43,6 +46,8 @@ enum Route { Dashboard, #[at("/requests")] Requests, + #[at("/history")] + History, #[at("/settings/:s")] Settings, #[at("/exclude")] @@ -73,6 +78,17 @@ fn not_found() -> Html { } } +#[function_component(HistoryRedirect)] +fn history_redirect() -> Html { + let navigator = use_navigator().unwrap(); + use_effect_with((), move |_| { + // Replace the old URL so Back does not revisit the redirect. + navigator.replace(&Route::Dashboard); + || () + }); + Html::default() +} + fn switch(route: Route) -> Html { fn get_classes(current_route: Route, for_route_link: Route) -> Classes { if current_route == for_route_link { @@ -129,6 +145,9 @@ fn switch(route: Route) -> Html { Route::Settings => { html! {<>{navigation}
    render={settings::switch_settings} />
    } } + Route::History => { + html! { } + } Route::Exclude => { set_title("Exclude host"); html! { <>{navigation}
    } @@ -144,9 +163,14 @@ fn switch(route: Route) -> Html { fn app() -> Html { html! { - - render={switch} /> - +
    +
    + + render={switch} /> + +
    + +
    } } diff --git a/web_frontend/src/settings.rs b/web_frontend/src/settings.rs index dc9d653..0484d53 100644 --- a/web_frontend/src/settings.rs +++ b/web_frontend/src/settings.rs @@ -6,6 +6,7 @@ use crate::general::GeneralSettings; use crate::pac::PacSettingsPage; use crate::set_title; use crate::settings_textarea::SettingsTextarea; +use crate::statistics_settings::StatisticsSettingsPage; use crate::userscripts::UserScriptsPage; use yew::prelude::*; use yew::{html, Html}; @@ -29,6 +30,8 @@ pub enum SettingsRoute { Account, #[at("/settings/debug")] Debug, + #[at("/settings/statistics")] + Statistics, } pub fn switch_settings(route: SettingsRoute) -> Html { @@ -112,6 +115,10 @@ pub fn switch_settings(route: SettingsRoute) -> Html { html! { } } + SettingsRoute::Statistics => { + set_title("Settings - Statistics"); + html! { } + } }; html! {
    @@ -124,6 +131,7 @@ pub fn switch_settings(route: SettingsRoute) -> Html { classes={get_classes(route, SettingsRoute::Pac)} to={SettingsRoute::Pac}> { "PAC" }> classes={get_classes(route, SettingsRoute::Account)} to={SettingsRoute::Account}> { "Account" }> classes={get_classes(route, SettingsRoute::Debug)} to={SettingsRoute::Debug}> { "Debug" }> + classes={get_classes(route, SettingsRoute::Statistics)} to={SettingsRoute::Statistics}> { "Statistics" }>
    { content }
    diff --git a/web_frontend/src/statistics_settings.rs b/web_frontend/src/statistics_settings.rs new file mode 100644 index 0000000..a4e31a6 --- /dev/null +++ b/web_frontend/src/statistics_settings.rs @@ -0,0 +1,319 @@ +use crate::api; +use gloo_net::http::Request; +use serde::{Deserialize, Serialize}; +use wasm_bindgen_futures::spawn_local; +use web_sys::{HtmlInputElement, HtmlSelectElement, HtmlTextAreaElement}; +use yew::prelude::*; + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub struct StatisticsConfig { + pub enabled: bool, + pub privacy: String, + pub retention_days: u16, + pub max_entries: u32, + #[serde(default)] + pub excluded_domains: Vec, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct HistoryStatus { + pub active: bool, + pub error: Option, + pub dropped_events: u64, + pub stored_requests: u64, + pub database_bytes: u64, +} + +#[derive(Clone, Deserialize)] +pub struct StatisticsSettings { + pub config: StatisticsConfig, + pub status: HistoryStatus, +} + +pub enum Message { + Load, + Loaded(Result), + Enabled(bool), + Privacy(String), + Days(String), + Limit(String), + Exclusions(String), + Save, + Saved(Result), + ConfirmClear(bool), + Clear, + Cleared(Result<(), String>), +} + +pub struct StatisticsSettingsPage { + settings: Option, + days: String, + limit: String, + exclusions: String, + busy: bool, + confirm_clear: bool, + error: Option, + notice: Option, +} + +impl Component for StatisticsSettingsPage { + type Message = Message; + type Properties = (); + + fn create(ctx: &Context) -> Self { + ctx.link().send_message(Message::Load); + Self { + settings: None, + days: String::new(), + limit: String::new(), + exclusions: String::new(), + busy: false, + confirm_clear: false, + error: None, + notice: None, + } + } + + fn update(&mut self, ctx: &Context, message: Message) -> bool { + match message { + Message::Load => { + self.busy = true; + self.error = None; + let link = ctx.link().clone(); + spawn_local(async move { + link.send_message(Message::Loaded( + api::get_json("/api/settings/statistics").await, + )); + }); + } + Message::Loaded(result) | Message::Saved(result) => { + self.busy = false; + match result { + Ok(settings) => { + self.days = settings.config.retention_days.to_string(); + self.limit = settings.config.max_entries.to_string(); + self.exclusions = settings.config.excluded_domains.join("\n"); + self.error = settings.status.error.clone(); + self.settings = Some(settings); + } + Err(error) => { + self.error = Some(error); + self.notice = None; + } + } + } + Message::Enabled(value) => { + if self.busy { + return false; + } + if let Some(settings) = &mut self.settings { + settings.config.enabled = value; + } + self.notice = None; + } + Message::Privacy(value) => { + if self.busy { + return false; + } + if let Some(settings) = &mut self.settings { + settings.config.privacy = value; + } + self.notice = None; + } + Message::Days(value) => { + if self.busy { + return false; + } + self.days = value; + self.notice = None; + } + Message::Limit(value) => { + if self.busy { + return false; + } + self.limit = value; + self.notice = None; + } + Message::Exclusions(value) => { + if self.busy { + return false; + } + self.exclusions = value; + self.notice = None; + } + Message::Save => { + if self.busy { + return false; + } + let Some(settings) = &self.settings else { + return false; + }; + let (Ok(days @ 1..=365), Ok(limit @ 100..=1_000_000)) = + (self.days.parse::(), self.limit.parse::()) + else { + self.error = Some("Choose 1–365 days and 100–1,000,000 requests.".into()); + return true; + }; + let mut config = settings.config.clone(); + config.retention_days = days; + config.max_entries = limit; + config.excluded_domains = self + .exclusions + .lines() + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(|s| s.trim_end_matches('.').to_ascii_lowercase()) + .collect(); + self.busy = true; + self.error = None; + self.notice = None; + let link = ctx.link().clone(); + spawn_local(async move { + let result = async { + let response = Request::put("/api/settings/statistics") + .json(&config) + .map_err(|e| e.to_string())? + .send() + .await + .map_err(|e| e.to_string())?; + if !response.ok() { + return Err(api::response_error(response).await); + } + response + .json::() + .await + .map_err(|e| e.to_string()) + } + .await; + link.send_message(Message::Saved(result)); + }); + self.notice = Some("Settings saved".into()); + } + Message::ConfirmClear(value) => { + if self.busy { + return false; + } + self.confirm_clear = value; + } + Message::Clear => { + if self.busy || !self.confirm_clear { + return false; + } + self.busy = true; + self.error = None; + self.notice = None; + let link = ctx.link().clone(); + spawn_local(async move { + let result = async { + let response = Request::delete("/api/history") + .send() + .await + .map_err(|e| e.to_string())?; + if response.ok() { + Ok(()) + } else { + Err(api::response_error(response).await) + } + } + .await; + link.send_message(Message::Cleared(result)); + }); + } + Message::Cleared(result) => { + self.busy = false; + self.confirm_clear = false; + match result { + Ok(()) => { + self.notice = Some("Saved history deleted".into()); + if let Some(settings) = &mut self.settings { + settings.status.stored_requests = 0; + settings.status.database_bytes = 0; + settings.status.dropped_events = 0; + settings.status.error = None; + } + } + Err(error) => self.error = Some(error), + } + } + } + true + } + + fn view(&self, ctx: &Context) -> Html { + let Some(settings) = &self.settings else { + return html! { <> +

    {"Statistics"}

    + if let Some(error) = &self.error { + + + } else {

    {"Loading…"}

    } + }; + }; + let config = &settings.config; + let on_enabled = ctx.link().callback(|e: Event| { + Message::Enabled(e.target_unchecked_into::().checked()) + }); + let on_privacy = ctx.link().callback(|e: Event| { + Message::Privacy(e.target_unchecked_into::().value()) + }); + let on_days = ctx.link().callback(|e: InputEvent| { + Message::Days(e.target_unchecked_into::().value()) + }); + let on_limit = ctx.link().callback(|e: InputEvent| { + Message::Limit(e.target_unchecked_into::().value()) + }); + let on_exclusions = ctx.link().callback(|e: InputEvent| { + Message::Exclusions(e.target_unchecked_into::().value()) + }); + html! { <> +

    {"Statistics"}

    +

    {"History is off by default. When enabled, the Dashboard shows saved totals, trends, and request history. With persistence off or Anonymous mode selected, it shows live session statistics. Saved history stays on this server and survives restarts. It records request times, hostnames, client IPs, and blocking decisions according to your privacy level. Full URLs, query strings, headers, and bodies are never saved in statistics."}

    + if let Some(error) = &self.error { } +
    +
    +
    + +
    +
    +

    {"Disabling stops recording. Previously saved history remains until you delete it or re-enable recording and retention removes it."}

    +
    +
    +
    + + +
    +

    {"Hidden identifiers are removed before collection. While enabled, increasing privacy also removes those identifiers from saved history. They cannot be recovered by lowering privacy. Anonymous mode keeps only session counters and does not record history."}

    +
    +
    + + +
    +
    + + +

    {"The oldest requests are removed when either limit is reached. Reducing a limit deletes older history when saved."}

    +
    + +