From 0096913eb71733d04b61ea31b2bf9e7409ae8164 Mon Sep 17 00:00:00 2001 From: csd113 Date: Sun, 4 Oct 2026 19:03:38 -0700 Subject: [PATCH 01/14] thumbnail loading impovements --- docs/thumbnail-evidence/browser-baseline.json | 433 ++++++++++++++ docs/thumbnail-evidence/browser-final.json | 433 ++++++++++++++ docs/thumbnail-evidence/http-baseline.json | 346 +++++++++++ .../thumbnail-evidence/http-final-repeat.json | 346 +++++++++++ docs/thumbnail-evidence/http-final.json | 346 +++++++++++ docs/thumbnail-pipeline-audit.md | 214 +++++++ src/handlers/board/media.rs | 553 ++++++++++++++---- src/handlers/board/media/performance.rs | 193 ++++++ 8 files changed, 2737 insertions(+), 127 deletions(-) create mode 100644 docs/thumbnail-evidence/browser-baseline.json create mode 100644 docs/thumbnail-evidence/browser-final.json create mode 100644 docs/thumbnail-evidence/http-baseline.json create mode 100644 docs/thumbnail-evidence/http-final-repeat.json create mode 100644 docs/thumbnail-evidence/http-final.json create mode 100644 docs/thumbnail-pipeline-audit.md create mode 100644 src/handlers/board/media/performance.rs diff --git a/docs/thumbnail-evidence/browser-baseline.json b/docs/thumbnail-evidence/browser-baseline.json new file mode 100644 index 00000000..93baa40e --- /dev/null +++ b/docs/thumbnail-evidence/browser-baseline.json @@ -0,0 +1,433 @@ +{ + "browser": "153.0.8010.12", + "thumbnail_bytes": 33016, + "unique_thumbnail_requests_after_full_scroll": 65, + "samples": [ + { + "posts_with_images": 0, + "sample": 0, + "state": "cold-browser-cache", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13.5, + "navigation_dom_content_loaded_ms": 12.800000011920929, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 1, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 10.700000047683716, + "navigation_dom_content_loaded_ms": 9.600000023841858, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 2, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13.100000023841858, + "navigation_dom_content_loaded_ms": 12.100000023841858, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 3, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13.099999964237213, + "navigation_dom_content_loaded_ms": 12, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 4, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13.199999988079071, + "navigation_dom_content_loaded_ms": 12.100000023841858, + "resource_duration_ms": [] + }, + { + "posts_with_images": 4, + "sample": 0, + "state": "cold-browser-cache", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 4, + "cached_media_responses": 0, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 133264, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 135536, + "navigation_load_event_end_ms": 20.899999976158142, + "navigation_dom_content_loaded_ms": 19.600000023841858, + "resource_duration_ms": [ + 4.899999976158142, + 2.099999964237213, + 2.299999952316284, + 2.100000023841858 + ] + }, + { + "posts_with_images": 4, + "sample": 1, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 14.900000035762787, + "navigation_dom_content_loaded_ms": 13.700000047683716, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 4, + "sample": 2, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 14.899999976158142, + "navigation_dom_content_loaded_ms": 13.600000023841858, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 4, + "sample": 3, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 15.800000011920929, + "navigation_dom_content_loaded_ms": 14.5, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 4, + "sample": 4, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 18.80000001192093, + "navigation_dom_content_loaded_ms": 17.100000023841858, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 0, + "state": "cold-browser-cache", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 25, + "cached_media_responses": 0, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 832900, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 847100, + "navigation_load_event_end_ms": 52, + "navigation_dom_content_loaded_ms": 45.09999996423721, + "resource_duration_ms": [ + 5, + 3.100000023841858, + 5.199999988079071, + 3.599999964237213, + 4.899999976158142, + 5, + 5.099999964237213, + 5.099999964237213, + 6.299999952316284, + 7.600000023841858, + 7.400000035762787, + 7.5, + 7.800000011920929, + 7, + 7.800000011920929, + 9.800000011920929, + 9.800000011920929, + 9.899999976158142, + 9.400000035762787, + 9.800000011920929, + 9.600000023841858, + 10.600000023841858, + 11.400000035762787, + 11.100000023841858, + 11.300000011920929 + ] + }, + { + "posts_with_images": 128, + "sample": 1, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 48.19999998807907, + "navigation_dom_content_loaded_ms": 41.19999998807907, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 2, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 48.39999997615814, + "navigation_dom_content_loaded_ms": 41.39999997615814, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 3, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 47.09999996423721, + "navigation_dom_content_loaded_ms": 40, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 4, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 51.19999998807907, + "navigation_dom_content_loaded_ms": 40.80000001192093, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "limitations": "Debug server, local loopback HTTP, synthetic 1024x768 gradient source, 250px lossless WebP preview, 128 posts share 65 unique URLs. Initial viewport measured for 500ms after load. Browser setup/CDP overhead; cold browser cache, warm server/OS cache. Resource Timing adds an estimated 300-byte header cost; CDP records transfer separately. Load-event timings are observations, not isolated proof of frontend speedup. Original loaded only after click; missing/corrupt fallback and missing-file recovery asserted. Raw output remains in local output/playwright and /tmp." +} diff --git a/docs/thumbnail-evidence/browser-final.json b/docs/thumbnail-evidence/browser-final.json new file mode 100644 index 00000000..f6bcefdc --- /dev/null +++ b/docs/thumbnail-evidence/browser-final.json @@ -0,0 +1,433 @@ +{ + "browser": "153.0.8010.12", + "thumbnail_bytes": 33016, + "unique_thumbnail_requests_after_full_scroll": 65, + "samples": [ + { + "posts_with_images": 0, + "sample": 0, + "state": "cold-browser-cache", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13.099999964237213, + "navigation_dom_content_loaded_ms": 12.299999952316284, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 1, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13.300000011920929, + "navigation_dom_content_loaded_ms": 12.400000035762787, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 2, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 14.199999988079071, + "navigation_dom_content_loaded_ms": 13.199999988079071, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 3, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 15, + "navigation_dom_content_loaded_ms": 14, + "resource_duration_ms": [] + }, + { + "posts_with_images": 0, + "sample": 4, + "state": "warm", + "thumbnail_elements": 0, + "loaded_elements_including_repeated_urls": 0, + "browser_media_response_events": 0, + "uncached_media_responses": 0, + "cached_media_responses": 0, + "thumbnail_resource_entries": 0, + "thumbnail_resource_unique_urls": 0, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 0, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 12.799999952316284, + "navigation_dom_content_loaded_ms": 11.799999952316284, + "resource_duration_ms": [] + }, + { + "posts_with_images": 4, + "sample": 0, + "state": "cold-browser-cache", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 4, + "cached_media_responses": 0, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 133264, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 135536, + "navigation_load_event_end_ms": 20.399999976158142, + "navigation_dom_content_loaded_ms": 19.19999998807907, + "resource_duration_ms": [ + 5.300000011920929, + 2.400000035762787, + 2.900000035762787, + 3 + ] + }, + { + "posts_with_images": 4, + "sample": 1, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 10, + "navigation_dom_content_loaded_ms": 9.199999988079071, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 4, + "sample": 2, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 13, + "navigation_dom_content_loaded_ms": 11.800000011920929, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 4, + "sample": 3, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 9.399999976158142, + "navigation_dom_content_loaded_ms": 8.5, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 4, + "sample": 4, + "state": "warm", + "thumbnail_elements": 4, + "loaded_elements_including_repeated_urls": 4, + "browser_media_response_events": 4, + "uncached_media_responses": 0, + "cached_media_responses": 4, + "thumbnail_resource_entries": 4, + "thumbnail_resource_unique_urls": 4, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 132064, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 9.800000011920929, + "navigation_dom_content_loaded_ms": 8.899999976158142, + "resource_duration_ms": [ + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 0, + "state": "cold-browser-cache", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 25, + "cached_media_responses": 0, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 832900, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 847100, + "navigation_load_event_end_ms": 52.80000001192093, + "navigation_dom_content_loaded_ms": 46.10000002384186, + "resource_duration_ms": [ + 4.699999988079071, + 3.300000011920929, + 4.699999988079071, + 3.700000047683716, + 5, + 5.100000023841858, + 5.100000023841858, + 6.199999988079071, + 7.399999976158142, + 7.599999964237213, + 7.699999988079071, + 7.799999952316284, + 7.799999952316284, + 7.800000011920929, + 9, + 9.600000023841858, + 10, + 9.900000035762787, + 10, + 9.199999988079071, + 11, + 11.5, + 11.300000011920929, + 11.400000035762787, + 11.5 + ] + }, + { + "posts_with_images": 128, + "sample": 1, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 44.10000002384186, + "navigation_dom_content_loaded_ms": 37.19999998807907, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 2, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 35.89999997615814, + "navigation_dom_content_loaded_ms": 30, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 3, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 36.30000001192093, + "navigation_dom_content_loaded_ms": 30, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + { + "posts_with_images": 128, + "sample": 4, + "state": "warm", + "thumbnail_elements": 128, + "loaded_elements_including_repeated_urls": 49, + "browser_media_response_events": 25, + "uncached_media_responses": 0, + "cached_media_responses": 25, + "thumbnail_resource_entries": 25, + "thumbnail_resource_unique_urls": 25, + "resource_timing_transfer_bytes": 0, + "resource_timing_encoded_body_bytes": 825400, + "cdp_encoded_transfer_bytes": 0, + "navigation_load_event_end_ms": 51.39999997615814, + "navigation_dom_content_loaded_ms": 40.89999997615814, + "resource_duration_ms": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + ], + "limitations": "Debug server, local loopback HTTP, synthetic 1024x768 gradient source, 250px lossless WebP preview, 128 posts share 65 unique URLs. Initial viewport measured for 500ms after load. Browser setup/CDP overhead; cold browser cache, warm server/OS cache. Resource Timing adds an estimated 300-byte header cost; CDP records transfer separately. Load-event timings are observations, not isolated proof of frontend speedup. Original loaded only after click; missing/corrupt fallback and missing-file recovery asserted. Raw output remains in local output/playwright and /tmp." +} diff --git a/docs/thumbnail-evidence/http-baseline.json b/docs/thumbnail-evidence/http-baseline.json new file mode 100644 index 00000000..a8b30287 --- /dev/null +++ b/docs/thumbnail-evidence/http-baseline.json @@ -0,0 +1,346 @@ +{ + "limitations": "Debug build, loopback HTTP/1.1, warm OS cache, 16 concurrent requests, current-thread runtime; middleware/browser cache excluded. DB checkouts measured; SQL count and syscall count require separate instrumentation. ps CPU cumulative; RSS sampled, not peak. Original removed before requests.", + "samples": [ + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 33439, + "latency": { + "max_us": 27858, + "p50_us": 2232, + "p95_us": 25280 + }, + "process_cpu_time_and_rss_kib": "0:00.72 47952", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 25510, + "latency": { + "max_us": 5208, + "p50_us": 2804, + "p95_us": 3498 + }, + "process_cpu_time_and_rss_kib": "0:00.76 48064", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 32583, + "latency": { + "max_us": 6700, + "p50_us": 3604, + "p95_us": 4722 + }, + "process_cpu_time_and_rss_kib": "0:00.80 48432", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 28545, + "latency": { + "max_us": 6059, + "p50_us": 3001, + "p95_us": 4957 + }, + "process_cpu_time_and_rss_kib": "0:00.84 48544", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 30023, + "latency": { + "max_us": 6610, + "p50_us": 3430, + "p95_us": 4088 + }, + "process_cpu_time_and_rss_kib": "0:00.88 48656", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 27221, + "latency": { + "max_us": 5912, + "p50_us": 2920, + "p95_us": 3883 + }, + "process_cpu_time_and_rss_kib": "0:00.92 48656", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 31531, + "latency": { + "max_us": 6433, + "p50_us": 3621, + "p95_us": 4483 + }, + "process_cpu_time_and_rss_kib": "0:00.96 48656", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 27485, + "latency": { + "max_us": 6402, + "p50_us": 2961, + "p95_us": 3613 + }, + "process_cpu_time_and_rss_kib": "0:01.00 48784", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 30575, + "latency": { + "max_us": 6260, + "p50_us": 3503, + "p95_us": 4425 + }, + "process_cpu_time_and_rss_kib": "0:01.04 48800", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 25194, + "latency": { + "max_us": 5474, + "p50_us": 2680, + "p95_us": 3471 + }, + "process_cpu_time_and_rss_kib": "0:01.08 48800", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 29966, + "latency": { + "max_us": 6182, + "p50_us": 3401, + "p95_us": 4376 + }, + "process_cpu_time_and_rss_kib": "0:01.12 48800", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 25804, + "latency": { + "max_us": 5505, + "p50_us": 2757, + "p95_us": 5041 + }, + "process_cpu_time_and_rss_kib": "0:01.15 48800", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 28657, + "latency": { + "max_us": 6732, + "p50_us": 3242, + "p95_us": 4174 + }, + "process_cpu_time_and_rss_kib": "0:01.19 48832", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 25678, + "latency": { + "max_us": 5548, + "p50_us": 2810, + "p95_us": 3251 + }, + "process_cpu_time_and_rss_kib": "0:01.22 48832", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 31150, + "latency": { + "max_us": 4532, + "p50_us": 3649, + "p95_us": 4131 + }, + "process_cpu_time_and_rss_kib": "0:01.27 48832", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 26708, + "latency": { + "max_us": 5658, + "p50_us": 2863, + "p95_us": 4983 + }, + "process_cpu_time_and_rss_kib": "0:01.30 48832", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 29845, + "latency": { + "max_us": 6263, + "p50_us": 3341, + "p95_us": 5603 + }, + "process_cpu_time_and_rss_kib": "0:01.34 48960", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 25653, + "latency": { + "max_us": 5442, + "p50_us": 2716, + "p95_us": 3586 + }, + "process_cpu_time_and_rss_kib": "0:01.38 48976", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 31436, + "latency": { + "max_us": 4213, + "p50_us": 3703, + "p95_us": 4012 + }, + "process_cpu_time_and_rss_kib": "0:01.42 48976", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 27288, + "latency": { + "max_us": 5716, + "p50_us": 2897, + "p95_us": 4989 + }, + "process_cpu_time_and_rss_kib": "0:01.46 48976", + "requests": 128 + } + } + ], + "thumbnail_bytes": 556 +} \ No newline at end of file diff --git a/docs/thumbnail-evidence/http-final-repeat.json b/docs/thumbnail-evidence/http-final-repeat.json new file mode 100644 index 00000000..2dcfb02f --- /dev/null +++ b/docs/thumbnail-evidence/http-final-repeat.json @@ -0,0 +1,346 @@ +{ + "limitations": "Debug build, loopback HTTP/1.1, warm OS cache, 16 concurrent requests, current-thread runtime; middleware/browser cache excluded. DB checkouts measured; SQL count and syscall count require separate instrumentation. ps CPU cumulative; RSS sampled, not peak. Original removed before requests.", + "samples": [ + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 34380, + "latency": { + "max_us": 28665, + "p50_us": 1811, + "p95_us": 17779 + }, + "process_cpu_time_and_rss_kib": "0:00.70 48064", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 17250, + "latency": { + "max_us": 3026, + "p50_us": 1807, + "p95_us": 2812 + }, + "process_cpu_time_and_rss_kib": "0:00.74 48224", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22986, + "latency": { + "max_us": 4204, + "p50_us": 2517, + "p95_us": 3220 + }, + "process_cpu_time_and_rss_kib": "0:00.79 48352", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18741, + "latency": { + "max_us": 3649, + "p50_us": 1953, + "p95_us": 3238 + }, + "process_cpu_time_and_rss_kib": "0:00.83 48496", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 21114, + "latency": { + "max_us": 4138, + "p50_us": 2277, + "p95_us": 3365 + }, + "process_cpu_time_and_rss_kib": "0:00.88 48608", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 17841, + "latency": { + "max_us": 3543, + "p50_us": 1845, + "p95_us": 2505 + }, + "process_cpu_time_and_rss_kib": "0:00.93 48608", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22351, + "latency": { + "max_us": 4437, + "p50_us": 2493, + "p95_us": 3324 + }, + "process_cpu_time_and_rss_kib": "0:00.98 48672", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 19291, + "latency": { + "max_us": 3764, + "p50_us": 2001, + "p95_us": 3560 + }, + "process_cpu_time_and_rss_kib": "0:01.02 48800", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 21414, + "latency": { + "max_us": 3477, + "p50_us": 2478, + "p95_us": 2745 + }, + "process_cpu_time_and_rss_kib": "0:01.07 48816", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18079, + "latency": { + "max_us": 3742, + "p50_us": 1869, + "p95_us": 3372 + }, + "process_cpu_time_and_rss_kib": "0:01.11 48816", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22742, + "latency": { + "max_us": 4190, + "p50_us": 2655, + "p95_us": 3413 + }, + "process_cpu_time_and_rss_kib": "0:01.16 48816", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18891, + "latency": { + "max_us": 3638, + "p50_us": 1987, + "p95_us": 3186 + }, + "process_cpu_time_and_rss_kib": "0:01.20 48816", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22022, + "latency": { + "max_us": 4206, + "p50_us": 2505, + "p95_us": 3632 + }, + "process_cpu_time_and_rss_kib": "0:01.24 48816", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18390, + "latency": { + "max_us": 3722, + "p50_us": 1890, + "p95_us": 3401 + }, + "process_cpu_time_and_rss_kib": "0:01.28 48816", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22432, + "latency": { + "max_us": 3022, + "p50_us": 2606, + "p95_us": 2897 + }, + "process_cpu_time_and_rss_kib": "0:01.33 48816", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18502, + "latency": { + "max_us": 3709, + "p50_us": 1924, + "p95_us": 3381 + }, + "process_cpu_time_and_rss_kib": "0:01.37 48816", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 20991, + "latency": { + "max_us": 4568, + "p50_us": 2301, + "p95_us": 3353 + }, + "process_cpu_time_and_rss_kib": "0:01.41 48832", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 17826, + "latency": { + "max_us": 3552, + "p50_us": 1839, + "p95_us": 2386 + }, + "process_cpu_time_and_rss_kib": "0:01.46 48832", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 21886, + "latency": { + "max_us": 3624, + "p50_us": 2505, + "p95_us": 2930 + }, + "process_cpu_time_and_rss_kib": "0:01.50 48832", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 19120, + "latency": { + "max_us": 3870, + "p50_us": 1983, + "p95_us": 3374 + }, + "process_cpu_time_and_rss_kib": "0:01.55 48832", + "requests": 128 + } + } + ], + "thumbnail_bytes": 556 +} \ No newline at end of file diff --git a/docs/thumbnail-evidence/http-final.json b/docs/thumbnail-evidence/http-final.json new file mode 100644 index 00000000..611f842e --- /dev/null +++ b/docs/thumbnail-evidence/http-final.json @@ -0,0 +1,346 @@ +{ + "limitations": "Debug build, loopback HTTP/1.1, warm OS cache, 16 concurrent requests, current-thread runtime; middleware/browser cache excluded. DB checkouts measured; SQL count and syscall count require separate instrumentation. ps CPU cumulative; RSS sampled, not peak. Original removed before requests.", + "samples": [ + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 29652, + "latency": { + "max_us": 27912, + "p50_us": 1705, + "p95_us": 25540 + }, + "process_cpu_time_and_rss_kib": "0:00.73 47888", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 17498, + "latency": { + "max_us": 3436, + "p50_us": 1802, + "p95_us": 2880 + }, + "process_cpu_time_and_rss_kib": "0:00.77 47984", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22750, + "latency": { + "max_us": 4217, + "p50_us": 2596, + "p95_us": 3475 + }, + "process_cpu_time_and_rss_kib": "0:00.82 48240", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18747, + "latency": { + "max_us": 3712, + "p50_us": 1938, + "p95_us": 2577 + }, + "process_cpu_time_and_rss_kib": "0:00.86 48368", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 21775, + "latency": { + "max_us": 3444, + "p50_us": 2514, + "p95_us": 2940 + }, + "process_cpu_time_and_rss_kib": "0:00.90 48416", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18058, + "latency": { + "max_us": 3776, + "p50_us": 1870, + "p95_us": 3274 + }, + "process_cpu_time_and_rss_kib": "0:00.95 48432", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22328, + "latency": { + "max_us": 4932, + "p50_us": 2488, + "p95_us": 3292 + }, + "process_cpu_time_and_rss_kib": "0:00.99 48480", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18924, + "latency": { + "max_us": 3689, + "p50_us": 1983, + "p95_us": 3355 + }, + "process_cpu_time_and_rss_kib": "0:01.03 48608", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22027, + "latency": { + "max_us": 3941, + "p50_us": 2530, + "p95_us": 3217 + }, + "process_cpu_time_and_rss_kib": "0:01.08 48624", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18363, + "latency": { + "max_us": 3765, + "p50_us": 1898, + "p95_us": 2403 + }, + "process_cpu_time_and_rss_kib": "0:01.13 48624", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22624, + "latency": { + "max_us": 4185, + "p50_us": 2542, + "p95_us": 3624 + }, + "process_cpu_time_and_rss_kib": "0:01.17 48624", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 19327, + "latency": { + "max_us": 3793, + "p50_us": 2003, + "p95_us": 3194 + }, + "process_cpu_time_and_rss_kib": "0:01.21 48624", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22308, + "latency": { + "max_us": 4124, + "p50_us": 2506, + "p95_us": 3108 + }, + "process_cpu_time_and_rss_kib": "0:01.26 48656", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18789, + "latency": { + "max_us": 3693, + "p50_us": 1930, + "p95_us": 2603 + }, + "process_cpu_time_and_rss_kib": "0:01.30 48656", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22698, + "latency": { + "max_us": 4602, + "p50_us": 2609, + "p95_us": 2880 + }, + "process_cpu_time_and_rss_kib": "0:01.35 48656", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18904, + "latency": { + "max_us": 4054, + "p50_us": 1963, + "p95_us": 2493 + }, + "process_cpu_time_and_rss_kib": "0:01.39 48656", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 21806, + "latency": { + "max_us": 3040, + "p50_us": 2521, + "p95_us": 2851 + }, + "process_cpu_time_and_rss_kib": "0:01.43 48656", + "requests": 128 + } + }, + { + "admin_cookie": false, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 18195, + "latency": { + "max_us": 3699, + "p50_us": 1878, + "p95_us": 2438 + }, + "process_cpu_time_and_rss_kib": "0:01.47 48672", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": false, + "measurement": { + "body_bytes": 71168, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 22505, + "latency": { + "max_us": 4205, + "p50_us": 2589, + "p95_us": 3549 + }, + "process_cpu_time_and_rss_kib": "0:01.52 48672", + "requests": 128 + } + }, + { + "admin_cookie": true, + "conditional": true, + "measurement": { + "body_bytes": 0, + "concurrency": 16, + "database_checkouts": 128, + "elapsed_us": 19186, + "latency": { + "max_us": 3865, + "p50_us": 1996, + "p95_us": 2664 + }, + "process_cpu_time_and_rss_kib": "0:01.56 48672", + "requests": 128 + } + } + ], + "thumbnail_bytes": 556 +} \ No newline at end of file diff --git a/docs/thumbnail-pipeline-audit.md b/docs/thumbnail-pipeline-audit.md new file mode 100644 index 00000000..59c0b2ea --- /dev/null +++ b/docs/thumbnail-pipeline-audit.md @@ -0,0 +1,214 @@ +# Thumbnail pipeline audit — 2026-10-04 + +Audited from `c7e4be7` on branch `1.7.0-staging`. No dependencies, schema, +thumbnail encoding, templates, CSS, post URLs, or production JavaScript changed. + +**Lifecycle and existing behavior** + +- Multipart uploads stream to temporary files. MIME/structure, board policy, + size and decoded-image limits are validated before persistence. The existing + `MediaUploadGate` permits one public media upload at a time, including parsing + and processing. Decoder validation remains mandatory even for deduplicated + uploads. Originals are capped at 40 million decoded pixels and decoder + allocations at 256 MiB. +- `handlers::process_primary_upload` hashes the upload and consults `file_hashes`. + A same-board hit reuses the stored original and thumbnail when both exist; + missing outputs cause upload-time processing again. Empty thumbnail paths are + valid for generic files. No content decode is performed on thumbnail GETs. +- `utils/files/storage.rs` stages UUID-named files, invokes + `MediaProcessor::process_upload`, and records the actual thumbnail output path. + Image conversion happens first; thumbnail generation decodes the persisted + result. JPEG orientation and metadata stripping retain their existing behavior. + Validation, conversion and thumbnailing can therefore decode a fresh image + separately; that work happens on upload, never on page or thumbnail delivery. +- `media/thumbnail.rs` fits still previews into the configured square (default + 250 × 250), preserving aspect ratio and avoiding enlargement of small images. + Triangle filtering and the `image` crate's lossless WebP encoding remain + unchanged; there is no lossy quality setting. Animated media uses a still first + composited frame for its preview; the original conversion preserves animation. + PDF first pages use Hayro; video frames use optional FFmpeg then the same Rust + WebP encoder. Failures can produce generated 250 × 250 SVG placeholders. Audio + initially uses SVG, then durable workers publish waveform PNGs. Background + workers are capped at four; queued job claiming/completion is transactional. +- Upload finalization persists post/file-hash paths together with filesystem + recovery intent. Referenced thumbnails survive cache eviction. Ordinary + page loads do not regenerate previews. Generation failure can leave an original + without a thumbnail; pending/failed/pruned media has dedicated template states. +- Board, catalog, archive, thread and preview/update APIs render stored paths + into `/boards/{board}/thumbs/{filename}` URLs. Rendering does not stat or decode + thumbnail files. Catalog/index/archive images are lazy with async decoding; + thread OPs are eager and replies lazy. CSS limits post previews to 150 px and + replies to 80 px (with responsive overrides). Posts store no thumbnail + width/height metadata, so post images have no intrinsic-size HTML hints. +- `static/main.js` installs one load/error fallback per image; an already complete + image with no natural size may call `decode()` before showing the fallback. + There is no custom thumbnail download queue or original-image prefetch. The + full image's `src` is assigned from `data-src` on expansion; videos/audio use + `preload="none"`, PDFs use `about:blank` until opened. Native browser request + scheduling and caching handle concurrency and repeated URLs. Cross-board hover + previews also deduplicate in-flight HTML fetches and retain a per-page HTML + cache; pending media is refreshed through the existing post-update endpoint. No-JS links still + open the original. Sensitive-content/NSFW, quote-preview and media hooks remain + intact. +- `/boards/{*media_path}` checks live board permissions before every origin + request, including conditional GETs. Path validation canonicalizes root and + target, checks every path component for symlinks, verifies a regular file and + rejects multiple hardlinks on Unix. The leaf is inspected during the component + walk and again after canonicalization; the file service then obtains metadata + from its opened file. Those checks were preserved. `ServeFile` streams only the + requested file in bounded chunks (default 64 KiB). It supports HEAD, byte + ranges, Last-Modified/If-Modified-Since and metadata-derived + ETag/If-None-Match; ETags do not hash/read image contents. GET/304 still opens + the file and reads metadata; 304 does not read a payload. WebP/PNG are not + HTTP-compressed again. Generated SVGs retain restrictive CSP/nosniff headers. +- Public UUID media already had `public, max-age=31536000, immutable`. + View-password media uses `private, no-cache, must-revalidate`; unversioned + replaceable board favicons use a one-hour TTL. Revalidated private media still + checks access. Missing SVG waveforms redirect temporarily to their completed + PNGs; old MP4/MKV URLs redirect permanently to WebM siblings. Corrupt persisted + thumbnail bytes receive an ordinary file response and the browser displays a + fallback; serving never decodes or repairs them. + +**Findings and implemented changes** + +The browser baseline did not expose duplicate downloads, eager original loading, +far-offscreen unique thumbnail floods, or unnecessary warm-cache media requests. +The worthwhile changes were on the server request path: + +1. Synchronous canonicalization, per-component filesystem checks and legacy + sibling resolution previously ran on the async executor after DB preflight. + They now run in that same blocking preflight. The DB connection is released + before filesystem work. A rare replacement/open race also resolves its legacy + sibling on a blocking worker. Security checks and their syscall count remain + intact; the improvement is executor responsiveness and overlapping work. +2. A public/post-password request carrying an admin cookie previously performed a + session SELECT despite already having view access. Session lookup now occurs + only when an unresolved view-password gate requires administrator access. + Valid board unlock cookies also avoid that extra lookup. Board access is still + loaded from the DB each time; no authorization cache was introduced. Public + cookie-bearing requests go from two application SELECTs to one (code-path + accounting; not a SQL trace). Pool validation SQL is separate. +3. Media errors now carry private no-store headers. Previously an initial 404 had + no explicit cache policy, and a failed open after successful validation could + inherit the one-year immutable policy. A missing/generated-later file or a + temporary denial must not be retained as immutable content. This is a + correctness improvement established from the response path; no negative-cache + performance win is claimed. Successful 200/206/304 policies remain unchanged. +4. Response metadata uses the validated file path, avoiding the earlier redundant + root/path construction. No large image buffers or application cache were added. + +**Measured evidence** + +The tracked ignored Rust workload runs the production handler and `ServeFile` +through real loopback HTTP/1.1: 128 pre-generated 250 px WebP files, concurrency +16, five batches for each visitor/admin-cookie and full/conditional combination. +The original is deleted before requests, proving delivery is independent of +source decoding/resizing. It measures complete-response latency, body bytes and +DB checkouts, with cumulative CPU/end-of-batch RSS observations from `ps`. + +Apple M2 Pro, arm64 macOS, Rust 1.99.0, debug build, current-thread Tokio harness, +warm filesystem/server cache. These are local workload results, not a release +capacity claim. Moving blocking calls can have a different timing effect on a +multi-thread production runtime or slower storage. + +| 128-request batch | Before median ms | After median ms | Before request p50/p95 ms | After request p50/p95 ms | +| --- | ---: | ---: | ---: | ---: | +| Visitor, full responses | 30.023 | 22.027 | 3.341 / 4.425 | 2.514 / 3.108 | +| Visitor, conditional 304 | 25.653 | 18.195 | 2.804 / 3.498 | 1.878 / 2.603 | +| Admin cookie, full responses | 31.436 | 22.624 | 3.621 / 4.376 | 2.589 / 3.475 | +| Admin cookie, conditional 304 | 27.288 | 18.924 | 2.897 / 4.983 | 1.983 / 2.664 | + +An independent optimized repeat recorded 21.414, 17.841, 22.432 and 18.891 ms, +respectively. Every full batch transferred 71,168 body bytes before and after; +304 batches transferred zero body bytes. All origin batches checked out 128 DB +connections. No reduction in filesystem calls, total CPU or RSS is established. +`ps` includes client/setup/instrumentation and is not a peak-memory or isolated +server-CPU measurement. Syscalls and SQL statements were not traced on this host. + +The local browser workload extends `tests/e2e/helpers.ts` and diagnostics rather +than introducing a framework. It uploads a real 1024 × 768 gradient, then seeds +0, 4 and 128 image-post cases. The many-post page has 65 unique thumbnails, +including repeated URLs. CDP observes cache/response events without request +routing (routing would disable browser cache); Resource Timing observes bytes. +Five visits per size include a cleared browser cache followed by four warm visits. + +| Browser case | Before / after unique origin thumbnail responses | Before / after Resource Timing transfer bytes | +| --- | ---: | ---: | +| No images, cold or warm | 0 / 0 | 0 / 0 | +| Four images, cold | 4 / 4 | 133,264 / 133,264 | +| Four images, warm | 0 / 0 | 0 / 0 | +| 128 posts, initial cold viewport | 25 / 25 | 832,900 / 832,900 | +| 128 posts, warm viewport | 0 / 0 | 0 / 0 | +| 128 posts, complete scroll | 65 / 65 total unique URLs | No duplicate URL requests | + +Browser WebP previews were 33,016 bytes in both runs. The initial many-post +viewport transferred 825,400 body bytes plus Resource Timing's estimated header +cost. Some far-offscreen repeated elements obtain natural dimensions from shared +cached URLs; that does not represent additional network downloads. Cold load-event +completion was 52.0 ms before and 52.8 ms after. Warm navigation timings varied, +including the no-image control, so they do not establish a frontend speedup. +Opening the image loaded the correct 1024 px original; missing/corrupt previews +showed fallbacks, and a missing file created later recovered on navigation. + +Raw HTTP samples: [baseline](thumbnail-evidence/http-baseline.json), +[optimized](thumbnail-evidence/http-final.json), +[repeat](thumbnail-evidence/http-final-repeat.json). Browser aggregate/per-resource +observations: [baseline](thumbnail-evidence/browser-baseline.json), +[optimized](thumbnail-evidence/browser-final.json). Full browser artifacts remain +under the repository's ignored `output/playwright/thumbnails/` directory. The new +browser spec remains local under ignored `tests/e2e/`, following `.gitignore`'s +existing policy; the Rust benchmark and evidence are versionable. + +**Validation and reproduction** + +```sh +cargo fmt --all --check +cargo clippy --workspace --all-targets --all-features -- -D warnings -D clippy::all -D clippy::pedantic -D clippy::nursery -D clippy::cargo +cargo test --workspace --all-features +cargo test --workspace --all-features -- --test-threads=1 +RUSTCHAN_THUMB_EVIDENCE=/tmp/thumbnail-http.json cargo test --locked --lib server::handlers::board::media::performance::thumbnail_http_workload -- --ignored --exact +RUSTCHAN_E2E_SKIP_BUILD=1 RUSTCHAN_AUDIT_OUTPUT=output/playwright/thumbnails/check npx playwright test tests/e2e/thumbnail-loading.spec.ts --project=chromium --workers=1 +RUSTCHAN_E2E_SKIP_BUILD=1 npx playwright test tests/e2e/media.spec.ts tests/e2e/password-boards.spec.ts tests/e2e/phase3-media-runtime.spec.ts tests/e2e/non-video-media.spec.ts --project=chromium --project=webkit --project=chromium-nojs --workers=2 +``` + +Build the current server with `cargo build --locked --bin rustchan-cli` before +using `RUSTCHAN_E2E_SKIP_BUILD=1`. Browser setup also generates its native update +fixture using the existing global setup. + +Formatting and strict Clippy passed. New request regressions cover ETag/date 304, +HEAD, byte ranges, original independence, missing-file recovery/no-store, +permission changes, unlock retirement, session revocation, and legacy redirects. +The browser matrix passed 29 tests with 10 existing engine/capability skips; +the many-image benchmark passed before and after. Full parallel Rust runs exposed +shared global configuration-save locking and thumbnail-failure-injection +interference with a PDF test. The complete serial run passed: 1,353 library +tests, 1,353 binary tests, five CLI runtime tests, one native UI fixture test and +one doc test (2,713 passes total; 10 intentional ignores across library/binary). +No unrelated test or production configuration logic was modified. + +**Deliberately deferred** + +- No stronger public caching, application thumbnail cache, static-route bypass, + or permission cache: existing warm visits already generate zero media origin + work, while private media must recheck authorization. +- No lossy codecs, smaller previews or extra responsive variants: preview quality, + compatibility and existing high-DPI presentation take precedence; network bytes + did not regress. Lossless gradient previews can be relatively large. +- No runtime dimension probing. Intrinsic dimension metadata would require a + coordinated schema/upload/restore/template change; guessing square dimensions + would alter portrait/landscape layout. Profile layout shifts before doing that. +- No shared decoded-image upload pipeline, new generation queue, or single-flight + cache: upload processing is already gated, requests never generate thumbnails, + and fusing validation/conversion would broaden this change into media semantics. +- No reduction of symlink/hardlink/canonicalization checks, permanent root trust + cache, or alternate file-server implementation: the checks protect untrusted + stored paths. A descriptor-based race-hardening design is separate work. +- Public immutable retention and cached corrupt-file repair at the same URL retain + their existing semantics; replacing immutable assets should use a new URL. + +**Files changed** + +- `src/handlers/board/media.rs`: delivery preflight, error cache policy and request regressions. +- `src/handlers/board/media/performance.rs`: opt-in actual HTTP workload. +- `docs/thumbnail-pipeline-audit.md` and `docs/thumbnail-evidence/{http-baseline,http-final,http-final-repeat,browser-baseline,browser-final}.json`: audit and measured evidence. +- Local ignored `tests/e2e/thumbnail-loading.spec.ts`: browser workload and recovery assertions. diff --git a/src/handlers/board/media.rs b/src/handlers/board/media.rs index 6f0a499f..0bd86000 100644 --- a/src/handlers/board/media.rs +++ b/src/handlers/board/media.rs @@ -1,7 +1,7 @@ use super::{ board_access_cookie_from_jar, db, load_board_access_context, templates, unlock_redirect_url, - user_preferences_from_jar, AppError, AppState, BoardAccessContext, CookieJar, HeaderMap, Path, - Redirect, Response, Result, State, StatusCode, ADMIN_SESSION_COOKIE, CONFIG, + user_preferences_from_jar, AppError, AppState, CookieJar, HeaderMap, Path, Redirect, Response, + Result, State, StatusCode, ADMIN_SESSION_COOKIE, CONFIG, }; use axum::http::header::{ HeaderValue, CONTENT_DISPOSITION, CONTENT_SECURITY_POLICY, CONTENT_TYPE, @@ -9,6 +9,10 @@ use axum::http::header::{ }; use axum::response::IntoResponse as _; +#[cfg(test)] +#[path = "media/performance.rs"] +mod performance; + fn media_content_type(path: &std::path::Path) -> Option<&'static str> { match path.extension().and_then(|e| e.to_str()) { Some("ico") => Some("image/x-icon"), @@ -109,6 +113,42 @@ fn stale_waveform_redirect_path(base: &std::path::Path, media_path: &str) -> Opt Some(location) } +/// Validated delivery target, resolved on a blocking worker after authorization. +enum BoardMediaTarget { + File(std::path::PathBuf), + Redirect(Redirect), + Missing, +} + +/// Preserve legacy URLs without doing filesystem work on the async executor. +fn stale_media_redirect(base: &std::path::Path, media_path: &str) -> Option { + stale_waveform_redirect_path(base, media_path) + .map(|path| Redirect::temporary(&path)) + .or_else(|| { + stale_webm_redirect_path(base, media_path).map(|path| Redirect::permanent(&path)) + }) +} + +/// Validate every component; only an absent file may resolve to a legacy sibling. +fn resolve_board_media_target(base: &std::path::Path, media_path: &str) -> BoardMediaTarget { + match safe_board_media_file(base, media_path) { + Ok(path) => BoardMediaTarget::File(path), + Err(error) if is_not_found_error(&error) => stale_media_redirect(base, media_path) + .map_or(BoardMediaTarget::Missing, BoardMediaTarget::Redirect), + Err(_) => BoardMediaTarget::Missing, + } +} + +/// A missing/generated-later file or denied request must not be negatively cached. +fn media_error_response(status: StatusCode) -> Response { + let mut response = status.into_response(); + crate::cache::set_cache_control( + response.headers_mut(), + crate::cache::CACHE_CONTROL_PRIVATE_NO_STORE, + ); + response +} + // Legacy `.mp4` and `.mkv` links redirect to transcoded `.webm` files; all // other validated paths are served directly. @@ -122,49 +162,52 @@ pub(in crate::server) async fn serve_board_media( jar: CookieJar, req: axum::extract::Request, ) -> Response { - use std::path::PathBuf; use tower::ServiceExt as _; use tower_http::services::ServeFile; // Reject path-traversal attempts and absolute-path escapes. if media_path.contains("..") || media_path.starts_with('/') { - return StatusCode::BAD_REQUEST.into_response(); + return media_error_response(StatusCode::BAD_REQUEST); } let Some(board_short) = media_path.split('/').next().filter(|part| !part.is_empty()) else { - return StatusCode::NOT_FOUND.into_response(); + return media_error_response(StatusCode::NOT_FOUND); }; let admin_session_id = jar .get(ADMIN_SESSION_COOKIE) .map(|cookie| cookie.value().to_owned()); let access_cookie = board_access_cookie_from_jar(&jar, board_short); - let access_context = match tokio::task::spawn_blocking({ + let (is_protected_board, target) = match tokio::task::spawn_blocking({ let pool = state.db.clone(); let board_short = board_short.to_owned(); - move || -> Result { + let media_path = media_path.clone(); + move || -> Result<(bool, BoardMediaTarget)> { let conn = pool.get()?; - load_board_access_context( - &conn, - &board_short, - admin_session_id.as_deref(), - access_cookie.as_deref(), - ) + let board = db::get_board_by_short(&conn, &board_short)? + .ok_or_else(|| AppError::NotFound(format!("Board /{board_short}/ not found")))?; + // Public/post-password boards and valid unlock cookies already grant + // viewing. Only a remaining password gate needs a session lookup. + if !super::can_view_board(&board, false, access_cookie.as_deref()) + && !super::posting::is_admin_session(&conn, admin_session_id.as_deref()) + { + return Err(AppError::Forbidden("Board access denied".to_owned())); + } + let is_protected_board = board.access_mode.requires_view_password(); + // Release the pooled connection before the synchronous path checks. + drop(conn); + let target = + resolve_board_media_target(std::path::Path::new(&CONFIG.upload_dir), &media_path); + Ok((is_protected_board, target)) } }) .await { Ok(Ok(context)) => context, - Ok(Err(AppError::NotFound(_))) => return StatusCode::NOT_FOUND.into_response(), - Ok(Err(_)) | Err(_) => return StatusCode::FORBIDDEN.into_response(), + Ok(Err(AppError::NotFound(_))) => return media_error_response(StatusCode::NOT_FOUND), + Ok(Err(_)) | Err(_) => return media_error_response(StatusCode::FORBIDDEN), }; - if !access_context.can_view { - return StatusCode::FORBIDDEN.into_response(); - } - - let base = PathBuf::from(&CONFIG.upload_dir); - let target = base.join(&media_path); let has_version = req .uri() .query() @@ -173,118 +216,93 @@ pub(in crate::server) async fn serve_board_media( .components() .nth(1) .is_some_and(|part| part.as_os_str() == "_favicon"); - let is_pdf = target + let updated_target = match target { + BoardMediaTarget::File(path) => path, + BoardMediaTarget::Redirect(redirect) => return redirect.into_response(), + BoardMediaTarget::Missing => return media_error_response(StatusCode::NOT_FOUND), + }; + let is_pdf = updated_target .extension() .and_then(|ext| ext.to_str()) .is_some_and(|ext| ext.eq_ignore_ascii_case("pdf")); - let updated_target = match safe_board_media_file(&base, &media_path) { - Ok(path) => Some(path), - Err(error) - if is_not_found_error(&error) - && (std::path::Path::new(&media_path) - .extension() - .is_some_and(|ext| { - ext.eq_ignore_ascii_case("mp4") || ext.eq_ignore_ascii_case("mkv") - }) - || is_generated_svg_placeholder_thumb(&media_path)) => - { - None + // File present — forward the real request (with Range, ETag, etc.) to + // ServeFile so it can respond with 206 Partial Content when needed. + // iOS Safari requires Range request support to play video — dropping + // the request headers caused it to receive 200 instead of 206 and + // refuse playback on videos it tried to stream in chunks. + let req = req.map(|_| axum::body::Body::empty()); + let response = ServeFile::new(&updated_target).oneshot(req).await; + // A worker may replace the SVG/video between validation and open. + if response + .as_ref() + .is_ok_and(|resp| resp.status() == StatusCode::NOT_FOUND) + { + let stale_media_path = media_path.clone(); + let redirect = tokio::task::spawn_blocking(move || { + stale_media_redirect(std::path::Path::new(&CONFIG.upload_dir), &stale_media_path) + }) + .await; + if let Ok(Some(redirect)) = redirect { + return redirect.into_response(); } - Err(_) => return StatusCode::NOT_FOUND.into_response(), - }; - - if let Some(updated_target) = updated_target { - // File present — forward the real request (with Range, ETag, etc.) to - // ServeFile so it can respond with 206 Partial Content when needed. - // iOS Safari requires Range request support to play video — dropping - // the request headers caused it to receive 200 instead of 206 and - // refuse playback on videos it tried to stream in chunks. - let req = req.map(|_| axum::body::Body::empty()); - ServeFile::new(&updated_target) - .oneshot(req) - .await - .map_or_else( - |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), - |resp| { - // The waveform worker can remove the SVG between our path check - // and ServeFile opening it. Resolve the completed PNG again. - if let Some(path) = (resp.status() == StatusCode::NOT_FOUND) - .then(|| stale_waveform_redirect_path(&base, &media_path)) - .flatten() - { - return Redirect::temporary(&path).into_response(); - } - if let Some(path) = (resp.status() == StatusCode::NOT_FOUND) - .then(|| stale_webm_redirect_path(&base, &media_path)) - .flatten() - { - return Redirect::permanent(&path).into_response(); - } - let mut resp = resp.map(axum::body::Body::new); - crate::cache::set_cache_control( - resp.headers_mut(), - board_media_cache_control( - access_context.board.access_mode.requires_view_password(), - is_board_favicon, - has_version, - ), - ); - if is_generated_svg_placeholder_thumb(&media_path) { - drop( - resp.headers_mut() - .insert(CONTENT_TYPE, HeaderValue::from_static("image/svg+xml")), - ); - drop( - resp.headers_mut().insert( - X_CONTENT_TYPE_OPTIONS, - HeaderValue::from_static("nosniff"), - ), - ); - drop(resp.headers_mut().insert( - CONTENT_SECURITY_POLICY, - HeaderValue::from_static("default-src 'none'; script-src 'none'"), - )); - } else if let Some(ct) = media_content_type(&updated_target) { - drop( - resp.headers_mut() - .insert(CONTENT_TYPE, HeaderValue::from_static(ct)), - ); - } else { - drop(resp.headers_mut().insert( - CONTENT_TYPE, - HeaderValue::from_static("application/octet-stream"), - )); - drop( - resp.headers_mut().insert( - X_CONTENT_TYPE_OPTIONS, - HeaderValue::from_static("nosniff"), - ), - ); - let filename = updated_target - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("download.bin") - .replace(['\\', '"'], "_"); - let disposition = format!("attachment; filename=\"{filename}\""); - let parsed_disposition = HeaderValue::from_str(&disposition); - if let Ok(value) = parsed_disposition { - drop(resp.headers_mut().insert(CONTENT_DISPOSITION, value)); - } - } - if is_pdf { - apply_pdf_embed_headers(resp.headers_mut()); - } - resp.into_response() - }, - ) - } else if let Some(redirect_path) = stale_waveform_redirect_path(&base, &media_path) { - Redirect::temporary(&redirect_path).into_response() - } else if let Some(redirect_path) = stale_webm_redirect_path(&base, &media_path) { - Redirect::permanent(&redirect_path).into_response() - } else { - StatusCode::NOT_FOUND.into_response() } + response.map_or_else( + |_| media_error_response(StatusCode::INTERNAL_SERVER_ERROR), + |resp| { + let mut resp = resp.map(axum::body::Body::new); + // Do not give a failed open (including a delete race) a year's TTL. + let cache_control = + if resp.status().is_success() || resp.status() == StatusCode::NOT_MODIFIED { + board_media_cache_control(is_protected_board, is_board_favicon, has_version) + } else { + crate::cache::CACHE_CONTROL_PRIVATE_NO_STORE + }; + crate::cache::set_cache_control(resp.headers_mut(), cache_control); + if is_generated_svg_placeholder_thumb(&media_path) { + drop( + resp.headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("image/svg+xml")), + ); + drop( + resp.headers_mut() + .insert(X_CONTENT_TYPE_OPTIONS, HeaderValue::from_static("nosniff")), + ); + drop(resp.headers_mut().insert( + CONTENT_SECURITY_POLICY, + HeaderValue::from_static("default-src 'none'; script-src 'none'"), + )); + } else if let Some(ct) = media_content_type(&updated_target) { + drop( + resp.headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static(ct)), + ); + } else { + drop(resp.headers_mut().insert( + CONTENT_TYPE, + HeaderValue::from_static("application/octet-stream"), + )); + drop( + resp.headers_mut() + .insert(X_CONTENT_TYPE_OPTIONS, HeaderValue::from_static("nosniff")), + ); + let filename = updated_target + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("download.bin") + .replace(['\\', '"'], "_"); + let disposition = format!("attachment; filename=\"{filename}\""); + let parsed_disposition = HeaderValue::from_str(&disposition); + if let Ok(value) = parsed_disposition { + drop(resp.headers_mut().insert(CONTENT_DISPOSITION, value)); + } + } + if is_pdf { + apply_pdf_embed_headers(resp.headers_mut()); + } + resp.into_response() + }, + ) } const fn board_media_cache_control( @@ -469,6 +487,287 @@ mod tests { use super::safe_board_media_file; use super::stale_webm_redirect_path; use anyhow::{ensure, Context as _, Result as AnyResult}; + use axum::{ + body::{to_bytes, Body}, + http::{header, Request}, + routing::get, + Router, + }; + use tower::ServiceExt as _; + + /// Creates real files below the configured root and an isolated database. + fn media_fixture() -> AnyResult<(crate::middleware::AppState, tempfile::TempDir, String)> { + let state = crate::test_support::app_state(); + let directory = tempfile::Builder::new() + .prefix("t") + .tempdir_in(&super::CONFIG.upload_dir)?; + let board = directory + .path() + .file_name() + .and_then(|name| name.to_str()) + .context("fixture board name")? + .to_owned(); + crate::db::create_board(&*state.db.get()?, &board, "Media", "", false).map(|_id| ())?; + std::fs::create_dir(directory.path().join("thumbs"))?; + std::fs::write( + directory.path().join("thumbs/image.webp"), + b"thumbnail bytes", + )?; + std::fs::write(directory.path().join("original.webp"), b"original bytes")?; + Ok((state, directory, board)) + } + + /// Runs an actual handler request, optionally carrying access and validators. + async fn request_media( + router: &Router, + uri: &str, + cookie: Option<&str>, + modified: Option<&str>, + ) -> AnyResult { + let mut request = Request::builder().uri(uri); + if let Some(cookie) = cookie { + request = request.header(header::COOKIE, cookie); + } + if let Some(modified) = modified { + request = request.header(header::IF_MODIFIED_SINCE, modified); + } + Ok(router.clone().oneshot(request.body(Body::empty())?).await?) + } + + #[tokio::test] + async fn thumbnail_delivery_preserves_http_validators() -> AnyResult<()> { + let (state, _directory, board) = media_fixture()?; + let router = Router::new() + .route("/boards/{*media_path}", get(super::serve_board_media)) + .with_state(state); + let uri = format!("/boards/{board}/thumbs/image.webp"); + let response = request_media(&router, &uri, None, None).await?; + ensure!(response.status() == super::StatusCode::OK); + ensure!( + response + .headers() + .get(header::CACHE_CONTROL) + .context("cache policy")? + == crate::cache::CACHE_CONTROL_IMMUTABLE_MEDIA + ); + ensure!( + response + .headers() + .get(header::CONTENT_TYPE) + .context("content type")? + == "image/webp" + ); + let modified = response + .headers() + .get(header::LAST_MODIFIED) + .context("validator")? + .to_str()? + .to_owned(); + let etag = response + .headers() + .get(header::ETAG) + .context("etag")? + .to_str()? + .to_owned(); + ensure!(to_bytes(response.into_body(), 1024).await?.as_ref() == b"thumbnail bytes"); + let conditional = request_media(&router, &uri, None, Some(&modified)).await?; + ensure!(conditional.status() == super::StatusCode::NOT_MODIFIED); + ensure!(to_bytes(conditional.into_body(), 1024).await?.is_empty()); + let conditional_etag = router + .clone() + .oneshot( + Request::builder() + .uri(&uri) + .header(header::IF_NONE_MATCH, &etag) + .body(Body::empty())?, + ) + .await?; + ensure!(conditional_etag.status() == super::StatusCode::NOT_MODIFIED); + ensure!( + conditional_etag + .headers() + .get(header::ETAG) + .context("conditional etag")? + .to_str()? + == etag + ); + ensure!(to_bytes(conditional_etag.into_body(), 1024) + .await? + .is_empty()); + Ok(()) + } + + #[tokio::test] + async fn thumbnail_delivery_preserves_head_ranges_and_missing_file_recovery() -> AnyResult<()> { + let (state, directory, board) = media_fixture()?; + let router = Router::new() + .route("/boards/{*media_path}", get(super::serve_board_media)) + .with_state(state); + let uri = format!("/boards/{board}/thumbs/image.webp"); + let head = router + .clone() + .oneshot( + Request::builder() + .method("HEAD") + .uri(&uri) + .body(Body::empty())?, + ) + .await?; + ensure!(head.status() == super::StatusCode::OK); + ensure!( + head.headers() + .get(header::CONTENT_LENGTH) + .context("head size")? + == "15" + ); + ensure!(to_bytes(head.into_body(), 1024).await?.is_empty()); + let range = router + .clone() + .oneshot( + Request::builder() + .uri(&uri) + .header(header::RANGE, "bytes=0-4") + .body(Body::empty())?, + ) + .await?; + ensure!(range.status() == super::StatusCode::PARTIAL_CONTENT); + ensure!(to_bytes(range.into_body(), 1024).await?.as_ref() == b"thumb"); + let original_uri = format!("/boards/{board}/original.webp"); + let original = request_media(&router, &original_uri, None, None).await?; + ensure!(to_bytes(original.into_body(), 1024).await?.as_ref() == b"original bytes"); + // No original file, post lookup, or runtime image generation is required. + std::fs::remove_file(directory.path().join("original.webp"))?; + ensure!(request_media(&router, &uri, None, None).await?.status() == super::StatusCode::OK); + let missing = format!("/boards/{board}/thumbs/missing.webp"); + let missing_response = request_media(&router, &missing, None, None).await?; + ensure!(missing_response.status() == super::StatusCode::NOT_FOUND); + ensure!( + missing_response + .headers() + .get(header::CACHE_CONTROL) + .context("missing policy")? + == crate::cache::CACHE_CONTROL_PRIVATE_NO_STORE + ); + std::fs::write( + directory.path().join("thumbs/missing.webp"), + b"generated later", + )?; + ensure!( + request_media(&router, &missing, None, None).await?.status() == super::StatusCode::OK + ); + Ok(()) + } + + #[tokio::test] + async fn conditional_media_requests_recheck_unlock_and_session_revocation() -> AnyResult<()> { + let (state, _directory, board) = media_fixture()?; + let router = Router::new() + .route("/boards/{*media_path}", get(super::serve_board_media)) + .with_state(state.clone()); + let uri = format!("/boards/{board}/thumbs/image.webp"); + let public = request_media(&router, &uri, None, None).await?; + let modified = public + .headers() + .get(header::LAST_MODIFIED) + .context("validator")? + .to_str()? + .to_owned(); + { + let conn = state.db.get()?; + conn.execute("UPDATE boards SET access_mode='view_password', access_password_hash='fixture-hash' WHERE short_name=?1", [&board]).map(|_rows| ())?; + let admin = crate::db::create_admin(&conn, "media-admin", "fixture-hash")?; + crate::db::create_session(&conn, "media-session", admin, i64::MAX)?; + } + let unlock_value = super::super::expected_board_access_cookie_value(&board, "fixture-hash") + .context("unlock value")?; + let unlock = format!( + "{}={unlock_value}", + super::super::board_access_cookie_name(&board) + ); + let admin = format!("{}=media-session", super::ADMIN_SESSION_COOKIE); + let invalid = format!("{}=invalid", super::ADMIN_SESSION_COOKIE); + for cookie in [None, Some(invalid.as_str())] { + ensure!( + request_media(&router, &uri, cookie, Some(&modified)) + .await? + .status() + == super::StatusCode::FORBIDDEN + ); + } + for cookie in [&unlock, &admin] { + let response = request_media(&router, &uri, Some(cookie), Some(&modified)).await?; + ensure!(response.status() == super::StatusCode::NOT_MODIFIED); + ensure!( + response + .headers() + .get(header::CACHE_CONTROL) + .context("private policy")? + == crate::cache::CACHE_CONTROL_PRIVATE_NO_CACHE + ); + } + state.db.get()?.execute_batch( + "DELETE FROM admin_sessions; UPDATE boards SET access_password_hash='changed';", + )?; + for cookie in [&unlock, &admin] { + ensure!( + request_media(&router, &uri, Some(cookie), Some(&modified)) + .await? + .status() + == super::StatusCode::FORBIDDEN + ); + } + state + .db + .get()? + .execute_batch("UPDATE boards SET access_mode='post_password';")?; + ensure!( + request_media(&router, &uri, Some(&invalid), Some(&modified)) + .await? + .status() + == super::StatusCode::NOT_MODIFIED + ); + Ok(()) + } + + #[tokio::test] + async fn legacy_media_redirects_preserve_status_and_safe_targets() -> AnyResult<()> { + let (state, directory, board) = media_fixture()?; + std::fs::write(directory.path().join("clip.webm"), b"video")?; + std::fs::write(directory.path().join("thumbs/audio.png"), b"waveform")?; + let router = Router::new() + .route("/boards/{*media_path}", get(super::serve_board_media)) + .with_state(state); + for (requested, target, status) in [ + ( + "clip.mp4", + "clip.webm", + super::StatusCode::PERMANENT_REDIRECT, + ), + ( + "clip.mkv", + "clip.webm", + super::StatusCode::PERMANENT_REDIRECT, + ), + ( + "thumbs/audio.svg", + "thumbs/audio.png", + super::StatusCode::TEMPORARY_REDIRECT, + ), + ] { + let uri = format!("/boards/{board}/{requested}"); + let response = request_media(&router, &uri, None, None).await?; + ensure!(response.status() == status); + ensure!( + response + .headers() + .get(header::LOCATION) + .context("redirect location")? + .to_str()? + == format!("/boards/{board}/{target}") + ); + } + Ok(()) + } #[test] fn stale_waveform_svg_redirects_only_to_an_existing_generated_png() -> AnyResult<()> { diff --git a/src/handlers/board/media/performance.rs b/src/handlers/board/media/performance.rs new file mode 100644 index 00000000..0f317f41 --- /dev/null +++ b/src/handlers/board/media/performance.rs @@ -0,0 +1,193 @@ +//! Opt-in HTTP thumbnail workload using the production handler and file service. + +use anyhow::{ensure, Context as _, Result}; +use axum::{routing::get, Router}; +use futures::{stream, StreamExt as _, TryStreamExt as _}; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; +use std::time::Instant; + +/// Counts pooled checkouts during measured requests, excluding fixture creation. +#[derive(Clone, Debug, Default)] +struct CheckoutProbe(Arc); + +impl r2d2::HandleEvent for CheckoutProbe { + fn handle_checkout(&self, _event: r2d2::event::CheckoutEvent) { + let _previous = self.0.fetch_add(1, Ordering::Relaxed); + } +} + +/// Summarizes complete HTTP response times without imposing noisy time limits. +fn latencies(mut values: Vec) -> serde_json::Value { + values.sort_unstable(); + let pick = |percent: usize| { + values + .get( + values + .len() + .saturating_mul(percent) + .div_ceil(100) + .saturating_sub(1), + ) + .copied() + .unwrap_or(0) + }; + serde_json::json!({"p50_us":pick(50), "p95_us":pick(95), "max_us":values.last().copied().unwrap_or(0)}) +} + +/// Samples process observations; CPU is cumulative and RSS is an end-of-batch sample. +fn process_observation() -> Result { + let output = std::process::Command::new("ps") + .args(["-o", "time=,rss=", "-p", &std::process::id().to_string()]) + .output()?; + ensure!(output.status.success(), "ps failed"); + Ok(String::from_utf8(output.stdout)?.trim().to_owned()) +} + +/// Sends a bounded concurrent batch through actual loopback HTTP connections. +async fn batch( + client: &reqwest::Client, + base: &str, + cookie: bool, + modified: Option<&str>, + probe: &CheckoutProbe, +) -> Result { + probe.0.store(0, Ordering::Relaxed); + let started = Instant::now(); + let results: Vec<_> = stream::iter(0_u32..128) + .map(|number| async move { + let request_started = Instant::now(); + let mut request = client.get(format!("{base}/thumbs/{number}.webp")); + if cookie { + request = request.header( + reqwest::header::COOKIE, + format!("{}=benchmark-invalid", super::ADMIN_SESSION_COOKIE), + ); + } + if let Some(modified) = modified { + request = request.header(reqwest::header::IF_MODIFIED_SINCE, modified); + } + let response = request.send().await?; + let expected = if modified.is_some() { + reqwest::StatusCode::NOT_MODIFIED + } else { + reqwest::StatusCode::OK + }; + ensure!( + response.status() == expected, + "unexpected thumbnail status {}", + response.status() + ); + ensure!( + response + .headers() + .get(reqwest::header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + == Some(crate::cache::CACHE_CONTROL_IMMUTABLE_MEDIA), + "immutable cache policy changed" + ); + let bytes = response.bytes().await?.len(); + Ok::<_, anyhow::Error>((request_started.elapsed().as_micros(), bytes)) + }) + .buffer_unordered(16) + .try_collect() + .await?; + Ok(serde_json::json!({ + "requests":results.len(), "concurrency":16_u32, "elapsed_us":started.elapsed().as_micros(), + "body_bytes":results.iter().map(|entry| entry.1).sum::(), + "latency":latencies(results.iter().map(|entry| entry.0).collect()), + "database_checkouts":probe.0.load(Ordering::Relaxed), + "process_cpu_time_and_rss_kib":process_observation()?, + })) +} + +#[tokio::test] +#[ignore = "explicit 128-thumbnail HTTP benchmark; set RUSTCHAN_THUMB_EVIDENCE"] +async fn thumbnail_http_workload() -> Result<()> { + // Normal server startup installs this; isolated handler tests bypass startup. + drop(rustls::crypto::ring::default_provider().install_default()); + let output = std::env::var("RUSTCHAN_THUMB_EVIDENCE")?; + let mut state = crate::test_support::app_state(); + let database = state + .db + .get()? + .path() + .context("test database path")? + .to_owned(); + let probe = CheckoutProbe::default(); + state.db = r2d2::Pool::builder() + .max_size(8) + .event_handler(Box::new(probe.clone())) + .build(r2d2_sqlite::SqliteConnectionManager::file(database))?; + let directory = tempfile::Builder::new() + .prefix("t") + .tempdir_in(&crate::config::CONFIG.upload_dir)?; + let board = directory + .path() + .file_name() + .and_then(|name| name.to_str()) + .context("board directory name")?; + crate::db::create_board(&*state.db.get()?, board, "Thumbnails", "", false).map(|_id| ())?; + let thumbs = directory.path().join("thumbs"); + std::fs::create_dir(&thumbs)?; + let original = directory.path().join("source.png"); + image::RgbImage::from_fn(1024, 768, |x, y| { + image::Rgb([ + u8::try_from(x % 256).unwrap_or(0), + u8::try_from(y % 256).unwrap_or(0), + 80, + ]) + }) + .save(&original)?; + let generated = crate::media::thumbnail::generate_thumbnail( + &original, + "image/png", + &thumbs.join("0.webp"), + 250, + false, + )?; + for number in 1_u32..128 { + std::fs::copy(&generated, thumbs.join(format!("{number}.webp"))).map(|_bytes| ())?; + } + // Delivery must succeed after the original disappears: no request-time resize. + std::fs::remove_file(original)?; + let router = Router::new() + .route("/boards/{*media_path}", get(super::serve_board_media)) + .with_state(state); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base = format!("http://{}/boards/{board}", listener.local_addr()?); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async { + drop(stopped.await); + }) + .await + }); + let result = async { + let client = reqwest::Client::new(); + let response = client.get(format!("{base}/thumbs/127.webp")).send().await?; + let modified = response.headers().get(reqwest::header::LAST_MODIFIED).context("Last-Modified missing")?.to_str()?.to_owned(); + drop(response.bytes().await?); + let mut samples = Vec::new(); + for _sample in 0_u32..5 { + for cookie in [false, true] { + for conditional in [false, true] { + let measured = batch(&client, &base, cookie, conditional.then_some(modified.as_str()), &probe).await?; + samples.push(serde_json::json!({"admin_cookie":cookie, "conditional":conditional, "measurement":measured})); + } + } + } + std::fs::write(output, serde_json::to_vec_pretty(&serde_json::json!({ + "thumbnail_bytes":std::fs::metadata(generated)?.len(), "samples":samples, + "limitations":"Debug build, loopback HTTP/1.1, warm OS cache, 16 concurrent requests, current-thread runtime; middleware/browser cache excluded. DB checkouts measured; SQL count and syscall count require separate instrumentation. ps CPU cumulative; RSS sampled, not peak. Original removed before requests." + }))?)?; + Ok::<_, anyhow::Error>(()) + }.await; + stop.send(()) + .map_err(|()| anyhow::anyhow!("benchmark server stopped early"))?; + server.await.context("join benchmark server")??; + result +} From 270127d2fd540c42533acc9a8de120e56e2e240b Mon Sep 17 00:00:00 2001 From: csd113 Date: Mon, 5 Oct 2026 14:49:11 -0700 Subject: [PATCH 02/14] Harden thread archiving and retention lifecycle --- SETUP.md | 39 ++ docs/strict-lint-exceptions.tsv | 1 - docs/thread-archive-audit.md | 255 ++++++++++++ src/config/template.rs | 3 +- src/db/posts.rs | 11 +- src/db/schema.rs | 19 +- src/db/threads.rs | 365 ++++++----------- src/db/threads/archive_tests.rs | 574 +++++++++++++++++++++++++++ src/handlers/admin/content.rs | 87 ++-- src/handlers/admin/settings/board.rs | 127 +++++- src/handlers/board/catalog.rs | 28 +- src/handlers/board/tests.rs | 199 ++++++++++ src/handlers/posting.rs | 13 +- src/handlers/thread.rs | 2 +- src/templates/admin.rs | 1 + src/templates/board.rs | 24 +- src/templates/thread.rs | 81 +++- src/workers/mod.rs | 104 +++-- static/style.css | 4 + 19 files changed, 1581 insertions(+), 356 deletions(-) create mode 100644 docs/thread-archive-audit.md create mode 100644 src/db/threads/archive_tests.rs diff --git a/SETUP.md b/SETUP.md index 45015054..8922652e 100644 --- a/SETUP.md +++ b/SETUP.md @@ -723,3 +723,42 @@ resource limits and verification. ## Applying administrator configuration Live controls keep applying immediately. Other controls show their active, saved and next-start values. Saving never restarts RustChan; managed Linux and opted-in supervised containers expose **Restart RustChan** for pending settings. See [settings restarts](docs/settings-restarts.md) for the complete classification, supervisor setup, graceful shutdown and failure recovery. + +## Thread archives and retention + +Threads leave the active index when background maintenance finds more than the +board's **Max threads** non-sticky active threads. It keeps the newest last-bump +and thread-ID pairs; locked threads count, sticky active threads do not. New +thread creation durably schedules maintenance, so overflow can be visible +briefly. Startup and periodic reconciliation recover outstanding work. + +Replies bump until the board's bump limit is reached. Sage replies count toward +that limit without bumping. The bump limit stops bumps; it does not immediately +archive the thread. There is no time-based thread expiry. + +**Archive overflow threads** controls automatic archival, with the global +`archive_before_prune` safety net (default `true`) also enabling it. Hard deletion +of active overflow occurs only when both are off. Existing archives and their +navigation remain available even when automatic archival is disabled. + +**Max archived threads** is a count cap (1–10,000), not permanent retention. +Maintenance keeps the most recently bumped archived threads, breaking ties by +ascending thread ID. Archived sticky threads are also subject to the cap; +sticky is not an archive preservation setting. Lowering the cap permanently +removes overflow threads. Invalid retention values are rejected rather than +clamped; omitted retention fields preserve the saved limits. + +Archival keeps the same `/{board}/thread/{id}` URL and media references. Archived +threads are read-only, including polls; users can browse their results, quoting, +spoilers and attachments. Board search includes archived posts. Board index, +catalog and unread history show active threads. Administrators can manually +archive or delete a thread using the existing authenticated, CSRF-protected +controls. There is no public unarchive/preservation control. + +The archive and prune transitions use one SQLite write transaction. Pruned +media is removed only when no other database references remain, using durable, +restart-replayable file cleanup. Failed filesystem cleanup stays queued and +logged. Archive policy is read when maintenance executes, not from stale job +payloads. Changing the global TOML/environment safety net requires restart; +board settings are live. Monitor background-job failures if maintenance is +unable to acquire the database lock or complete cleanup. diff --git a/docs/strict-lint-exceptions.tsv b/docs/strict-lint-exceptions.tsv index aa34f802..58c2e3e8 100644 --- a/docs/strict-lint-exceptions.tsv +++ b/docs/strict-lint-exceptions.tsv @@ -341,7 +341,6 @@ src/workers/mod.rs 1380 expect clippy::cognitive_complexity source validation an src/workers/mod.rs 1551 expect clippy::too_many_arguments the finalisation boundary carries the durable job and all validated filesystem identities fn transcode_video_finalise src/workers/mod.rs 1593 expect clippy::too_many_arguments output installation and atomic DB completion require the full validated transcode identity fn persist_transcoded_webm src/workers/mod.rs 1934 expect clippy::too_many_arguments the finalisation boundary explicitly carries all validated paths and DB identity fn waveform_finalise -src/workers/mod.rs 2115 expect clippy::cognitive_complexity archive, prune, and filesystem finalization remain one consistency operation fn prune_threads src/workers/mod.rs 2561 expect clippy::cognitive_complexity,clippy::too_many_lines the bounded legacy-state matrix is intentionally explicit fn reconcile_media_job_states src/workers/mod.rs 3050 expect clippy::expect_used test database queries stop locally with fixture-specific failure context fn file_hash_count src/workers/mod.rs 3227 expect clippy::expect_used fixture setup failures should stop this queue durability test fn required_thread_prune_bypasses_capacity_and_survives_queue_recreation diff --git a/docs/thread-archive-audit.md b/docs/thread-archive-audit.md new file mode 100644 index 00000000..c5dc5de9 --- /dev/null +++ b/docs/thread-archive-audit.md @@ -0,0 +1,255 @@ +# Thread/archive production-readiness audit + +## Architecture before changes + +Threads remain in `threads` with an `archived` boolean; posts, polls, submission +tokens and media references stay in place. There is no separate archive table, +archive timestamp, time-based expiry, or public unarchive action. The internal +unarchive helper preserves the lock until explicitly unlocked. + +Public thread creation commits the thread, OP, optional poll, file publication +intent and coalesced durable `thread_prune` job in one immediate transaction. +Replies serialize thread state, reply count and bump-limit checks under SQLite's +write lock. Sage replies increment the count without bumping. Reaching the bump +limit stops future bumps; it does not itself archive a thread. Deleting replies +reduces the reply count but preserves bump time. Deleting the OP deletes its thread. + +`workers::prune_threads` loads current board settings and archives overflow +non-sticky active threads by descending bump time. Sticky active threads do not +consume `max_threads`; locked threads do. Archiving atomically sets archived and +locked. `allow_archive || CONFIG.archive_before_prune` decides whether active +overflow is archived or deleted. The global safety net defaults to true. +Startup and periodic bounded reconciliation schedule any outstanding overflow; +creation and moderation can temporarily exceed limits pending maintenance. + +Archive retention is a count cap, `max_archived_threads`, not a duration. All +archived threads, including manually archived sticky threads, participate; sticky +is not an archive preservation flag. Retention keeps most recently bumped +content, not most recently manually archived content. Deletion cascades database +references and commits a durable file-delete intent after checking all remaining +references. Filesystem replay checks references again and resumes after restart. +Archiving itself neither moves nor deletes media. Active-media pruning excludes +archives, and the normal media reconciliation pipeline remains authoritative. + +`/{board}/archive` lists 20 threads per numbered page. It paginates threads before +aggregating post/media counts. `/{board}/thread/{id}` remains the stable URL across +archival and shares the active renderer, quoting, escaping, thumbnails and CSP. +Thread loads check the actual board. Board-password gates apply to archive, +thread, search and media routes. Catalog, index and unread history exclude +archives; board FTS and admin searches include archived posts. Replies and owner +edit/delete controls reject archived content under the write lock. + +Administrators can archive and delete threads using session, origin and CSRF +checks. No moderator-scoped role or preservation/unarchive UI exists. Board +limits are exposed through established board settings; global archive policy +uses TOML/environment configuration. Workers validate limits before deletion. + +## Findings and implementation evidence + +| Problem and root cause | Repair | Regression coverage | +| --- | --- | --- | +| Equal bump timestamps had no explicit retention tie-breaker, allowing planner choice to disagree with visible ordering. | Active retention uses `bumped_at DESC, id DESC`; archive retention uses the existing archive order, `bumped_at DESC, id ASC`. | `archive_threshold_ties_sticky_locked_and_idempotency`, `live_deletion_ties_keep_the_visible_newest_threads`, `archive_pages_and_retention_share_tie_order_and_search_survives` | +| Maintenance read policy before acquiring its write lock, then archived and pruned in separate transactions. A failed prune could leave a partially completed maintenance pass. | Read current policy, archive/live-prune, archive-cap pruning and file intents under one immediate transaction; finalize files after commit. | `archive_and_prune_worker_rolls_back_as_one_operation`, `combined_retention_failure_rolls_back_archive_transition`, existing `delayed_prune_uses_only_current_limits` and `delayed_prune_uses_current_archived_limit` | +| Overflow recovery built an unbounded SQLite parameter list. Large restored/backlogged boards could fail maintenance. | Archive using a parameterized selection subquery; batch file collection and deletion into at most 500 IDs per statement. | `archive_recovery_handles_more_than_sqlite_parameter_limit` uses 33,000 threads. | +| Hand-written archive/prune transactions relied on individual rollback/commit paths. | RAII transactions roll back on body or commit failure and release the write lock. | `failed_archive_commit_releases_the_lock_and_rolls_back`, `archive_busy_failure_and_restart_are_recoverable`, `process_interruption_during_archive_recovers_without_content_loss` | +| Invalid administrator retention values were clamped to one or defaulted to 150, potentially deleting content; omitted fields reset existing caps. Direct DB callers also lacked destructive-limit validation. | Reject invalid explicit values with HTTP 400; preserve omitted caps; validate again at the archive/prune DB boundary. | `retention_settings_reject_invalid_values_and_preserve_omitted_limits`, `invalid_retention_http_settings_cannot_reduce_saved_caps`, `destructive_limits_fail_closed_at_database_boundary`; browser settings rejection assertions | +| Existing/manual/safety-net archives became inaccessible through the archive index when `allow_archive` was false. Navigation used the same incorrect condition. | Archive access and navigation are independent of automatic overflow policy. | `archive_index_remains_accessible_when_overflow_archiving_is_disabled`; browser automatic/manual archival scenarios | +| Archived polls still offered voting and accepted mutations, despite the thread being read-only. | Render poll results and `[archived]`; enforce `threads.archived=0` inside the vote INSERT itself. | `archived_poll_is_read_only_in_thread_and_vote_route`, extended vote membership/expiry regression, browser poll rejection assertions | +| Manual moderation could succeed without an audit record, and archive logs used the supplied board name. Manual archive/unsticky changes lacked an immediate durable retention request. | Session authorization, state mutation, canonical archive log and retention intent commit together. | `manual_archive_rolls_back_when_audit_logging_fails`; browser anonymous/CSRF rejection and forged-board audit/redirect checks | +| Archive counts and rows were separate snapshots; access preflight could become stale during a password change. | Revalidate board access and read board/count/page under one SQLite read snapshot. | `archived_pages_enforce_board_view_passwords` checks protected archive and stable thread URLs, forged cookies and post-password viewability; pagination regressions | +| Archived replies were reported as merely locked because that check ran first. | Prioritize archive status during preflight and the authoritative reply write check. | Extended `reply_rechecks_thread_state_after_independent_preparation`; browser checks the precise archived error. | + +These changes preserve background enforcement: the active cap is authoritative +when maintenance runs, not a synchronous cap on thread creation. Sticky active +threads remain exempt, locked active threads remain eligible, and archived sticky +threads remain subject to archive retention. No time-based expiry, preservation +flag, separate archive table, or reopening interface was introduced. + +## Database, concurrency and recovery + +The only schema change replaces `idx_threads_board_sticky_bumped` with: + +```sql +CREATE INDEX idx_threads_active_order +ON threads(board_id, sticky DESC, bumped_at DESC, id DESC) +WHERE archived = 0; +``` + +The existing strict schema-repair path recognizes the exact legacy index, +installs the replacement and removes the redundant old index in its migration +transaction. The existing upgrade regression reconstructs that legacy schema and +verifies data survives repair. No columns, foreign keys or dependency versions +changed. The existing archive index continues to support archive pagination and +retention order. + +File-backed WAL tests race creators, replies and maintenance, and race +stickying/locking/deleting against archival. SQLite write transactions determine +the outcome; no application-wide mutex was added. Two overlapping worker passes +converge to the same caps. Tests also exercise lock contention, failed DELETE +triggers, deferred-foreign-key commit failure, reopening the database, and a +subprocess killed after its archive UPDATE but before commit. The latter verifies +the original active state returns intact and maintenance can safely retry. + +New concurrency tests are +`overlapping_creators_and_archive_passes_preserve_every_thread`, +`concurrent_oldest_reply_and_archive_obey_the_committed_bump`, and +`concurrent_moderation_and_archive_are_serialized_by_sqlite`. +`sage_bump_limit_and_deleted_replies_preserve_archive_order` verifies sage, +bump-limit and reply-deletion behavior, including rejection of archived replies +even if an administrator clears the lock flag. + +## Media, routing, search and security + +Archive rendering uses the existing shared post renderer. The new +`archived_media_and_markup_use_the_same_safe_post_renderer` compares active and +archived reply output for image, video, audio, PDF, other-file and legacy media, +including companion audio, quotes, spoilers and hostile names/filenames. +`archived_media_survives_and_shared_paths_are_never_pruned` verifies archival +preserves references and retention never schedules deletion of a shared original, +thumbnail or companion audio path. + +The unchanged media subsystem already tests missing-thumbnail fallback, pruned +originals, processing failures, HTTP validators/ranges, legacy redirects, +reference rechecks and interrupted durable cleanup. Those regressions are +included in the full test suite. Board restore rerenders `body_html` from escaped +raw body through the shared sanitizer; it does not trust archived HTML supplied +by a board-backup manifest. No separate archive rendering or conversion pipeline +was added. + +Stable thread URLs remain valid after archival; wrong-board, negative, +nonexistent and pruned IDs fail through the existing route checks. Board archive +and archived thread pages retain view-password protection. Poll voting is now +read-only at the SQL write boundary. Administrator actions retain session, +origin and CSRF enforcement and now fail atomically when audit logging fails. +Archive requests remain parameterized, user display fields stay escaped, media +delivery retains path validation, and the existing CSP is unchanged. + +Search still includes archives and uses the existing FTS index. Index, catalog +and unread activity still exclude archives. Pruning removes searchable posts +through existing cascades/triggers. No cache of archive rows was introduced; +cookie-dependent responses retain `Vary: Cookie`. + +## Archive UI and administration + +Archive navigation is always discoverable from board/index/catalog views. Rows +show thread number, state, reply and attachment counts, creation and last-bump +times. The page states its actual retention cap; empty-state copy is clearer. +Decorative thumbnails use empty alternative text and archive rows have visible +theme-aware keyboard focus outlines. Existing routes, numbered pagination, +element hooks, progressive enhancement and no-JS browsing are preserved. + +Administrator guidance now explicitly says lowering the archive cap permanently +prunes overflow, archived sticky threads are included, and the global safety net +can override the automatic-archive checkbox. `SETUP.md` documents creation, +sage/bump limits, count retention, media cleanup, manual moderation, search and +live versus restart-required policy changes. The generated TOML comment correctly +describes in-place archival rather than a nonexistent archive table. + +## Performance evidence + +The explicit `archive_query_profile` test uses 10,000 threads and 210,000 posts, +including 9,900 archived threads. Both index candidates receive `ANALYZE` before +comparison. These local Rust 1.99/SQLite debug-build measurements are illustrative, +not production latency guarantees. + +| Operation | Evidence | +| --- | --- | +| 200 active-retention selections before | 13.903 ms; `SEARCH threads USING INDEX idx_threads_archived (board_id=? AND archived=?)`, then `USE TEMP B-TREE FOR LAST TERM OF ORDER BY` | +| 200 selections after | 6.169 ms; `SEARCH threads USING COVERING INDEX idx_threads_active_order (board_id=? AND sticky=?)`; no temporary sort, approximately 2.25× faster | +| Archive page, offset 0 | 4.328 ms for 20 loads, approximately 0.216 ms/page | +| Archive page, offset 5,000 | 52.855 ms for 20 loads, approximately 2.643 ms/page | +| Archive page, offset 9,880 | 99.045 ms for 20 loads, approximately 4.952 ms/page | +| Archived thread and posts | 0.404 ms | +| FTS search including archives | 11.893 ms | +| Archive plus retention deleting 950 threads/19,950 posts | 268.596 ms | + +Deep OFFSET cost increases, but the validated archive cap is 10,000 and this +measured cost does not justify replacing established numbered URLs with cursors. +The page-first aggregation remains in place. Separately, the 33,000-thread test +proves backlog maintenance avoids the SQLite bind-parameter scaling trap. + +Reproduce profiling with: + +```sh +RUSTCHAN_ARCHIVE_EVIDENCE=/tmp/rustchan-archive-performance.txt \ + cargo test --locked --lib db::threads::archive_tests::archive_query_profile \ + --all-features -- --ignored --exact +``` + +## Files changed + +- `src/db/threads.rs` and new `src/db/threads/archive_tests.rs`: transitions, deterministic pruning, batching, boundary validation and lifecycle tests. +- `src/db/schema.rs`: partial active-order index and legacy upgrade regression. +- `src/db/posts.rs`: atomic archived-poll vote rejection. +- `src/workers/mod.rs`: atomic current-policy retention and worker rollback regression. +- `src/handlers/admin/content.rs`: atomic audited moderation and retention scheduling. +- `src/handlers/admin/settings/board.rs`: safe retention parsing and handler regressions. +- `src/handlers/board/catalog.rs` and `src/handlers/board/tests.rs`: archive accessibility, consistent authorized snapshot and routing regressions. +- `src/handlers/posting.rs` and `src/handlers/thread.rs`: precise archived reply/vote errors. +- `src/templates/board.rs`, `src/templates/thread.rs`, `src/templates/admin.rs`, `static/style.css`: archive metadata, navigation, read-only polls, administrator guidance and focus styling. +- `src/config/template.rs`, `SETUP.md`, `docs/thread-archive-audit.md`: corrected policy/operational documentation and audit evidence. +- `docs/strict-lint-exceptions.tsv`: remove the worker complexity expectation made obsolete by the focused simplification. + +The new `tests/e2e/archive-production.spec.ts` stays local, as required by the +repository's existing Playwright ignore/check policy. No browser harness, +package manifest or tooling is force-added to Git. + +## Validation results + +Validated on macOS with `rustc 1.99.0 (b940084d7 2026-09-28)`. + +| Command | Result | +| --- | --- | +| `cargo fmt --all --check` | Passed | +| `cargo check --locked --workspace --all-targets --all-features` | Passed | +| `cargo clippy --workspace --all-targets --all-features -- -D warnings -D clippy::all -D clippy::pedantic -D clippy::nursery -D clippy::cargo` | Passed; no new lint allowances | +| `cargo test --workspace --all-features` | Passed: 1,376 unit tests in each library/binary target, six integration tests and one doctest; zero failures. Seven tests are explicitly ignored in each unit target. The archive profiler was run separately; the interrupted child helper runs through its parent regression. | +| `cargo test --locked --lib db::threads::archive_tests --all-features` | Passed: 14 regressions; profiler and subprocess helper intentionally ignored | +| Explicit ignored archive profiler command above | Passed; before/after evidence recorded in this report | +| `RUSTDOCFLAGS='-D warnings' cargo doc --locked --workspace --all-features --no-deps` | Passed | +| `cargo deny --locked check` | Passed advisories, bans, licenses and sources; existing missing-license metadata for `captcha` and duplicate-version warnings remain. Dependencies were unchanged. | +| `python3 tools/check_local_tooling.py` | Passed; local browser tooling remains untracked | +| `git diff --check` | Passed | + +Browser validation used the existing local isolated-runtime Playwright harness: + +```sh +RUSTCHAN_E2E_SKIP_BUILD=1 RUSTCHAN_AUDIT_OUTPUT=output/playwright/archive-final \ + npx playwright test tests/e2e/archive-production.spec.ts --workers=2 +RUSTCHAN_E2E_SKIP_BUILD=1 RUSTCHAN_AUDIT_OUTPUT=output/playwright/archive-existing-final \ + npx playwright test tests/e2e/settings.spec.ts --grep 'bump/archive|thread sage' \ + --project=chromium --workers=2 +RUSTCHAN_E2E_SKIP_BUILD=1 RUSTCHAN_AUDIT_OUTPUT=output/playwright/archive-backlinks-final \ + npx playwright test tests/e2e/archive-production.spec.ts --grep 'automatic archival' \ + --workers=2 +``` + +Results: 14/14 new scenarios across Chromium, WebKit, Firefox, mobile Firefox, +mobile WebKit, Firefox without JS and Chromium without JS; 2/2 unchanged existing +archive scenarios; 7/7 final automatic-archive scenarios after adding explicit +backlink navigation. Coverage includes real image uploads, automatic archival, +stable URLs, quotes/backlinks, media loading, search, pagination, restart, denied +replies/votes, manual moderation authorization/CSRF/canonical audit logs and +invalid settings. All nine built-in themes were checked at 1,280 and 390 pixels, +with overflow/focus assertions and visual screenshot review. No existing test or +diagnostic assertion was weakened. + +One earlier parallel full-suite run hit a transient lock-acquisition failure in +the unchanged `config::admin::tests::atomic_save_validates_before_mutation_and_does_not_change_active_config` +test. The complete default-parallelism rerun passed. This was recorded rather +than changing unrelated configuration behavior or suppressing that test. + +## Remaining limitations + +- Creation can temporarily exceed the active cap while its durable maintenance + job is pending. Failure/retry/reconciliation remain operational concerns. +- Archives are capped by count and bump order. There is no permanent/protected + archive mode, archive timestamp, time retention or public reopening control. +- File removal follows the committed database transition through durable cleanup + and reference checks. It cannot be one atomic transaction across SQLite and + the filesystem; failures remain queued and logged for replay. +- The interrupted-process regression tests SQLite recovery from process death, + not physical power loss or every storage failure. Invalid retention policy + fails closed; this audit does not invent an automatic corrupted-database repair. +- Timing evidence is from this machine and debug fixtures. Deployments should + monitor maintenance backlog and SQLite contention under their own workloads. diff --git a/src/config/template.rs b/src/config/template.rs index 4bc7a92a..c3ac91ab 100644 --- a/src/config/template.rs +++ b/src/config/template.rs @@ -200,7 +200,8 @@ db_warn_threshold_mb = 2048 job_queue_capacity = 1000 # When true, threads that would be hard-deleted by the prune worker are instead -# moved to the archive table, even on boards where archiving is disabled. +# marked archived in place, even on boards where archiving is disabled. +# The board archive count cap still applies; this does not guarantee permanent retention. archive_before_prune = true # Maximum total size (MiB) of all thumbnail/waveform cache files across all boards. diff --git a/src/db/posts.rs b/src/db/posts.rs index 7cd2dcdf..c5da59d0 100644 --- a/src/db/posts.rs +++ b/src/db/posts.rs @@ -1110,7 +1110,7 @@ pub fn get_poll_for_thread( /// /// This returns false for two distinct cases: /// 1. The voter has already voted (UNIQUE constraint fires INSERT OR IGNORE) -/// 2. The option does not belong to the poll, or the poll has expired +/// 2. The option does not belong to the poll, or the poll has expired/archived /// /// Callers that need to distinguish these cases should call `cast_vote` and, on /// false, separately query whether the IP has voted on this poll. A future @@ -1129,7 +1129,9 @@ pub fn cast_vote( SELECT ?1, ?2, ?3 FROM poll_options AS po JOIN polls AS p ON p.id = po.poll_id - WHERE po.id = ?2 + JOIN threads AS t ON t.id = p.thread_id + WHERE t.archived = 0 + AND po.id = ?2 AND po.poll_id = ?1 AND p.expires_at > unixepoch()", params![poll_id, option_id, ip_hash], @@ -4110,6 +4112,11 @@ mod tests { !cast_vote(&conn, expired_poll, expired_option, "late")?, "an expired poll should reject the vote in the write statement" ); + crate::db::set_thread_archived(&conn, thread_id, true)?; + assert!( + !cast_vote(&conn, open_poll, open_option, "archived-voter")?, + "archival must reject votes in the same write statement" + ); Ok(()) } } diff --git a/src/db/schema.rs b/src/db/schema.rs index 99040aa0..727e131b 100644 --- a/src/db/schema.rs +++ b/src/db/schema.rs @@ -274,8 +274,8 @@ const BASE_SCHEMA_SQL: &str = " /// Complete baseline secondary-index definitions. const INDEX_SCHEMA_SQL: &str = " - CREATE INDEX IF NOT EXISTS idx_threads_board_sticky_bumped - ON threads(board_id, sticky DESC, bumped_at DESC); + CREATE INDEX IF NOT EXISTS idx_threads_active_order + ON threads(board_id, sticky DESC, bumped_at DESC, id DESC) WHERE archived = 0; CREATE INDEX IF NOT EXISTS idx_posts_thread ON posts(thread_id, created_at ASC); CREATE INDEX IF NOT EXISTS idx_posts_thread_live @@ -340,7 +340,8 @@ const INDEX_SCHEMA_SQL: &str = " /// Obsolete theme index accepted only during the known legacy repair path. const LEGACY_THEME_SORT_INDEX: &str = "idx_themes_enabled_sort"; /// Additive indexes introduced after the first package-version baseline. -const ADDITIVE_BASELINE_INDEXES: [&str; 11] = [ +const ADDITIVE_BASELINE_INDEXES: [&str; 12] = [ + "idx_threads_active_order", "idx_posts_thread_live", "idx_posts_board_ip_created", "idx_posts_file_path", @@ -354,7 +355,11 @@ const ADDITIVE_BASELINE_INDEXES: [&str; 11] = [ "idx_ban_appeals_ip_created", ]; /// Redundant indexes removed when the additive index set is installed. -const REDUNDANT_LEGACY_INDEXES: [&str; 2] = ["idx_file_hashes", "idx_posts_thread_id"]; +const REDUNDANT_LEGACY_INDEXES: [&str; 3] = [ + "idx_file_hashes", + "idx_posts_thread_id", + "idx_threads_board_sticky_bumped", +]; /// Board columns whose historical default was zero instead of the baseline value. const LEGACY_BOARD_ZERO_DEFAULT_COLUMNS: [&str; 4] = [ "allow_editing", @@ -978,6 +983,8 @@ fn schema_objects_are_legacy_repairable(expected: &SchemaShape, actual: &SchemaS /// Return whether SQL exactly describes a removed redundant index. fn is_redundant_legacy_index(name: &str, sql: &str) -> bool { let expected = match name { + "idx_threads_board_sticky_bumped" => + "CREATE INDEX idx_threads_board_sticky_bumped ON threads(board_id, sticky DESC, bumped_at DESC)", "idx_file_hashes" => "CREATE INDEX idx_file_hashes ON file_hashes(sha256)", "idx_posts_thread_id" => "CREATE INDEX idx_posts_thread_id ON posts(thread_id)", _ => return false, @@ -1080,6 +1087,7 @@ fn apply_additive_schema_repairs_in_transaction(conn: &rusqlite::Connection) -> conn.execute_batch( "DROP INDEX IF EXISTS idx_themes_enabled_sort; DROP INDEX IF EXISTS idx_file_hashes; + DROP INDEX IF EXISTS idx_threads_board_sticky_bumped; DROP INDEX IF EXISTS idx_posts_thread_id;", ) .context("Remove obsolete indexes failed")?; @@ -2484,7 +2492,8 @@ mod tests { conn.execute_batch(&format!("DROP INDEX {index}"))?; } conn.execute_batch( - "CREATE INDEX idx_file_hashes ON file_hashes(sha256); + "CREATE INDEX idx_threads_board_sticky_bumped ON threads(board_id, sticky DESC, bumped_at DESC); + CREATE INDEX idx_file_hashes ON file_hashes(sha256); CREATE INDEX idx_posts_thread_id ON posts(thread_id); CREATE TABLE schema_version (version TEXT NOT NULL PRIMARY KEY);", )?; diff --git a/src/db/threads.rs b/src/db/threads.rs index 5dd525e0..2d45020f 100644 --- a/src/db/threads.rs +++ b/src/db/threads.rs @@ -76,7 +76,7 @@ fn map_thread(row: &rusqlite::Row<'_>) -> rusqlite::Result { /// Collect all file paths (`file_path`, `thumb_path`, `audio_file_path`) for every /// post in the given set of thread ids. Returns a flat Vec of non-null paths. /// -/// Uses a single JOIN query instead of one query per thread. +/// Uses bounded batches instead of one query per thread. /// /// Call before deleting the thread rows; cascading deletion removes the posts /// that supply these paths. @@ -88,35 +88,37 @@ fn collect_thread_file_paths( return Ok(Vec::new()); } - // Build WHERE thread_id IN (?, ?, ...) dynamically. - let placeholders: String = thread_ids - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i.saturating_add(1))) - .collect::>() - .join(", "); - let sql = format!( - "SELECT file_path, thumb_path, audio_file_path + let mut paths = Vec::new(); + for batch in thread_ids.chunks(500) { + // Bound parameters so restored or backlogged boards cannot exceed SQLite limits. + let placeholders: String = batch + .iter() + .enumerate() + .map(|(i, _)| format!("?{}", i.saturating_add(1))) + .collect::>() + .join(", "); + let sql = format!( + "SELECT file_path, thumb_path, audio_file_path FROM posts WHERE thread_id IN ({placeholders})" - ); + ); - let mut stmt = conn.prepare(&sql)?; - let rows: Vec<(Option, Option, Option)> = stmt - .query_map(rusqlite::params_from_iter(thread_ids), |r| { - Ok((r.get(0)?, r.get(1)?, r.get(2)?)) - })? - .collect::>()?; + let mut stmt = conn.prepare(&sql)?; + let rows: Vec<(Option, Option, Option)> = stmt + .query_map(rusqlite::params_from_iter(batch), |r| { + Ok((r.get(0)?, r.get(1)?, r.get(2)?)) + })? + .collect::>()?; - let mut paths = Vec::new(); - for (f, t, a) in rows { - if let Some(p) = f { - paths.push(p); - } - if let Some(p) = t { - paths.push(p); - } - if let Some(p) = a { - paths.push(p); + for (f, t, a) in rows { + if let Some(p) = f { + paths.push(p); + } + if let Some(p) = t { + paths.push(p); + } + if let Some(p) = a { + paths.push(p); + } } } Ok(paths) @@ -495,12 +497,12 @@ pub(crate) fn create_reply_submission( post.thread_id ); }; - if locked { - return Err(anyhow::Error::new(ThreadClosed::Locked)); - } if archived { return Err(anyhow::Error::new(ThreadClosed::Archived)); } + if locked { + return Err(anyhow::Error::new(ThreadClosed::Locked)); + } let post_id = super::posts::create_post_inner(conn, post)?; let updated = if should_bump { @@ -710,243 +712,125 @@ pub fn delete_thread( } // Archive / prune -/// Archive oldest non-sticky threads that exceed the board's `max_threads` limit. +/// Archive overflow non-sticky active threads, keeping the newest bump/ID pairs. /// -/// Archived threads are locked and marked read-only; their content remains -/// accessible via `/{board}/archive`. Returns the count of threads archived -/// (no file deletion occurs). -/// -/// ID selection and the bulk update share a transaction so a concurrent bump -/// cannot change the ordering between those operations. -/// -/// Note: LIMIT -1 OFFSET ? is a SQLite-specific idiom for "skip the first -/// max rows, return everything else". It is not standard SQL. The LIMIT -1 -/// means "no upper bound on the result set after the offset is applied". +/// Archiving locks threads and preserves all posts and media references. +/// Sticky active threads do not count against the limit; locked threads do. /// /// # Errors -/// Returns an error if the database operation fails. +/// Returns an error for invalid limits or a failed transaction. pub fn archive_old_threads(conn: &rusqlite::Connection, board_id: i64, max: i64) -> Result { - conn.execute_batch("BEGIN IMMEDIATE") - .context("Failed to begin archive_old_threads transaction")?; - - let result: Result = (|| { - // Collect inside the transaction to prevent races with concurrent bumps. - let ids: Vec = { - let mut stmt = conn.prepare_cached( - "SELECT id FROM threads - WHERE board_id = ?1 AND sticky = 0 AND archived = 0 - ORDER BY bumped_at DESC LIMIT -1 OFFSET ?2", - )?; - // Bind `collected` explicitly so `stmt` is dropped before the - // block ends — the MappedRows iterator borrows `stmt`, and the - // compiler requires the borrow to end before the binding goes out - // of scope at the closing `}`. - let collected = stmt - .query_map(params![board_id, max], |r| r.get(0))? - .collect::>>()?; - collected - }; - - let count = ids.len(); - if count == 0 { - return Ok(0); - } - - // Single bulk UPDATE instead of N individual statements. - let placeholders: String = ids - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i.saturating_add(1))) - .collect::>() - .join(", "); - let sql = - format!("UPDATE threads SET archived = 1, locked = 1 WHERE id IN ({placeholders})"); - conn.execute(&sql, rusqlite::params_from_iter(&ids)) - .context("Failed to bulk archive threads") - .map(|_affected_rows| ())?; - - Ok(count) - })(); + let tx = rusqlite::Transaction::new_unchecked(conn, rusqlite::TransactionBehavior::Immediate)?; + let count = archive_old_threads_in_tx(&tx, board_id, max)?; + tx.commit()?; + Ok(count) +} - match result { - Ok(0) => { - // Nothing to archive — roll back the (empty) transaction cleanly. - drop(conn.execute_batch("ROLLBACK")); - Ok(0) - } - Ok(count) => { - super::commit_transaction(conn, "Failed to commit archive_old_threads transaction")?; - Ok(count) - } - Err(e) => { - drop(conn.execute_batch("ROLLBACK")); - Err(e) - } - } +/// Archive overflow while the caller holds the SQLite write transaction. +pub(crate) fn archive_old_threads_in_tx( + conn: &rusqlite::Transaction<'_>, + board_id: i64, + max: i64, +) -> Result { + validate_retention_limit(max, false)?; + // A subquery avoids building an unbounded parameter list for backlog recovery. + conn.execute( + "UPDATE threads SET archived = 1, locked = 1 + WHERE id IN ( + SELECT id FROM threads + WHERE board_id = ?1 AND sticky = 0 AND archived = 0 + ORDER BY bumped_at DESC, id DESC LIMIT -1 OFFSET ?2 + )", + params![board_id, max], + ) + .context("Failed to archive overflow threads") } -/// Hard-delete oldest non-sticky, non-archived threads that exceed `max_threads`. -/// Used when a board has archiving disabled — threads are permanently removed. -/// -/// Returns the on-disk paths that are now safe to delete (i.e. no longer -/// referenced by any remaining post after the prune). The caller is responsible -/// for actually removing these files from disk. -/// -/// ID selection, bulk deletion, and the final path-reference check share a -/// transaction so a concurrent insert cannot make a returned path live again. -/// File paths are collected with one joined query. +/// Hard-delete overflow non-sticky active threads when archiving is disabled. /// -/// Note: LIMIT -1 OFFSET ? is a SQLite-specific idiom — see `archive_old_threads`. +/// Returns safe file paths and a durable cleanup intent for filesystem replay. /// /// # Errors -/// Returns an error if the database operation fails. +/// Returns an error for invalid limits or a failed transaction. pub fn prune_old_threads( conn: &rusqlite::Connection, board_id: i64, max: i64, ) -> Result { - conn.execute_batch("BEGIN IMMEDIATE") - .context("Failed to begin prune_old_threads transaction")?; - - let result: Result = (|| { - // Collect ids inside the transaction to prevent concurrent bumps from - // changing the ordering between the SELECT and the DELETE. - let ids: Vec = { - let mut stmt = conn.prepare_cached( - "SELECT id FROM threads - WHERE board_id = ?1 AND sticky = 0 AND archived = 0 - ORDER BY bumped_at DESC LIMIT -1 OFFSET ?2", - )?; - let collected = stmt - .query_map(params![board_id, max], |r| r.get(0))? - .collect::>>()?; - collected - }; - - if ids.is_empty() { - return Ok(crate::db::DeletePathsResult { - paths: Vec::new(), - pending_fs_op_id: None, - }); - } - - // Collect all file paths in a single query BEFORE the DELETEs. - let candidates = collect_thread_file_paths(conn, &ids)?; - - // Single bulk DELETE instead of N individual statements. - let placeholders: String = ids - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i.saturating_add(1))) - .collect::>() - .join(", "); - let sql = format!("DELETE FROM threads WHERE id IN ({placeholders})"); - conn.execute(&sql, rusqlite::params_from_iter(&ids)) - .context("Failed to bulk delete pruned threads") - .map(|_affected_rows| ())?; - - // Determine safe paths INSIDE the transaction so the check sees the - // post-delete state before any concurrent writer can insert new references. - let safe = super::paths_safe_to_delete(conn, candidates)?; - let pending_fs_op = super::build_delete_files_pending_op(&safe)?; - if let Some(op) = pending_fs_op.as_ref() { - super::insert_pending_fs_op(conn, op)?; - } - Ok(crate::db::DeletePathsResult { - paths: safe, - pending_fs_op_id: pending_fs_op.map(|op| op.id), - }) - })(); - - match result { - Ok(result) => { - super::commit_transaction(conn, "Failed to commit prune_old_threads transaction")?; - Ok(result) - } - Err(e) => { - drop(conn.execute_batch("ROLLBACK")); - Err(e) - } - } + let tx = rusqlite::Transaction::new_unchecked(conn, rusqlite::TransactionBehavior::Immediate)?; + let result = prune_old_threads_in_tx(&tx, board_id, max, false)?; + tx.commit()?; + Ok(result) } -/// Hard-delete oldest archived threads that exceed the archive retention cap. -/// -/// Returns the on-disk paths that are now safe to remove. As with live-thread -/// pruning, the caller is responsible for deleting those files from disk. +/// Hard-delete archived overflow, keeping the first entries in archive order. /// -/// The ordering uses `bumped_at DESC`, matching the archive page and ensuring -/// we keep the most recently-active archived threads. +/// Retention uses bump time, not time of archival. Archived sticky threads are +/// subject to this cap too. File cleanup is durably scheduled before commit. /// /// # Errors -/// Returns an error if the transaction cannot be opened or committed, if the -/// candidate threads cannot be queried, or if the bulk delete/safe-path -/// calculation fails. +/// Returns an error for invalid limits or a failed transaction. pub fn prune_old_archived_threads( conn: &rusqlite::Connection, board_id: i64, max: i64, ) -> Result { - conn.execute_batch("BEGIN IMMEDIATE") - .context("Failed to begin prune_old_archived_threads transaction")?; - - let result: Result = (|| { - let ids: Vec = { - let mut stmt = conn.prepare_cached( - "SELECT id FROM threads - WHERE board_id = ?1 AND archived = 1 - ORDER BY bumped_at DESC LIMIT -1 OFFSET ?2", - )?; - let collected = stmt - .query_map(params![board_id, max], |r| r.get(0))? - .collect::>>()?; - collected - }; - - if ids.is_empty() { - return Ok(crate::db::DeletePathsResult { - paths: Vec::new(), - pending_fs_op_id: None, - }); - } - - let candidates = collect_thread_file_paths(conn, &ids)?; - - let placeholders: String = ids - .iter() - .enumerate() - .map(|(i, _)| format!("?{}", i.saturating_add(1))) - .collect::>() - .join(", "); - let sql = format!("DELETE FROM threads WHERE id IN ({placeholders})"); - conn.execute(&sql, rusqlite::params_from_iter(&ids)) - .context("Failed to bulk delete archived threads") - .map(|_affected_rows| ())?; + let tx = rusqlite::Transaction::new_unchecked(conn, rusqlite::TransactionBehavior::Immediate)?; + let result = prune_old_threads_in_tx(&tx, board_id, max, true)?; + tx.commit()?; + Ok(result) +} - let safe = super::paths_safe_to_delete(conn, candidates)?; - let pending_fs_op = super::build_delete_files_pending_op(&safe)?; - if let Some(op) = pending_fs_op.as_ref() { - super::insert_pending_fs_op(conn, op)?; - } - Ok(crate::db::DeletePathsResult { - paths: safe, - pending_fs_op_id: pending_fs_op.map(|op| op.id), - }) - })(); +/// Reject invalid destructive limits even when called outside the worker. +fn validate_retention_limit(max: i64, archived: bool) -> Result<()> { + let upper = if archived { 10_000 } else { 1_000 }; + anyhow::ensure!( + (1..=upper).contains(&max), + "invalid retention limit {max}; expected 1..={upper}; preserving content" + ); + Ok(()) +} - match result { - Ok(result) => { - super::commit_transaction( - conn, - "Failed to commit prune_old_archived_threads transaction", - )?; - Ok(result) - } - Err(e) => { - drop(conn.execute_batch("ROLLBACK")); - Err(e) - } +/// Select, delete, reference-check and journal files in the caller's transaction. +pub(crate) fn prune_old_threads_in_tx( + conn: &rusqlite::Transaction<'_>, + board_id: i64, + max: i64, + archived: bool, +) -> Result { + validate_retention_limit(max, archived)?; + // Archive ties historically ascend by ID. Active ties descend by ID, exactly + // matching the board index. Never let query planner choice decide retention. + let sql = if archived { + "SELECT id FROM threads WHERE board_id = ?1 AND archived = 1 + ORDER BY bumped_at DESC, id ASC LIMIT -1 OFFSET ?2" + } else { + "SELECT id FROM threads WHERE board_id = ?1 AND sticky = 0 AND archived = 0 + ORDER BY bumped_at DESC, id DESC LIMIT -1 OFFSET ?2" + }; + let ids = conn + .prepare_cached(sql)? + .query_map(params![board_id, max], |row| row.get::<_, i64>(0))? + .collect::>>()?; + let candidates = collect_thread_file_paths(conn, &ids)?; + for batch in ids.chunks(500) { + let placeholders = vec!["?"; batch.len()].join(", "); + conn.execute( + &format!("DELETE FROM threads WHERE id IN ({placeholders})"), + rusqlite::params_from_iter(batch), + ) + .context("Failed to delete overflow threads") + .map(|_affected_rows| ())?; + } + let safe = super::paths_safe_to_delete(conn, candidates)?; + let pending_fs_op = super::build_delete_files_pending_op(&safe)?; + if let Some(op) = pending_fs_op.as_ref() { + super::insert_pending_fs_op(conn, op)?; } + Ok(crate::db::DeletePathsResult { + paths: safe, + pending_fs_op_id: pending_fs_op.map(|op| op.id), + }) } // Archive listing @@ -1707,3 +1591,6 @@ mod tests { Ok(()) } } + +#[cfg(test)] +mod archive_tests; diff --git a/src/db/threads/archive_tests.rs b/src/db/threads/archive_tests.rs new file mode 100644 index 00000000..687268f7 --- /dev/null +++ b/src/db/threads/archive_tests.rs @@ -0,0 +1,574 @@ +//! Archive lifecycle regressions using real schema, transactions and file-backed WAL. + +use super::*; +use anyhow::ensure; +use rusqlite::Connection; + +fn fixture(conn: &Connection) -> Result { + crate::db::schema::install_or_migrate_schema(conn)?; + crate::db::create_board(conn, "archive", "Archive", "", false) +} + +fn thread(conn: &Connection, board: i64, bump: i64) -> Result { + let id = conn.query_row( + "INSERT INTO threads(board_id, subject, bumped_at) VALUES(?1, 'café ").to_owned(), + file_name: Some(concat!("unsafe\".webp").to_owned()), + body_html: crate::utils::sanitize::render_post_body( + &crate::utils::sanitize::escape_html(concat!( + ">quote\n>>1 [spoiler]hidden[/spoiler] alert(1)" + )), + false, + ), + audio_file_path: Some("test/companion.opus".to_owned()), + audio_file_name: Some("companion.opus".to_owned()), + audio_mime_type: Some("audio/ogg".to_owned()), + ..sample_post() + }; + let render = |archived| { + render_post( + &post, + "test", + "csrf", + RenderPostOpts { + show_delete: false, + is_admin: false, + admin_csrf_token: None, + show_media: true, + allow_editing: false, + allow_self_delete: false, + owned_post_controls: None, + show_poster_ids: false, + collapse_greentext: false, + thread_state: Some((false, archived, archived)), + thread_op_id: Some(1), + video_audio_muted: false, + }, + 0, + ) + }; + let active = render(false); + let archived = render(true); + assert_eq!( + active, archived, + "archive state must preserve reply media and markup" + ); + assert!( + !archived.contains(""), + false, + ); + assert!( + valid.contains("data-pid=\"12\""), + "leading zeros must resolve to a canonical post ID" + ); + assert!( + valid.contains("data-pid=\"9223372036854775807\""), + "largest supported post ID must remain representable" + ); + assert!( + !valid.contains("